Skip to content

Commit 76f6677

Browse files
cuisongliucuisongliu
authored
feat(image-cri-shim): Dynamic Configuration Reloading (#6067)
* chore(image-cri-shim): align replacements and add license headers * chore(fmt): format code Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> * chore(default.yaml): add license headers to the configuration file Signed-off-by: cuisongliu <[email protected]> * feat(image-cri-shim): make reload heartbeat configurable * test(image-cri-shim): verify mirror config sync * feat(config): add name field to Registry and improve domain matching logic Signed-off-by: cuisongliu <[email protected]> * refactor(auth): remove unused skip login logic and simplify config retrieval Signed-off-by: cuisongliu <[email protected]> * feat(image-cri-shim): update registry mirror test with new image and address Signed-off-by: cuisongliu <[email protected]> * refactor(image-cri-shim): remove deferred cleanup for image service client Signed-off-by: cuisongliu <[email protected]> --------- Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Signed-off-by: cuisongliu <[email protected]> Co-authored-by: cuisongliu <[email protected]>
1 parent 5e54b6b commit 76f6677

File tree

17 files changed

+960
-150
lines changed

17 files changed

+960
-150
lines changed

‎lifecycle/cmd/image-cri-shim/cmd/root.go‎

Lines changed: 100 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -17,10 +17,14 @@ limitations under the License.
1717
package cmd
1818

1919
import (
20+
"context"
21+
"crypto/sha256"
22+
"encoding/hex"
2023
"fmt"
2124
"os"
2225
"os/signal"
2326
"syscall"
27+
"time"
2428

2529
"github.com/labring/image-cri-shim/pkg/shim"
2630
"github.com/labring/image-cri-shim/pkg/types"
@@ -68,7 +72,7 @@ func Execute() {
6872
}
6973

7074
func init() {
71-
rootCmd.Flags().StringVarP(&cfgFile, "file", "f", "", "image shim root config")
75+
rootCmd.Flags().StringVarP(&cfgFile, "file", "f", types.DefaultImageCRIShimConfig, "image shim root config")
7276
}
7377

7478
func run(cfg *types.Config, auth *types.ShimAuthConfig) {
@@ -78,6 +82,9 @@ func run(cfg *types.Config, auth *types.ShimAuthConfig) {
7882
logger.Fatal("failed to new image_shim, %s", err)
7983
}
8084

85+
ctx, cancel := context.WithCancel(context.Background())
86+
defer cancel()
87+
8188
err = imgShim.Setup()
8289
if err != nil {
8390
logger.Fatal("failed to setup image_shim, %s", err)
@@ -88,15 +95,101 @@ func run(cfg *types.Config, auth *types.ShimAuthConfig) {
8895
logger.Fatal(fmt.Sprintf("failed to start image_shim, %s", err))
8996
}
9097

98+
watchDone := make(chan struct{})
99+
go func() {
100+
defer close(watchDone)
101+
if err := watchAuthConfig(ctx, cfgFile, imgShim, cfg.ReloadInterval.Duration); err != nil {
102+
logger.Error("config watcher stopped with error: %v", err)
103+
}
104+
}()
105+
91106
signalCh := make(chan os.Signal, 1)
92107
signal.Notify(signalCh, syscall.SIGINT, syscall.SIGTERM)
93108

94-
stopCh := make(chan struct{}, 1)
95-
select {
96-
case <-signalCh:
97-
close(stopCh)
98-
case <-stopCh:
99-
}
109+
<-signalCh
110+
cancel()
111+
<-watchDone
112+
imgShim.Stop()
100113
_ = os.Remove(cfg.ImageShimSocket)
101114
logger.Info("shutting down the image_shim")
102115
}
116+
117+
func watchAuthConfig(ctx context.Context, path string, imgShim shim.Shim, interval time.Duration) error {
118+
if path == "" {
119+
logger.Warn("config file path is empty, skip dynamic auth reload")
120+
return nil
121+
}
122+
123+
logger.Info("start watching shim config: %s", path)
124+
if interval <= 0 {
125+
interval = types.DefaultReloadInterval
126+
}
127+
128+
lastHash := ""
129+
if data, err := os.ReadFile(path); err == nil {
130+
if cfg, err := types.UnmarshalData(data); err == nil {
131+
cfg.RegistryDir = types.NormalizeRegistryDir(cfg.RegistryDir)
132+
digest, err := types.RegistryDirDigest(cfg.RegistryDir)
133+
if err != nil {
134+
logger.Warn("failed to fingerprint registry.d %s: %v", cfg.RegistryDir, err)
135+
} else {
136+
sum := sha256.Sum256(append(data, digest...))
137+
lastHash = hex.EncodeToString(sum[:])
138+
}
139+
} else {
140+
logger.Warn("failed to parse shim config %s: %v", path, err)
141+
}
142+
}
143+
144+
currentInterval := interval
145+
// ticker acts like a pulse; changing the config lets us tune the heartbeat without restart.
146+
ticker := time.NewTicker(currentInterval)
147+
defer ticker.Stop()
148+
149+
for {
150+
select {
151+
case <-ctx.Done():
152+
return nil
153+
case <-ticker.C:
154+
data, err := os.ReadFile(path)
155+
if err != nil {
156+
logger.Warn("failed to read shim config %s: %v", path, err)
157+
continue
158+
}
159+
cfg, err := types.UnmarshalData(data)
160+
if err != nil {
161+
logger.Warn("failed to parse shim config %s: %v", path, err)
162+
continue
163+
}
164+
cfg.RegistryDir = types.NormalizeRegistryDir(cfg.RegistryDir)
165+
digest, err := types.RegistryDirDigest(cfg.RegistryDir)
166+
if err != nil {
167+
logger.Warn("failed to fingerprint registry.d %s: %v", cfg.RegistryDir, err)
168+
continue
169+
}
170+
sum := sha256.Sum256(append(data, digest...))
171+
hash := hex.EncodeToString(sum[:])
172+
if hash == lastHash {
173+
continue
174+
}
175+
auth, err := cfg.PreProcess()
176+
if err != nil {
177+
logger.Warn("failed to preprocess shim config %s: %v", path, err)
178+
continue
179+
}
180+
imgShim.UpdateAuth(auth)
181+
lastHash = hash
182+
logger.Info("reloaded shim auth configuration from %s", path)
183+
newInterval := cfg.ReloadInterval.Duration
184+
if newInterval <= 0 {
185+
newInterval = types.DefaultReloadInterval
186+
}
187+
if newInterval != currentInterval {
188+
ticker.Stop()
189+
ticker = time.NewTicker(newInterval)
190+
currentInterval = newInterval
191+
logger.Info("updated reload interval to %s", newInterval)
192+
}
193+
}
194+
}
195+
}

‎lifecycle/cmd/image-cri-shim/cmd/root_test.go‎

Lines changed: 163 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,163 @@
1+
/*
2+
Copyright 2024 [email protected].
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
https://github.com/labring/sealos/blob/main/LICENSE.md
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package cmd
18+
19+
import (
20+
"context"
21+
"fmt"
22+
"os"
23+
"path/filepath"
24+
"testing"
25+
"time"
26+
27+
"github.com/labring/image-cri-shim/pkg/types"
28+
)
29+
30+
type fakeShim struct {
31+
updates chan *types.ShimAuthConfig
32+
}
33+
34+
func newFakeShim() *fakeShim {
35+
return &fakeShim{updates: make(chan *types.ShimAuthConfig, 1)}
36+
}
37+
38+
func (f *fakeShim) Setup() error { return nil }
39+
40+
func (f *fakeShim) Start() error { return nil }
41+
42+
func (f *fakeShim) Stop() {}
43+
44+
func (f *fakeShim) UpdateAuth(auth *types.ShimAuthConfig) {
45+
select {
46+
case f.updates <- auth:
47+
default:
48+
}
49+
}
50+
51+
func TestWatchAuthConfigReloads(t *testing.T) {
52+
dir := t.TempDir()
53+
cfgPath := filepath.Join(dir, "shim-config.yaml")
54+
registryDir := filepath.Join(dir, "registry.d")
55+
56+
if err := os.MkdirAll(registryDir, 0o755); err != nil {
57+
t.Fatalf("failed to create registry.d directory: %v", err)
58+
}
59+
60+
mirrorPath := filepath.Join(registryDir, "mirror.yaml")
61+
if err := os.WriteFile(mirrorPath, []byte("address: \"https://mirror.example.com\"\nauth: \"user:pass\"\n"), 0o644); err != nil {
62+
t.Fatalf("failed to write mirror registry config: %v", err)
63+
}
64+
if err := os.WriteFile(filepath.Join(registryDir, "public.yaml"), []byte("address: \"https://public.example.com\"\n"), 0o644); err != nil {
65+
t.Fatalf("failed to write public registry config: %v", err)
66+
}
67+
68+
initialConfig := []byte(fmt.Sprintf(`shim: "/tmp/test.sock"
69+
cri: "/var/run/containerd/containerd.sock"
70+
address: "https://example.com"
71+
force: true
72+
auth: "offline:initial"
73+
reloadInterval: 10ms
74+
registry.d: %q
75+
`, registryDir))
76+
77+
if err := os.WriteFile(cfgPath, initialConfig, 0o644); err != nil {
78+
t.Fatalf("failed to write initial config: %v", err)
79+
}
80+
81+
shim := newFakeShim()
82+
ctx, cancel := context.WithCancel(context.Background())
83+
defer cancel()
84+
85+
done := make(chan error, 1)
86+
go func() {
87+
done <- watchAuthConfig(ctx, cfgPath, shim, 10*time.Millisecond)
88+
}()
89+
90+
time.Sleep(20 * time.Millisecond)
91+
92+
updatedConfig := []byte(fmt.Sprintf(`shim: "/tmp/test.sock"
93+
cri: "/var/run/containerd/containerd.sock"
94+
address: "https://example.com"
95+
force: true
96+
auth: "offline:updated"
97+
reloadInterval: 10ms
98+
registry.d: %q
99+
`, registryDir))
100+
101+
if err := os.WriteFile(cfgPath, updatedConfig, 0o644); err != nil {
102+
t.Fatalf("failed to write updated config: %v", err)
103+
}
104+
105+
var auth *types.ShimAuthConfig
106+
select {
107+
case auth = <-shim.updates:
108+
case <-time.After(2 * time.Second):
109+
t.Fatal("timed out waiting for auth update")
110+
}
111+
112+
offline, ok := auth.OfflineCRIConfigs["example.com"]
113+
if !ok {
114+
t.Fatalf("expected offline auth for example.com, got %#v", auth.OfflineCRIConfigs)
115+
}
116+
if offline.Password != "updated" {
117+
t.Fatalf("expected updated password, got %q", offline.Password)
118+
}
119+
120+
mirror, ok := auth.CRIConfigs["mirror.example.com"]
121+
if !ok {
122+
t.Fatalf("expected registry credentials for mirror.example.com, got %#v", auth.CRIConfigs)
123+
}
124+
if mirror.Password != "pass" {
125+
t.Fatalf("expected initial password from registry.d, got %q", mirror.Password)
126+
}
127+
128+
if !auth.SkipLoginRegistries["public.example.com"] {
129+
t.Fatalf("expected public.example.com to skip login, got %#v", auth.SkipLoginRegistries)
130+
}
131+
if cfg, ok := auth.CRIConfigs["public.example.com"]; !ok || cfg.Username != "" || cfg.Password != "" {
132+
t.Fatalf("expected public.example.com to use anonymous credentials, got %#v", cfg)
133+
}
134+
135+
if err := os.WriteFile(mirrorPath, []byte("address: \"https://mirror.example.com\"\nauth: \"user:changed\"\n"), 0o644); err != nil {
136+
t.Fatalf("failed to write updated mirror registry config: %v", err)
137+
}
138+
139+
select {
140+
case auth = <-shim.updates:
141+
case <-time.After(2 * time.Second):
142+
t.Fatal("timed out waiting for registry.d auth update")
143+
}
144+
145+
mirror, ok = auth.CRIConfigs["mirror.example.com"]
146+
if !ok {
147+
t.Fatalf("expected registry credentials for mirror.example.com after update, got %#v", auth.CRIConfigs)
148+
}
149+
if mirror.Password != "changed" {
150+
t.Fatalf("expected updated password from registry.d, got %q", mirror.Password)
151+
}
152+
153+
cancel()
154+
155+
select {
156+
case err := <-done:
157+
if err != nil {
158+
t.Fatalf("watcher exited with error: %v", err)
159+
}
160+
case <-time.After(2 * time.Second):
161+
t.Fatal("watcher did not exit after context cancel")
162+
}
163+
}

‎lifecycle/go.mod‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ require (
1515
github.com/containers/ocicrypt v1.1.7
1616
github.com/containers/storage v1.50.2
1717
github.com/davecgh/go-spew v1.1.1
18-
github.com/docker/docker v25.0.6+incompatible
18+
github.com/docker/docker v25.0.1+incompatible
1919
github.com/docker/go-units v0.5.0
2020
github.com/emicklei/go-restful/v3 v3.11.0
2121
github.com/emirpasic/gods v1.18.1

‎lifecycle/staging/src/github.com/labring/image-cri-shim/go.mod‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,4 +66,12 @@ require (
6666

6767
replace github.com/labring/sealos => ../../../../../
6868

69+
replace github.com/docker/docker => github.com/docker/docker v25.0.1+incompatible
70+
71+
replace gotest.tools/v3 => gotest.tools/v3 v3.5.0
72+
73+
replace go.uber.org/goleak => go.uber.org/goleak v1.3.0
74+
75+
replace github.com/containers/common => github.com/containers/common v0.53.1-0.20230613173441-e1ea4d9a74e5
76+
6977
replace k8s.io/endpointslice => k8s.io/endpointslice v0.30.3

‎lifecycle/staging/src/github.com/labring/image-cri-shim/go.sum‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,8 @@ github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I=
1919
github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo=
2020
github.com/containerd/stargz-snapshotter/estargz v0.14.3 h1:OqlDCK3ZVUO6C3B/5FSkDwbkEETK84kQgEeFwDC+62k=
2121
github.com/containerd/stargz-snapshotter/estargz v0.14.3/go.mod h1:KY//uOCIkSuNAHhJogcZtrNHdKrA99/FCCRjE3HD36o=
22-
github.com/containers/common v0.53.0 h1:Ax814cLeX5VXSnkKUdxz762g+27fJj1st4UvKoXmkKs=
23-
github.com/containers/common v0.53.0/go.mod h1:pABPxJwlTE8oYk9/2BW0e0mumkuhJHIPsABHTGRXN3w=
22+
github.com/containers/common v0.53.1-0.20230613173441-e1ea4d9a74e5 h1:Lc5zOwO6+G/OItXPt4sF1DnE/UAGygiDuVKWW5bqplw=
23+
github.com/containers/common v0.53.1-0.20230613173441-e1ea4d9a74e5/go.mod h1:F+dtzPF95PXAvc6Rxat7h3PVdBTvifOeBS+tQE/fiNw=
2424
github.com/containers/image/v5 v5.25.1-0.20230605120906-abe51339f34d h1:rNGRwMeck1s0/IPWFT7hN1WJ0R/vWP+ikueMTf8rmlI=
2525
github.com/containers/image/v5 v5.25.1-0.20230605120906-abe51339f34d/go.mod h1:52TGKzziLzutVX0cskgbWjQOzTDCHGPUEQH7/N2Ofnc=
2626
github.com/containers/libtrust v0.0.0-20230121012942-c1716e8a8d01 h1:Qzk5C6cYglewc+UyGf6lc8Mj2UaPTHy/iF2De0/77CA=
@@ -42,8 +42,8 @@ github.com/docker/cli v25.0.1+incompatible h1:mFpqnrS6Hsm3v1k7Wa/BO23oz0k121MTbT
4242
github.com/docker/cli v25.0.1+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
4343
github.com/docker/distribution v2.8.3+incompatible h1:AtKxIZ36LoNK51+Z6RpzLpddBirtxJnzDrHLEKxTAYk=
4444
github.com/docker/distribution v2.8.3+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
45-
github.com/docker/docker v25.0.6+incompatible h1:5cPwbwriIcsua2REJe8HqQV+6WlWc1byg2QSXzBxBGg=
46-
github.com/docker/docker v25.0.6+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
45+
github.com/docker/docker v25.0.1+incompatible h1:k5TYd5rIVQRSqcTwCID+cyVA0yRg86+Pcrz1ls0/frA=
46+
github.com/docker/docker v25.0.1+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
4747
github.com/docker/docker-credential-helpers v0.7.0 h1:xtCHsjxogADNZcdv1pKUHXryefjlVRqWqIhk/uXJp0A=
4848
github.com/docker/docker-credential-helpers v0.7.0/go.mod h1:rETQfLdHNT3foU5kuNkFR1R1V12OJRRO5lzt2D1b5X0=
4949
github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c=
@@ -219,8 +219,8 @@ go.opentelemetry.io/otel/metric v1.19.0 h1:aTzpGtV0ar9wlV4Sna9sdJyII5jTVJEvKETPi
219219
go.opentelemetry.io/otel/metric v1.19.0/go.mod h1:L5rUsV9kM1IxCj1MmSdS+JQAcVm319EUrDVLrt7jqt8=
220220
go.opentelemetry.io/otel/trace v1.19.0 h1:DFVQmlVbfVeOuBRrwdtaehRrWiL1JoVs9CPIQ1Dzxpg=
221221
go.opentelemetry.io/otel/trace v1.19.0/go.mod h1:mfaSyvGyEJEI0nyV2I4qhNQnbBOUUmYZpYojqMnX2vo=
222-
go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk=
223-
go.uber.org/goleak v1.2.0/go.mod h1:XJYK+MuIchqpmGmUSAzotztawfKvYLUIgg7guXrwVUo=
222+
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
223+
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
224224
go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
225225
go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
226226
go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo=
@@ -294,8 +294,8 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C
294294
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
295295
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
296296
gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo=
297-
gotest.tools/v3 v3.4.0 h1:ZazjZUfuVeZGLAmlKKuyv3IKP5orXcwtOwDQH6YVr6o=
298-
gotest.tools/v3 v3.4.0/go.mod h1:CtbdzLSsqVhDgMtKsx03ird5YTGB3ar27v0u/yKBW5g=
297+
gotest.tools/v3 v3.5.0 h1:Ljk6PdHdOhAb5aDMWXjDLMMhph+BpztA4v1QdqEW2eY=
298+
gotest.tools/v3 v3.5.0/go.mod h1:isy3WKz7GK6uNw/sbHzfKBLvlvXwUyV06n6brMxxopU=
299299
k8s.io/apimachinery v0.30.3 h1:q1laaWCmrszyQuSQCfNB8cFgCuDAoPszKY4ucAjDwHc=
300300
k8s.io/apimachinery v0.30.3/go.mod h1:iexa2somDaxdnj7bha06bhb43Zpa6eWH8N8dbqVjTUc=
301301
k8s.io/cri-api v0.30.3 h1:o7AAGb3645Ik44WkHI0eqUc7JbQVmstlINLlLAtU/rI=

0 commit comments

Comments
 (0)