Skip to content

Commit 159b16d

Browse files
p-sep-se
authored
Support for setting priority of agent (#4057)
Refers to #3491
1 parent bb4f73b commit 159b16d

File tree

11 files changed

+705
-8
lines changed

11 files changed

+705
-8
lines changed

‎charts/fleet-crd/templates/crds.yaml‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5614,6 +5614,25 @@ spec:
56145614
More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
56155615
type: object
56165616
type: object
5617+
agentSchedulingCustomization:
5618+
properties:
5619+
podDisruptionBudget:
5620+
properties:
5621+
maxUnavailable:
5622+
type: string
5623+
minAvailable:
5624+
type: string
5625+
type: object
5626+
priorityClass:
5627+
properties:
5628+
preemptionPolicy:
5629+
description: PreemptionPolicy describes a policy for if/when
5630+
to preempt a pod.
5631+
type: string
5632+
value:
5633+
type: integer
5634+
type: object
5635+
type: object
56175636
agentTolerations:
56185637
description: AgentTolerations defines an extra set of Tolerations
56195638
to be added to the Agent deployment.
@@ -5801,6 +5820,8 @@ spec:
58015820
used to detect changes.'
58025821
nullable: true
58035822
type: string
5823+
agentSchedulingCustomizationHash:
5824+
type: string
58045825
agentTLSMode:
58055826
description: 'AgentTLSMode supports two values: `system-store` and
58065827
`strict`. If set to

‎e2e/single-cluster/agent_scheduling_customization_test.go‎

Lines changed: 268 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,268 @@
1+
package singlecluster_test
2+
3+
import (
4+
"context"
5+
"strings"
6+
7+
. "github.com/onsi/ginkgo/v2"
8+
. "github.com/onsi/gomega"
9+
fleet "github.com/rancher/fleet/pkg/apis/fleet.cattle.io/v1alpha1"
10+
corev1 "k8s.io/api/core/v1"
11+
schedulingv1 "k8s.io/api/scheduling/v1"
12+
"k8s.io/apimachinery/pkg/api/errors"
13+
"sigs.k8s.io/controller-runtime/pkg/client"
14+
)
15+
16+
func ptr[T any](v T) *T {
17+
return &v
18+
}
19+
20+
var _ = Describe("Agent Scheduling Customization", func() {
21+
var (
22+
cluster *fleet.Cluster
23+
)
24+
25+
BeforeEach(func() {
26+
var err error
27+
cluster = &fleet.Cluster{}
28+
err = clientUpstream.Get(context.TODO(), client.ObjectKey{
29+
Namespace: env.Namespace,
30+
Name: "local",
31+
}, cluster)
32+
Expect(err).ToNot(HaveOccurred())
33+
})
34+
35+
When("agentSchedulingCustomization.PriorityClass is configured on the cluster resource", func() {
36+
BeforeEach(func() {
37+
// Update the cluster with agentSchedulingCustomization
38+
Eventually(func(g Gomega) {
39+
// Re-fetch the cluster to get the latest version
40+
latestCluster := &fleet.Cluster{}
41+
err := clientUpstream.Get(context.TODO(), client.ObjectKey{
42+
Namespace: env.Namespace,
43+
Name: "local",
44+
}, latestCluster)
45+
g.Expect(err).ToNot(HaveOccurred())
46+
47+
latestCluster.Spec.AgentSchedulingCustomization = &fleet.AgentSchedulingCustomization{
48+
PriorityClass: &fleet.PriorityClassSpec{
49+
Value: 1000,
50+
PreemptionPolicy: ptr(corev1.PreemptNever),
51+
},
52+
}
53+
54+
err = clientUpstream.Update(context.TODO(), latestCluster)
55+
g.Expect(err).ToNot(HaveOccurred())
56+
}).Should(Succeed(), "Should be able to update cluster with agentSchedulingCustomization")
57+
})
58+
59+
AfterEach(func() {
60+
// Clean up by removing the agentSchedulingCustomization
61+
Eventually(func(g Gomega) {
62+
// Re-fetch the cluster to get the latest version
63+
latestCluster := &fleet.Cluster{}
64+
err := clientUpstream.Get(context.TODO(), client.ObjectKey{
65+
Namespace: env.Namespace,
66+
Name: "local",
67+
}, latestCluster)
68+
g.Expect(err).ToNot(HaveOccurred())
69+
70+
latestCluster.Spec.AgentSchedulingCustomization = nil
71+
err = clientUpstream.Update(context.TODO(), latestCluster)
72+
g.Expect(err).ToNot(HaveOccurred())
73+
}).Should(Succeed(), "Should be able to update cluster to remove agentSchedulingCustomization")
74+
75+
Eventually(func(g Gomega) {
76+
latestCluster := &fleet.Cluster{}
77+
err := clientUpstream.Get(context.TODO(), client.ObjectKey{
78+
Namespace: env.Namespace,
79+
Name: "local",
80+
}, latestCluster)
81+
g.Expect(err).ToNot(HaveOccurred())
82+
g.Expect(cluster.Status.AgentSchedulingCustomizationHash).To(Equal(""))
83+
}).Should(Succeed(), "Should have cleared the AgentSchedulingCustomizationHash")
84+
85+
// Wait for PriorityClass to be deleted
86+
Eventually(func() bool {
87+
pc := &schedulingv1.PriorityClass{}
88+
err := clientUpstream.Get(context.TODO(), client.ObjectKey{
89+
Name: "fleet-agent-priority-class",
90+
}, pc)
91+
return errors.IsNotFound(err)
92+
}).Should(BeTrue(), "PriorityClass should be deleted")
93+
})
94+
95+
It("should create a PriorityClass on the cluster", func() {
96+
By("waiting for the PriorityClass to be created")
97+
Eventually(func(g Gomega) {
98+
pc := &schedulingv1.PriorityClass{}
99+
err := clientUpstream.Get(context.TODO(), client.ObjectKey{
100+
Name: "fleet-agent-priority-class",
101+
}, pc)
102+
g.Expect(err).ToNot(HaveOccurred(), "PriorityClass should be created")
103+
104+
g.Expect(pc.Value).To(Equal(int32(1000)), "PriorityClass should have correct value")
105+
g.Expect(pc.PreemptionPolicy).ToNot(BeNil())
106+
g.Expect(*pc.PreemptionPolicy).To(Equal(corev1.PreemptNever), "PriorityClass should have correct preemption policy")
107+
g.Expect(pc.Description).To(Equal("Priority class for Fleet Agent"))
108+
}).Should(Succeed())
109+
110+
By("checking that the agent deployment uses the priority class")
111+
k := env.Kubectl.Namespace("cattle-fleet-local-system")
112+
Eventually(func(g Gomega) {
113+
out, err := k.Get("deployment", "fleet-agent", "-o", "jsonpath={.spec.template.spec.priorityClassName}")
114+
g.Expect(err).ToNot(HaveOccurred())
115+
priorityClassName := strings.TrimSpace(out)
116+
g.Expect(priorityClassName).To(Equal("fleet-agent-priority-class"))
117+
}).Should(Succeed())
118+
})
119+
120+
It("should update cluster status hash when agentSchedulingCustomization changes", func() {
121+
By("checking that agentSchedulingCustomizationHash is set in cluster status")
122+
Eventually(func(g Gomega) {
123+
updatedCluster := &fleet.Cluster{}
124+
err := clientUpstream.Get(context.TODO(), client.ObjectKey{
125+
Namespace: env.Namespace,
126+
Name: "local",
127+
}, updatedCluster)
128+
g.Expect(err).ToNot(HaveOccurred())
129+
g.Expect(updatedCluster.Status.AgentSchedulingCustomizationHash).ToNot(BeEmpty())
130+
}).Should(Succeed())
131+
})
132+
})
133+
134+
When("agentSchedulingCustomization.PodDisruptionBudget is configured on local cluster", func() {
135+
BeforeEach(func() {
136+
// Update the cluster with agentSchedulingCustomization for PDB
137+
Eventually(func(g Gomega) {
138+
// Re-fetch the cluster to get the latest version
139+
latestCluster := &fleet.Cluster{}
140+
err := clientUpstream.Get(context.TODO(), client.ObjectKey{
141+
Namespace: env.Namespace,
142+
Name: "local",
143+
}, latestCluster)
144+
g.Expect(err).ToNot(HaveOccurred())
145+
146+
latestCluster.Spec.AgentSchedulingCustomization = &fleet.AgentSchedulingCustomization{
147+
PodDisruptionBudget: &fleet.PodDisruptionBudgetSpec{
148+
MaxUnavailable: "1",
149+
},
150+
}
151+
152+
err = clientUpstream.Update(context.TODO(), latestCluster)
153+
g.Expect(err).ToNot(HaveOccurred())
154+
}).Should(Succeed(), "Should be able to update cluster with agentSchedulingCustomization")
155+
})
156+
157+
AfterEach(func() {
158+
// Clean up by removing the agentSchedulingCustomization
159+
Eventually(func(g Gomega) {
160+
// Re-fetch the cluster to get the latest version
161+
latestCluster := &fleet.Cluster{}
162+
err := clientUpstream.Get(context.TODO(), client.ObjectKey{
163+
Namespace: env.Namespace,
164+
Name: "local",
165+
}, latestCluster)
166+
g.Expect(err).ToNot(HaveOccurred())
167+
168+
latestCluster.Spec.AgentSchedulingCustomization = nil
169+
err = clientUpstream.Update(context.TODO(), latestCluster)
170+
g.Expect(err).ToNot(HaveOccurred())
171+
}).Should(Succeed(), "Should be able to update cluster to remove agentSchedulingCustomization")
172+
173+
Eventually(func(g Gomega) {
174+
latestCluster := &fleet.Cluster{}
175+
err := clientUpstream.Get(context.TODO(), client.ObjectKey{
176+
Namespace: env.Namespace,
177+
Name: "local",
178+
}, latestCluster)
179+
g.Expect(err).ToNot(HaveOccurred())
180+
g.Expect(cluster.Status.AgentSchedulingCustomizationHash).To(Equal(""))
181+
}).Should(Succeed(), "Should have cleared the AgentSchedulingCustomizationHash")
182+
})
183+
184+
It("should create a PodDisruptionBudget on the cluster", func() {
185+
By("waiting for the PodDisruptionBudget to be created")
186+
k := env.Kubectl.Namespace("cattle-fleet-local-system")
187+
Eventually(func(g Gomega) {
188+
out, err := k.Get("pdb", "fleet-agent-pod-disruption-budget", "-o", "jsonpath={.spec.maxUnavailable}")
189+
g.Expect(err).ToNot(HaveOccurred(), "PodDisruptionBudget should be created")
190+
maxUnavailable := strings.TrimSpace(out)
191+
g.Expect(maxUnavailable).To(Equal("1"), "PodDisruptionBudget should have correct maxUnavailable")
192+
}).Should(Succeed())
193+
194+
By("checking that the PDB has correct selector")
195+
Eventually(func(g Gomega) {
196+
out, err := k.Get("pdb", "fleet-agent-pod-disruption-budget", "-o", "jsonpath={.spec.selector.matchLabels.app}")
197+
g.Expect(err).ToNot(HaveOccurred())
198+
app := strings.TrimSpace(out)
199+
g.Expect(app).To(Equal("fleet-agent"))
200+
}).Should(Succeed())
201+
})
202+
})
203+
204+
When("both PriorityClass and PodDisruptionBudget are configured", func() {
205+
BeforeEach(func() {
206+
// Update the cluster with both configurations
207+
Eventually(func(g Gomega) {
208+
// Re-fetch the cluster to get the latest version
209+
latestCluster := &fleet.Cluster{}
210+
err := clientUpstream.Get(context.TODO(), client.ObjectKey{
211+
Namespace: env.Namespace,
212+
Name: "local",
213+
}, latestCluster)
214+
g.Expect(err).ToNot(HaveOccurred())
215+
216+
latestCluster.Spec.AgentSchedulingCustomization = &fleet.AgentSchedulingCustomization{
217+
PriorityClass: &fleet.PriorityClassSpec{
218+
Value: 500,
219+
},
220+
PodDisruptionBudget: &fleet.PodDisruptionBudgetSpec{
221+
MinAvailable: "1",
222+
},
223+
}
224+
225+
err = clientUpstream.Update(context.TODO(), latestCluster)
226+
g.Expect(err).ToNot(HaveOccurred())
227+
}).Should(Succeed(), "Should be able to update cluster with agentSchedulingCustomization")
228+
})
229+
230+
AfterEach(func() {
231+
// Clean up by removing the agentSchedulingCustomization
232+
Eventually(func(g Gomega) {
233+
// Re-fetch the cluster to get the latest version
234+
latestCluster := &fleet.Cluster{}
235+
err := clientUpstream.Get(context.TODO(), client.ObjectKey{
236+
Namespace: env.Namespace,
237+
Name: "local",
238+
}, latestCluster)
239+
g.Expect(err).ToNot(HaveOccurred())
240+
241+
latestCluster.Spec.AgentSchedulingCustomization = nil
242+
err = clientUpstream.Update(context.TODO(), latestCluster)
243+
g.Expect(err).ToNot(HaveOccurred())
244+
}).Should(Succeed(), "Should be able to update cluster to remove agentSchedulingCustomization")
245+
})
246+
247+
It("should create both PriorityClass and PodDisruptionBudget", func() {
248+
By("checking PriorityClass is created")
249+
Eventually(func(g Gomega) {
250+
pc := &schedulingv1.PriorityClass{}
251+
err := clientUpstream.Get(context.TODO(), client.ObjectKey{
252+
Name: "fleet-agent-priority-class",
253+
}, pc)
254+
g.Expect(err).ToNot(HaveOccurred())
255+
g.Expect(pc.Value).To(Equal(int32(500)))
256+
}).Should(Succeed())
257+
258+
By("checking PodDisruptionBudget is created")
259+
k := env.Kubectl.Namespace("cattle-fleet-local-system")
260+
Eventually(func(g Gomega) {
261+
out, err := k.Get("pdb", "fleet-agent-pod-disruption-budget", "-o", "jsonpath={.spec.minAvailable}")
262+
g.Expect(err).ToNot(HaveOccurred())
263+
minAvailable := strings.TrimSpace(out)
264+
g.Expect(minAvailable).To(Equal("1"))
265+
}).Should(Succeed())
266+
})
267+
})
268+
})

‎internal/cmd/controller/agentmanagement/agent/manifest.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@ type ManifestOptions struct {
4444
BundleDeploymentWorkers string
4545
DriftWorkers string
4646
cmd.LeaderElectionOptions
47+
PriorityClassName string
4748
}
4849

4950
// Manifest builds and returns a deployment manifest for the fleet-agent with a
@@ -165,6 +166,7 @@ func agentApp(namespace string, agentScope string, opts ManifestOptions) *appsv1
165166
},
166167
},
167168
Spec: corev1.PodSpec{
169+
PriorityClassName: opts.PriorityClassName,
168170
ServiceAccountName: serviceAccount,
169171
Containers: []corev1.Container{
170172
{

‎internal/cmd/controller/agentmanagement/agent/manifest_test.go‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -302,3 +302,30 @@ func TestManifestAgentResources(t *testing.T) {
302302
})
303303
}
304304
}
305+
306+
func TestPriorityClassName(t *testing.T) {
307+
tests := []struct {
308+
name string
309+
priorityClassName string
310+
}{
311+
{
312+
name: "empty priorityClassName",
313+
priorityClassName: "",
314+
},
315+
{
316+
name: "priorityClassName specified",
317+
priorityClassName: "foo",
318+
},
319+
}
320+
321+
for _, test := range tests {
322+
t.Run(test.name, func(t *testing.T) {
323+
d := getAgentFromManifests("test-scope", agent.ManifestOptions{
324+
PriorityClassName: test.priorityClassName,
325+
})
326+
if d.Spec.Template.Spec.PriorityClassName != test.priorityClassName {
327+
t.Fatalf("expected PriorityClassName to be %s, got %s", test.priorityClassName, d.Spec.Template.Spec.PriorityClassName)
328+
}
329+
})
330+
}
331+
}

0 commit comments

Comments
 (0)