Retry policy resolution when no tags found in database

Signed-off-by: Stefan Prodan <[email protected]>

Author: stefanprodan

internal/controller/imagepolicy_controller.go

@@ -28,7 +28,9 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
 	kerrors "k8s.io/apimachinery/pkg/util/errors"
+	"k8s.io/apimachinery/pkg/util/wait"
 	kuberecorder "k8s.io/client-go/tools/record"
+	"k8s.io/client-go/util/retry"
 	"k8s.io/client-go/util/workqueue"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/builder"
@@ -105,11 +107,12 @@ type ImagePolicyReconciler struct {
 	kuberecorder.EventRecorder
 	helper.Metrics
 
-	ControllerName    string
-	Database          DatabaseReader
-	ACLOptions        acl.Options
-	AuthOptionsGetter *registry.AuthOptionsGetter
-	TokenCache        *cache.TokenCache
+	ControllerName            string
+	Database                  DatabaseReader
+	ACLOptions                acl.Options
+	AuthOptionsGetter         *registry.AuthOptionsGetter
+	TokenCache                *cache.TokenCache
+	DependencyRequeueInterval time.Duration
 
 	patchOptions []patch.Option
 }
@@ -361,10 +364,12 @@ func (r *ImagePolicyReconciler) reconcile(ctx context.Context, sp *patch.SerialP
 			return
 		}
 
-		// If there's no tag in the database, mark not ready.
-		if err == errNoTagsInDatabase {
-			conditions.MarkFalse(obj, meta.ReadyCondition, imagev1.DependencyNotReadyReason, "%s", err)
-			result, retErr = ctrl.Result{}, nil
+		// If there's no tag in the database, mark not ready and
+		// requeue according to --requeue-dependency flag.
+		if errors.Is(err, errNoTagsInDatabase) {
+			depsErr := fmt.Errorf("retrying in %s error: %w", r.DependencyRequeueInterval.Round(time.Second), err)
+			conditions.MarkFalse(obj, meta.ReadyCondition, imagev1.DependencyNotReadyReason, "%s", depsErr)
+			result, retErr = ctrl.Result{RequeueAfter: r.DependencyRequeueInterval}, nil
 			return
 		}
 
@@ -510,15 +515,10 @@ func (r *ImagePolicyReconciler) applyPolicy(obj *imagev1.ImagePolicy, repo *imag
 		return "", errInvalidPolicy{err: fmt.Errorf("invalid policy: %w", err)}
 	}
 
-	// Read tags from database, apply and filter is configured and compute the
-	// result.
-	tags, err := r.Database.Tags(repo.Status.CanonicalImageName)
+	// Read tags from database with a maximum of 3 retries.
+	tags, err := r.listTagsWithBackoff(repo.Status.CanonicalImageName)
 	if err != nil {
-		return "", fmt.Errorf("failed to read tags from database: %w", err)
-	}
-
-	if len(tags) == 0 {
-		return "", errNoTagsInDatabase
+		return "", err
 	}
 
 	// Apply tag filter.
@@ -566,3 +566,32 @@ func (r *ImagePolicyReconciler) imagePoliciesForRepository(ctx context.Context,
 	}
 	return reqs
 }
+
+// listTagsWithBackoff lists the tags of the given image from the
+// internal database with retries if there are no tags in the database.
+func (r *ImagePolicyReconciler) listTagsWithBackoff(canonicalImageName string) ([]string, error) {
+	var backoff = wait.Backoff{
+		Steps:    4,
+		Duration: 1 * time.Second,
+		Factor:   1.5,
+		Jitter:   0.1,
+	}
+
+	var tags []string
+
+	err := retry.OnError(backoff, func(err error) bool {
+		return errors.Is(err, errNoTagsInDatabase)
+	}, func() error {
+		var err error
+		tags, err = r.Database.Tags(canonicalImageName)
+		if err != nil {
+			return fmt.Errorf("failed to read tags from database: %w", err)
+		}
+		if len(tags) == 0 {
+			return errNoTagsInDatabase
+		}
+		return nil
+	})
+
+	return tags, err
+}

internal/controller/imagepolicy_controller_test.go

@@ -98,7 +98,9 @@ func TestImagePolicyReconciler_imageRepoHasNoTags(t *testing.T) {
 		err := k8sClient.Get(ctx, client.ObjectKeyFromObject(imagePolicy), imagePolicy)
 		return err == nil && !conditions.IsReady(imagePolicy) &&
 			conditions.GetReason(imagePolicy, meta.ReadyCondition) == imagev1.DependencyNotReadyReason
-	}).Should(BeTrue())
+	}, 6*time.Second).Should(BeTrue())
+
+	g.Expect(conditions.GetMessage(imagePolicy, meta.ReadyCondition)).To(BeEquivalentTo("retrying in 30s error: no tags in database"))
 }
 
 func TestImagePolicyReconciler_ignoresImageRepoNotReadyEvent(t *testing.T) {
@@ -527,10 +529,9 @@ func TestImagePolicyReconciler_objectLevelWorkloadIdentityFeatureGate(t *testing
 
 		g.Eventually(func() bool {
 			err := k8sClient.Get(ctx, client.ObjectKeyFromObject(imagePolicy), imagePolicy)
-			logPolicyStatus(t, imagePolicy)
 			return err == nil && !conditions.IsReady(imagePolicy) &&
 				conditions.GetReason(imagePolicy, meta.ReadyCondition) == imagev1.DependencyNotReadyReason
-		}).Should(BeTrue())
+		}, 6*time.Second).Should(BeTrue())
 	})
 }
 

internal/controller/suite_test.go

@@ -49,7 +49,7 @@ import (
 
 // for Eventually
 const (
-	timeout                = time.Second * 30
+	timeout                = time.Second * 25
 	contextTimeout         = time.Second * 20
 	interval               = time.Second * 1
 	reconciliationInterval = time.Second * 2
@@ -101,10 +101,11 @@ func TestMain(m *testing.M) {
 	}
 
 	if err = (&ImagePolicyReconciler{
-		Client:            testEnv,
-		Database:          database.NewBadgerDatabase(testBadgerDB),
-		EventRecorder:     record.NewFakeRecorder(256),
-		AuthOptionsGetter: optGetter,
+		Client:                    testEnv,
+		Database:                  database.NewBadgerDatabase(testBadgerDB),
+		EventRecorder:             record.NewFakeRecorder(256),
+		AuthOptionsGetter:         optGetter,
+		DependencyRequeueInterval: 30 * time.Second,
 	}).SetupWithManager(testEnv, ImagePolicyReconcilerOptions{
 		RateLimiter: controller.GetDefaultRateLimiter(),
 	}); err != nil {

main.go

@@ -97,6 +97,7 @@ func main() {
 		featureGates            feathelper.FeatureGates
 		tokenCacheOptions       pkgcache.TokenFlags
 		defaultServiceAccount   string
+		requeueDependency       time.Duration
 	)
 
 	flag.StringVar(&metricsAddr, "metrics-addr", ":8080", "The address the metric endpoint binds to.")
@@ -106,6 +107,7 @@ func main() {
 	flag.Int64Var(&storageValueLogFileSize, "storage-value-log-file-size", 1<<28, "Set the database's memory mapped value log file size in bytes. Effective memory usage is about two times this size.")
 	flag.Uint16Var(&gcInterval, "gc-interval", 10, "The number of minutes to wait between garbage collections. 0 disables the garbage collector.")
 	flag.IntVar(&concurrent, "concurrent", 4, "The number of concurrent resource reconciles.")
+	flag.DurationVar(&requeueDependency, "requeue-dependency", 30*time.Second, "The interval at which failing dependencies are reevaluated.")
 
 	clientOptions.BindFlags(flag.CommandLine)
 	logOptions.BindFlags(flag.CommandLine)
@@ -283,14 +285,15 @@ func main() {
 		os.Exit(1)
 	}
 	if err := (&controller.ImagePolicyReconciler{
-		Client:            mgr.GetClient(),
-		EventRecorder:     eventRecorder,
-		Metrics:           metricsH,
-		Database:          db,
-		ACLOptions:        aclOptions,
-		ControllerName:    controllerName,
-		AuthOptionsGetter: authOptionsGetter,
-		TokenCache:        tokenCache,
+		Client:                    mgr.GetClient(),
+		EventRecorder:             eventRecorder,
+		Metrics:                   metricsH,
+		Database:                  db,
+		ACLOptions:                aclOptions,
+		ControllerName:            controllerName,
+		AuthOptionsGetter:         authOptionsGetter,
+		TokenCache:                tokenCache,
+		DependencyRequeueInterval: requeueDependency,
 	}).SetupWithManager(mgr, controller.ImagePolicyReconcilerOptions{
 		RateLimiter: helper.GetRateLimiter(rateLimiterOptions),
 	}); err != nil {