Merge branch 'dev' into main for job deployment.

See PR #10562.

Author: joelverhagen

NuGetGallery.sln

@@ -1,7 +1,7 @@
 
 Microsoft Visual Studio Solution File, Format Version 12.00
-# Visual Studio Version 17
-VisualStudioVersion = 17.8.34118.359
+# Visual Studio Version 18
+VisualStudioVersion = 18.0.10925.136
 MinimumVisualStudioVersion = 10.0.40219.1
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "2. Frontend", "2. Frontend", "{05998089-58F5-4A84-8C11-C5C6244A6F89}"
 EndProject

src/GitHubVulnerabilities2Db/Configuration/GitHubVulnerabilities2DbConfiguration.cs

@@ -1,4 +1,4 @@
-// Copyright (c) .NET Foundation. All rights reserved.
+// Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
 using NuGet.Services.GitHub.Configuration;
@@ -21,5 +21,10 @@ public class GitHubVulnerabilities2DbConfiguration : GraphQLQueryConfiguration
         /// The name of the blob to save the job's advisories cursor in.
         /// </summary>
         public string AdvisoryCursorBlobName { get; set; } = "cursor.json";
+
+        /// <summary>
+        /// The User-Agent header to send with each request to GitHub.
+        /// </summary>
+        public override string UserAgent { get; set; } = "NuGet.GitHubVulnerabilities2Db";
     }
-}
+}

src/GitHubVulnerabilities2v3/Configuration/GitHubVulnerabilities2v3Configuration.cs

@@ -1,4 +1,4 @@
-// Copyright (c) .NET Foundation. All rights reserved.
+// Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
 using NuGet.Services.GitHub.Configuration;
@@ -50,5 +50,10 @@ public class GitHubVulnerabilities2v3Configuration : GraphQLQueryConfiguration
         /// Enable/disable HTTP file compression.
         /// </summary>
         public bool GzipFileContent { get; set; } = true;
+
+        /// <summary>
+        /// The User-Agent header to send with each request to GitHub.
+        /// </summary>
+        public override string UserAgent { get; set; } = "NuGet.GitHubVulnerabilities2v3";
     }
-}
+}

src/NuGet.Services.GitHub/Configuration/GraphQLQueryConfiguration.cs

@@ -1,11 +1,11 @@
-// Copyright (c) .NET Foundation. All rights reserved.
+// Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
 using System;
 
 namespace NuGet.Services.GitHub.Configuration
 {
-    public class GraphQLQueryConfiguration
+    public abstract class GraphQLQueryConfiguration
     {
         /// <summary>
         /// GitHub's v4 GraphQL API endpoint.
@@ -16,5 +16,15 @@ public class GraphQLQueryConfiguration
         /// The personal access token to use to authenticate with GitHub.
         /// </summary>
         public string GitHubPersonalAccessToken { get; set; }
+
+        /// <summary>
+        /// The User-Agent header to send with each request to GitHub.
+        /// </summary>
+        public abstract string UserAgent { get; set; }
+
+        /// <summary>
+        /// Time to wait between retries when a request to GitHub fails.
+        /// </summary>
+        public TimeSpan RetryDelay { get; set; } = TimeSpan.FromSeconds(3);
     }
-}
+}

src/NuGet.Services.GitHub/GraphQL/QueryService.cs

@@ -1,34 +1,34 @@
-// Copyright (c) .NET Foundation. All rights reserved.
+// Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
 using System;
+using System.Net;
 using System.Net.Http;
 using System.Net.Http.Headers;
 using System.Text;
 using System.Threading;
 using System.Threading.Tasks;
-using NuGet.Services.GitHub.Configuration;
+using Microsoft.Extensions.Logging;
 using Newtonsoft.Json;
 using Newtonsoft.Json.Linq;
+using NuGet.Services.GitHub.Configuration;
 
 namespace NuGet.Services.GitHub.GraphQL
 {
     public class QueryService : IQueryService
     {
-        /// <remarks>
-        /// GitHub requires that every request includes a UserAgent.
-        /// </remarks>
-        public const string UserAgent = "NuGet.Services.GitHub";
-
         private readonly GraphQLQueryConfiguration _configuration;
         private readonly HttpClient _client;
+        private readonly ILogger<QueryService> _logger;
 
         public QueryService(
             GraphQLQueryConfiguration configuration,
-            HttpClient client)
+            HttpClient client,
+            ILogger<QueryService> logger)
         {
             _configuration = configuration ?? throw new ArgumentNullException(nameof(configuration));
             _client = client ?? throw new ArgumentNullException(nameof(client));
+            _logger = logger ?? throw new ArgumentNullException(nameof(logger));
         }
 
         public async Task<QueryResponse> QueryAsync(string query, CancellationToken token)
@@ -53,18 +53,32 @@ public async Task<QueryResponse> QueryAsync(string query, CancellationToken toke
 
         private async Task<string> MakeWebRequestAsync(string query, CancellationToken token)
         {
-            using (var request = CreateRequest(query))
-            using (var response = await _client.SendAsync(request, token))
+            var attempt = 0;
+            const int maxAttempts = 5;
+            HttpStatusCode? lastStatusCode = null;
+            while (true)
             {
-                var responseBody = await response.Content.ReadAsStringAsync();
-                if (!response.IsSuccessStatusCode)
+                try
                 {
-                    throw new InvalidOperationException(
-                        $"The GitHub GraphQL response returned status code {(int)response.StatusCode} {response.ReasonPhrase}. " +
-                        $"Response body:{Environment.NewLine}{responseBody}");
+                    attempt++;
+                    lastStatusCode = null;
+                    using (var request = CreateRequest(query))
+                    using (var response = await _client.SendAsync(request, token))
+                    {
+                        lastStatusCode = response.StatusCode;
+                        response.EnsureSuccessStatusCode();
+                        return await response.Content.ReadAsStringAsync();
+                    }
+                }
+                catch (Exception ex) when (
+                    !token.IsCancellationRequested
+                    && attempt < maxAttempts
+                    && (!lastStatusCode.HasValue || lastStatusCode >= HttpStatusCode.InternalServerError)) // do not retry for 4XX errors
+                {
+                    var delay = _configuration.RetryDelay;
+                    _logger.LogWarning(ex, "Failed attempt {Attempt} to query GitHub GraphQL API. Last HTTP status code: {Status}. Waiting {Seconds} seconds.", attempt, lastStatusCode, delay.TotalSeconds);
+                    await Task.Delay(delay, token);
                 }
-
-                return responseBody;
             }
         }
 
@@ -79,7 +93,7 @@ private HttpRequestMessage CreateRequest(string query)
 
             message.Headers.Authorization = new AuthenticationHeaderValue(
                 "Bearer", _configuration.GitHubPersonalAccessToken);
-            message.Headers.UserAgent.TryParseAdd(UserAgent);
+            message.Headers.UserAgent.TryParseAdd(_configuration.UserAgent);
             return message;
         }
     }

src/VerifyGitHubVulnerabilities/Configuration/VerifyGitHubVulnerabilitiesConfiguration.cs

@@ -1,22 +1,13 @@
-// Copyright (c) .NET Foundation. All rights reserved.
+// Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
 using System;
+using NuGet.Services.GitHub.Configuration;
 
 namespace VerifyGitHubVulnerabilities.Configuration
 {
-    public class VerifyGitHubVulnerabilitiesConfiguration
+    public class VerifyGitHubVulnerabilitiesConfiguration : GraphQLQueryConfiguration
     {
-        /// <summary>
-        /// GitHub's v4 GraphQL API endpoint.
-        /// </summary>
-        public Uri GitHubGraphQLQueryEndpoint { get; set; } = new Uri("https://api.github.com/graphql");
-
-        /// <summary>
-        /// The personal access token to use to authenticate with GitHub.
-        /// </summary>
-        public string GitHubPersonalAccessToken { get; set; }
-
         /// <summary>
         /// The v3 index URI string for fetching registration metadata
         /// </summary>
@@ -31,5 +22,10 @@ public class VerifyGitHubVulnerabilitiesConfiguration
         /// Whether to verify GitHubVulnerabilities in the registration blobs
         /// </summary>
         public bool VerifyRegistrationMetadata { get; set; } = true;
+
+        /// <summary>
+        /// The User-Agent header to send with each request to GitHub.
+        /// </summary>
+        public override string UserAgent { get; set; } = "NuGet.VerifyGitHubVulnerabilities";
     }
-}
+}

src/VerifyGitHubVulnerabilities/GraphQL/QueryService.cs

@@ -1,4 +1,4 @@
-// Copyright (c) .NET Foundation. All rights reserved.
+// Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
 using System;
@@ -7,19 +7,20 @@
 using System.Threading;
 using System.Threading.Tasks;
 using Autofac;
-using NuGet.Services.GitHub.Collector;
-using NuGet.Services.GitHub.GraphQL;
-using NuGet.Services.GitHub.Ingest;
+using GitHubVulnerabilities2Db.Gallery;
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
 using NuGet.Jobs;
 using NuGet.Jobs.Configuration;
+using NuGet.Services.GitHub.Collector;
+using NuGet.Services.GitHub.Configuration;
+using NuGet.Services.GitHub.GraphQL;
+using NuGet.Services.GitHub.Ingest;
 using NuGetGallery;
 using VerifyGitHubVulnerabilities.Configuration;
 using VerifyGitHubVulnerabilities.Verify;
-using GitHubVulnerabilities2Db.Gallery;
 
 namespace VerifyGitHubVulnerabilities
 {
@@ -60,6 +61,8 @@ protected override void ConfigureAutofacServices(ContainerBuilder containerBuild
         {
             containerBuilder
                 .RegisterAdapter<IOptionsSnapshot<VerifyGitHubVulnerabilitiesConfiguration>, VerifyGitHubVulnerabilitiesConfiguration>(c => c.Value);
+            containerBuilder
+                .RegisterAdapter<IOptionsSnapshot<VerifyGitHubVulnerabilitiesConfiguration>, GraphQLQueryConfiguration>(c => c.Value);
 
             ConfigureQueryServices(containerBuilder);
             ConfigureIngestionServices(containerBuilder);
@@ -110,7 +113,7 @@ protected void ConfigureQueryServices(ContainerBuilder containerBuilder)
                 .As<HttpClient>();
 
             containerBuilder
-                .RegisterType<VerifyGitHubVulnerabilities.GraphQL.QueryService>()
+                .RegisterType<QueryService>()
                 .As<IQueryService>();
 
             containerBuilder

src/VerifyGitHubVulnerabilities/Job.cs

@@ -4,6 +4,7 @@
   </PropertyGroup>
   <ItemGroup>
     <ProjectReference Include="..\..\src\AccountDeleter\AccountDeleter.csproj" />
+    <ProjectReference Include="..\NuGetGallery.Core.Facts\NuGetGallery.Core.Facts.csproj" />
   </ItemGroup>
   <ItemGroup>
     <PackageReference Include="Moq" />

tests/AccountDeleter.Facts/AccountDeleter.Facts.csproj

@@ -4,6 +4,8 @@
   </PropertyGroup>
   <ItemGroup>
     <ProjectReference Include="..\..\src\GitHubVulnerabilities2Db\GitHubVulnerabilities2Db.csproj" />
+    <ProjectReference Include="..\NuGetGallery.Core.Facts\NuGetGallery.Core.Facts.csproj" />
+    <ProjectReference Include="..\NuGetGallery.Facts\NuGetGallery.Facts.csproj" />
   </ItemGroup>
   <ItemGroup>
     <PackageReference Include="Moq" />