Merge pull request #645 from internetarchive/deps-3.8

Update dependencies and fix some warnings for 3.8.0 release

Author: ato

CHANGELOG.md

@@ -4,6 +4,35 @@
 
 [Full Changelog](https://github.com/internetarchive/heritrix3/compare/3.6.0...HEAD)
 
+## 3.8.0
+
+[Download distribution zip](https://repo1.maven.org/maven2/org/archive/heritrix/heritrix/3.8.0/heritrix-3.8.0-dist.zip) (or [tar.gz](https://repo1.maven.org/maven2/org/archive/heritrix/heritrix/3.8.0/heritrix-3.8.0-dist.tar.gz))
+
+[Full Changelog](https://github.com/internetarchive/heritrix3/compare/3.7.0...3.8.0) | [Javadoc](https://www.javadoc.io/doc/org.archive.heritrix/heritrix-engine/3.8.0/index.html) | [Maven Central](https://search.maven.org/artifact/org.archive.heritrix/heritrix/3.8.0/pom)
+
+#### New Features
+
+- **ExtractorYoutubeDL processArguments**: New option for overriding the default `yt-dlp` process arguments. [#644](https://github.com/internetarchive/heritrix3/pull/644)
+
+#### Fixes
+
+- **Slow tests**: Fixed `ObjectIdentityBdbManualCacheTest` so it no longer fails when running tests with `-DrunSlowTests=true`.
+- **Test stability**: Disabled `FetchHTTPTest.testHostHeaderDefaultPort` due to sporadic test failures.
+- **Code cleanup**: Fixed some compiler and IDE warnings. Removed unused utility classes (JavaLiterals, LogUtils).
+
+#### Dependency Upgrades
+
+- **amqp-client**: 5.24.0 → 5.25.0
+- **beanshell**: 2.0b5 → 2.0b6
+- **commons-codec**: 1.17.2 → 1.18.0
+- **dnsjava**: 3.6.2 → 3.6.3
+- **groovy**: 4.0.24 → 4.0.26
+- **gson**: 2.11.0 → 2.12.1
+- **jsch**: 0.2.22 → 0.2.24
+- **pdfbox**: 3.0.3 → 3.0.4
+- **slf4j**: 2.0.16 → 2.0.17
+- **spring**: 6.1.16 → 6.2.5
+
 ## 3.7.0
 
 [Download distribution zip](https://repo1.maven.org/maven2/org/archive/heritrix/heritrix/3.7.0/heritrix-3.7.0-dist.zip) (or [tar.gz](https://repo1.maven.org/maven2/org/archive/heritrix/heritrix/3.7.0/heritrix-3.7.0-dist.tar.gz))

commons/pom.xml

@@ -57,7 +57,7 @@
 		<dependency>
 			<groupId>commons-codec</groupId>
 			<artifactId>commons-codec</artifactId>
-			<version>1.17.2</version>
+			<version>1.18.0</version>
 			<scope>compile</scope>
 		</dependency>
 		<dependency>
@@ -81,7 +81,7 @@
 		<dependency>
 			<groupId>dnsjava</groupId>
 			<artifactId>dnsjava</artifactId>
-			<version>3.6.2</version>
+			<version>3.6.3</version>
 			<scope>compile</scope>
 		</dependency>
 		<!--
@@ -151,7 +151,7 @@
 		<dependency>
 			<groupId>com.github.mwiede</groupId>
 			<artifactId>jsch</artifactId>
-			<version>0.2.22</version>
+			<version>0.2.24</version>
 		</dependency>
 		<dependency>
 			<groupId>org.apache.groovy</groupId>
@@ -211,6 +211,6 @@
 	</build>
 	<properties>
 		<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
-		<spring.version>6.1.16</spring.version>
+		<spring.version>6.2.5</spring.version>
 	</properties>
 </project>

commons/src/main/java/org/archive/util/JavaLiterals.java

@@ -108,6 +108,7 @@ public static boolean isPossibleUri(CharSequence candidate) {
      * @deprecated produces too many false positives,
      *             {@link #isVeryLikelyUri(CharSequence)} is preferred
      */
+    @Deprecated
     public static boolean isLikelyUri(CharSequence candidate) {
         return isPossibleUri(candidate) && !isLikelyFalsePositive(candidate);
     }

commons/src/main/java/org/archive/util/UriUtils.java

@@ -54,7 +54,7 @@ public final class FPGenerator {
         array <code>polynomials</code> contains some irreducible
         polynomials). */
     public static FPGenerator make(long polynomial, int degree) {
-        Long l = new Long(polynomial);
+        Long l = polynomial;
         FPGenerator fpgen = (FPGenerator) generators.get(l);
         if (fpgen == null) {
             fpgen = new FPGenerator(polynomial, degree);

commons/src/main/java/st/ata/util/FPGenerator.java

@@ -27,7 +27,7 @@
 		<dependency>
 			<groupId>com.rabbitmq</groupId>
 			<artifactId>amqp-client</artifactId>
-			<version>5.24.0</version>
+			<version>5.25.0</version>
 		</dependency>
 		<dependency>
 			<groupId>org.easymock</groupId>
@@ -48,7 +48,7 @@
 		<dependency>
 			<groupId>com.google.code.gson</groupId>
 			<artifactId>gson</artifactId>
-			<version>2.11.0</version>
+			<version>2.12.1</version>
 		</dependency>
 	</dependencies>
 	<build>

contrib/pom.xml

@@ -176,25 +176,25 @@ protected void scanJobLog() {
                 isLaunchInfoPartial = true;
                 startPosition = jobLog.length()-(FileUtils.ONE_KB * 100);
             }
-            FileInputStream jobLogIn = new FileInputStream(jobLog);
-            jobLogIn.getChannel().position(startPosition);
-            BufferedReader jobLogReader = new BufferedReader(
-                    new InputStreamReader(jobLogIn));
-            String line;
-            // If we sliced into the file, make sure we skip to the next line:
-            // (See https://github.com/internetarchive/heritrix3/issues/239)
-            if (startPosition != 0) {
-                line = jobLogReader.readLine();
-            }
-            // Parse lines looking for launch details:
-            while ((line = jobLogReader.readLine()) != null) {
-                Matcher m = launchLine.matcher(line);
-                if (m.matches()) {
-                    launchCount++;
-                    lastLaunch = Instant.parse(m.group(1));
+            try (FileInputStream jobLogIn = new FileInputStream(jobLog)) {
+                jobLogIn.getChannel().position(startPosition);
+                BufferedReader jobLogReader = new BufferedReader(
+                        new InputStreamReader(jobLogIn));
+                String line;
+                // If we sliced into the file, make sure we skip to the next line:
+                // (See https://github.com/internetarchive/heritrix3/issues/239)
+                if (startPosition != 0) {
+                    line = jobLogReader.readLine();
+                }
+                // Parse lines looking for launch details:
+                while ((line = jobLogReader.readLine()) != null) {
+                    Matcher m = launchLine.matcher(line);
+                    if (m.matches()) {
+                        launchCount++;
+                        lastLaunch = Instant.parse(m.group(1));
+                    }
                 }
             }
-            jobLogReader.close();
         } catch (IOException e) {
             // TODO Auto-generated catch block
             e.printStackTrace();

engine/src/main/java/org/archive/crawler/framework/CrawlJob.java

@@ -72,10 +72,8 @@ public enum Step {
     private int serialNumber;
 
     /**
-     * Each ToeThead has an instance of HttpRecord that gets used
+     * Each ToeThead has an instance of Recorder that gets used
      * over and over by each request.
-     * 
-     * @see org.archive.util.RecorderMarker
      */
     private Recorder httpRecorder = null;
 

engine/src/main/java/org/archive/crawler/framework/ToeThread.java

@@ -166,8 +166,6 @@ public void run() {
      * recovery log into the frontier as considered included. 
      * 
      * @param source recovery log file to use
-     * @param controller CrawlController of crawl to update
-     * @param retainFailures whether failure ('Ff') URIs should count as done
      * @return number of lines in recovery log (for reference)
      * @throws IOException
      */
@@ -235,7 +233,6 @@ private static int importCompletionInfoFromLog(File source,
      * (excepting those the frontier drops as already having been included)
      * 
      * @param source recovery log file to use
-     * @param controller CrawlController of crawl to update
      * @param params Map of options to apply
      * @param enough latch signalling 'enough' URIs queued to begin crawling
      */

engine/src/main/java/org/archive/crawler/frontier/FrontierJournal.java

@@ -940,7 +940,7 @@ protected void processFinish(CrawlURI curi) {
             wq.setSessionBudget(getBalanceReplenishAmount());
             wq.setTotalBudget(getQueueTotalBudget());
 
-            assert (wq.peek(this) == curi) : "unexpected peek " + wq;
+            if ((wq.peek(this) != curi)) throw new AssertionError("unexpected peek " + wq);
 
             int holderCost = curi.getHolderCost();
 
@@ -1224,10 +1224,6 @@ protected int getTotalIneligibleInactiveQueues() {
                 getInactiveQueuesByPrecedence().tailMap(getPrecedenceFloor()));
     }
 
-    /**
-     * @param iqueue 
-     * @return
-     */
     private int tallyInactiveTotals(SortedMap<Integer,Queue<String>> iqueues) {
         int inactiveCount = 0; 
         for(Queue<String> q : iqueues.values()) {