feat: use okhttp caching http client fetch rss feed.

avoid rss article updates repeated.

app/server/.gitignore

@@ -110,4 +110,5 @@ fabric.properties
 # Android studio 3.1+ serialized cache file
 .idea/caches/build_file_checksums.ser
 
-lucene/
+lucene/
+feed_cache/

app/server/huntly-server/pom.xml

@@ -146,6 +146,11 @@
             <groupId>cn.shenyanchao.ik-analyzer</groupId>
             <artifactId>ik-analyzer</artifactId>
         </dependency>
+        <dependency>
+            <groupId>com.squareup.okhttp3</groupId>
+            <artifactId>okhttp</artifactId>
+            <version>4.12.0</version>
+        </dependency>
 
         <dependency>
             <groupId>org.springframework.boot</groupId>

app/server/huntly-server/src/main/java/com/huntly/server/connector/rss/FeedUtils.java

@@ -8,55 +8,76 @@
 import com.rometools.rome.io.FeedException;
 import com.rometools.rome.io.SyndFeedInput;
 import lombok.experimental.UtilityClass;
+import okhttp3.OkHttpClient;
+import okhttp3.Request;
+import okhttp3.Response;
 import org.apache.commons.lang3.ArrayUtils;
 import org.apache.commons.lang3.StringUtils;
 
 import java.io.IOException;
 import java.io.StringReader;
-import java.net.URI;
-import java.net.http.HttpClient;
-import java.net.http.HttpRequest;
-import java.net.http.HttpResponse;
 import java.nio.charset.Charset;
-import java.time.Duration;
 
 /**
  * Utility methods related to feed handling
- * code from <a href="https://github.com/Athou/commafeed">commafeed</a> project
  */
 @UtilityClass
 public class FeedUtils {
-    public static SyndFeed parseFeedUrl(String feedUrl, HttpClient client) {
-        HttpRequest request = HttpRequest.newBuilder().GET().uri(URI.create(feedUrl))
+    public static SyndFeed parseFeedUrl(String feedUrl, OkHttpClient client) {
+        Request request = new Request.Builder()
+                .url(feedUrl)
                 .build();
-        HttpResponse<byte[]> response = null;
-        try {
-            response = client.send(request, HttpResponse.BodyHandlers.ofByteArray());
+        try(Response response = client.newCall(request).execute()) {
+            byte[] xmlBytes = null;
+            if (response.body() == null) {
+                throw new ConnectorFetchException("xml response null for url: " + feedUrl);
+            }
+
+            xmlBytes = response.body().bytes();
+            Charset encoding = FeedUtils.guessEncoding(xmlBytes);
+            String xmlString = XmlUtils.removeInvalidXmlCharacters(new String(xmlBytes, encoding));
+            if (xmlString == null) {
+                throw new ConnectorFetchException("xml fetch failed for url: " + feedUrl);
+            }
+            return new SyndFeedInput().build(new StringReader(xmlString));
         } catch (IOException e) {
             throw new RuntimeException(e);
-        } catch (InterruptedException e) {
-            throw new RuntimeException(e);
-        }
-        var xmlBytes = response.body();
-        Charset encoding = FeedUtils.guessEncoding(xmlBytes);
-        String xmlString = XmlUtils.removeInvalidXmlCharacters(new String(xmlBytes, encoding));
-        if (xmlString == null) {
-            throw new ConnectorFetchException("xml fetch failed for url: " + feedUrl);
-        }
-
-        try {
-            SyndFeed feed = new SyndFeedInput().build(new StringReader(xmlString));
-            return feed;
         } catch (FeedException e) {
             throw new RuntimeException(e);
         }
     }
 
-    public static SyndFeed parseFeedUrl(String feedUrl) {
-        var client = HttpClient.newBuilder().connectTimeout(Duration.ofSeconds(60))
-                .followRedirects(HttpClient.Redirect.ALWAYS).build();
-        return parseFeedUrl(feedUrl, client);
-    }
+//    public static SyndFeed parseFeedUrl(String feedUrl, HttpClient client) {
+//        HttpRequest request = HttpRequest.newBuilder().GET().uri(URI.create(feedUrl))
+//                .build();
+//        HttpResponse<byte[]> response = null;
+//        try {
+//            response = client.send(request, HttpResponse.BodyHandlers.ofByteArray());
+//        } catch (IOException e) {
+//            throw new RuntimeException(e);
+//        } catch (InterruptedException e) {
+//            throw new RuntimeException(e);
+//        }
+//        var xmlBytes = response.body();
+//        Charset encoding = FeedUtils.guessEncoding(xmlBytes);
+//        String xmlString = XmlUtils.removeInvalidXmlCharacters(new String(xmlBytes, encoding));
+//        if (xmlString == null) {
+//            throw new ConnectorFetchException("xml fetch failed for url: " + feedUrl);
+//        }
+//
+//        try {
+//            SyndFeed feed = new SyndFeedInput().build(new StringReader(xmlString));
+//            return feed;
+//        } catch (FeedException e) {
+//            throw new RuntimeException(e);
+//        }
+//    }
+
+//    public static SyndFeed parseFeedUrl(String feedUrl) {
+//        var client = HttpClient.newBuilder().connectTimeout(Duration.ofSeconds(60))
+//                .followRedirects(HttpClient.Redirect.ALWAYS).build();
+//        return parseFeedUrl(feedUrl, client);
+//    }
 
     public static Charset guessEncoding(byte[] bytes) {
         String extracted = extractDeclaredEncoding(bytes);

app/server/huntly-server/src/main/java/com/huntly/server/connector/rss/RSSConnector.java

@@ -5,12 +5,14 @@
 import com.huntly.server.connector.ConnectorProperties;
 import com.huntly.server.connector.InfoConnector;
 import com.huntly.server.domain.exceptions.ConnectorFetchException;
+import com.huntly.server.util.HttpUtils;
 import com.huntly.server.util.SiteUtils;
 import com.rometools.rome.feed.synd.SyndCategory;
 import com.rometools.rome.feed.synd.SyndContent;
 import com.rometools.rome.feed.synd.SyndEntry;
 import com.rometools.rome.feed.synd.SyndFeed;
 import lombok.extern.slf4j.Slf4j;
+import okhttp3.OkHttpClient;
 import org.apache.commons.lang3.ObjectUtils;
 import org.apache.commons.lang3.StringUtils;
 
@@ -28,18 +30,16 @@
 public class RSSConnector extends InfoConnector {
     private final ConnectorProperties connectorProperties;
 
+    private final OkHttpClient okClient;
+
     private final HttpClient client;
 
     public RSSConnector(ConnectorProperties connectorProperties) {
         this.connectorProperties = connectorProperties;
+        this.okClient = HttpUtils.buildFeedOkHttpClient(connectorProperties.getProxySetting());
         this.client = buildHttpClient(connectorProperties);
     }
 
-    public RSSConnector(ConnectorProperties connectorProperties, HttpClient httpClient) {
-        this.connectorProperties = connectorProperties;
-        this.client = httpClient;
-    }
-
     @Override
     public List<CapturePage> fetchAllPages() {
         return fetchNewestPages();
@@ -52,7 +52,7 @@ public List<CapturePage> fetchNewestPages() {
         }
 
         try {
-            SyndFeed feed = FeedUtils.parseFeedUrl(connectorProperties.getSubscribeUrl(), client);
+            SyndFeed feed = FeedUtils.parseFeedUrl(connectorProperties.getSubscribeUrl(), okClient);
             var entries = feed.getEntries();
             List<CapturePage> pages = new ArrayList<>();
             for (var entry : entries) {

app/server/huntly-server/src/main/java/com/huntly/server/domain/constant/AppConstants.java

@@ -9,6 +9,10 @@
 public class AppConstants {
     public static final String DEFAULT_LUCENE_DIR = "lucene";
 
+    public static final String HTTP_FEED_CACHE_DIR = "feed_cache";
+
+    public static final Long HTTP_FEED_CACHE_MAXSIZE = 50L * 1024L * 1024L; // 50 MB
+
     public static final Integer DEFAULT_FETCH_INTERVAL_SECONDS = 600;
 
     public static final Integer DEFAULT_COLD_DATA_KEEP_DAYS = 60;

app/server/huntly-server/src/main/java/com/huntly/server/service/ConnectorFetchService.java

@@ -147,7 +147,13 @@ private void fetchPages(Connector connector) {
                 }
             }
 
-            var savedPage = capturePageService.save(page);
+            Page savedPage = null;
+            //Avoid frequent updates of RSS articles.
+            if (isRssFetch && existPage != null && Objects.equals(existPage.getTitle(), page.getTitle()) && Objects.equals(existPage.getConnectedAt(), page.getConnectedAt())) {
+                savedPage = existPage;
+            } else {
+                savedPage = capturePageService.save(page);
+            }
 
             if (isRssFetch && isExecuteFetch) {
                 pageArticleContentService.saveContent(savedPage.getId(), rawContent, ArticleContentCategory.RAW_CONTENT);

app/server/huntly-server/src/main/java/com/huntly/server/service/FeedsService.java

@@ -71,8 +71,10 @@ public Connector followFeed(String subscribeUrl) {
     public PreviewFeedsInfo previewFeeds(String subscribeUrl) {
         PreviewFeedsInfo feedsInfo = new PreviewFeedsInfo();
         feedsInfo.setFeedUrl(subscribeUrl);
-        var httpClient = HttpUtils.buildHttpClient(globalSettingService.getProxySetting());
-        SyndFeed syndFeed = FeedUtils.parseFeedUrl(subscribeUrl, httpClient);
+        var proxySetting = globalSettingService.getProxySetting();
+        var httpClient = HttpUtils.buildHttpClient(proxySetting);
+        var feedClient = HttpUtils.buildFeedOkHttpClient(proxySetting);
+        SyndFeed syndFeed = FeedUtils.parseFeedUrl(subscribeUrl, feedClient);
         if (syndFeed != null) {
             feedsInfo.setSiteLink(syndFeed.getLink());
             feedsInfo.setTitle(syndFeed.getTitle());

app/server/huntly-server/src/main/java/com/huntly/server/util/HttpUtils.java

@@ -1,27 +1,59 @@
 package com.huntly.server.util;
 
+import com.huntly.server.domain.constant.AppConstants;
 import com.huntly.server.domain.model.ProxySetting;
 import lombok.experimental.UtilityClass;
+import okhttp3.Cache;
+import okhttp3.ConnectionSpec;
+import okhttp3.OkHttpClient;
 import org.apache.commons.lang3.StringUtils;
 
 import javax.net.ssl.KeyManager;
 import javax.net.ssl.SSLContext;
 import javax.net.ssl.TrustManager;
 import javax.net.ssl.X509TrustManager;
+import java.io.File;
 import java.net.InetSocketAddress;
+import java.net.Proxy;
 import java.net.ProxySelector;
 import java.net.http.HttpClient;
 import java.security.KeyManagementException;
 import java.security.NoSuchAlgorithmException;
 import java.security.cert.CertificateException;
 import java.security.cert.X509Certificate;
 import java.time.Duration;
+import java.util.Arrays;
 
 /**
  * @author lcomplete
  */
 @UtilityClass
 public class HttpUtils {
+    public static OkHttpClient buildFeedOkHttpClient(ProxySetting proxySetting, Integer timeoutSeconds) {
+        var builder = new OkHttpClient.Builder()
+                .cache(new Cache(
+                        new File(AppConstants.HTTP_FEED_CACHE_DIR), AppConstants.HTTP_FEED_CACHE_MAXSIZE
+                ))
+                .connectionSpecs(Arrays.asList(ConnectionSpec.MODERN_TLS, ConnectionSpec.COMPATIBLE_TLS, ConnectionSpec.CLEARTEXT))
+                .followRedirects(true);
+        if (proxySetting != null && StringUtils.isNotBlank(proxySetting.getHost())) {
+            builder = builder.proxy(
+                    new Proxy(
+                            Proxy.Type.HTTP,
+                            new InetSocketAddress(proxySetting.getHost(), proxySetting.getPort())
+                    )
+            );
+        }
+        if (timeoutSeconds != null) {
+            builder = builder.callTimeout(Duration.ofSeconds(timeoutSeconds));
+        }
+        return builder.build();
+    }
+
+    public static OkHttpClient buildFeedOkHttpClient(ProxySetting proxySetting) {
+        return buildFeedOkHttpClient(proxySetting, 30);
+    }
+
     public static HttpClient buildHttpClient(ProxySetting proxySetting, Integer timeoutSeconds) {
         // Configure SSLContext with a TrustManager that accepts any certificate
         SSLContext sslContext = null;
@@ -31,7 +63,7 @@ public static HttpClient buildHttpClient(ProxySetting proxySetting, Integer time
         } catch (NoSuchAlgorithmException | KeyManagementException e) {
             throw new RuntimeException(e);
         }
-        
+
         var clientBuilder = HttpClient.newBuilder().connectTimeout(Duration.ofSeconds(timeoutSeconds))
                 .sslContext(sslContext)
                 .followRedirects(HttpClient.Redirect.ALWAYS);