爬虫设置间隔时间

src/main/java/com/northtecom/visatrack/api/config/ApplicationBeanConfig.java

@@ -1,5 +1,6 @@
 package com.northtecom.visatrack.api.config;
 
+import com.configcat.ConfigCatClient;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
@@ -24,6 +25,11 @@ public class ApplicationBeanConfig {
     @Autowired
     private AuthenticationConfiguration authenticationConfiguration;
 
+    @Bean
+    public ConfigCatClient configCatClient() throws Exception {
+        return new ConfigCatClient("d5naCOKEsUeKSEB2aamvxg/JRdvJ42xcUKZGqnHq1vQgQ");
+    }
+
     /**
      * 用于配置 AuthenticationManager 实例
      */

src/main/java/com/northtecom/visatrack/api/service/impl/CrawlService.java

@@ -1,6 +1,7 @@
 package com.northtecom.visatrack.api.service.impl;
 
 import cn.hutool.core.date.DateUtil;
+import com.configcat.ConfigCatClient;
 import com.northtecom.visatrack.api.config.CrawlConfig;
 import com.northtecom.visatrack.api.data.entity.VisaCheckeeCrawlHtml;
 import com.northtecom.visatrack.api.data.repository.VisaCheckeeCrawlHtmlRepository;
@@ -39,12 +40,21 @@ public class CrawlService {
 
     private final CrawlConfig crawlConfig;
 
+    private final ConfigCatClient configCatClient;
+
+    private Integer crawlInterval = 2000;
+
     @Autowired
-    public CrawlService(VisaCheckeeCrawlHtmlRepository visaCheckeeCrawlHtmlRepository, CrawlConfig crawlConfig) {
+    public CrawlService(VisaCheckeeCrawlHtmlRepository visaCheckeeCrawlHtmlRepository, CrawlConfig crawlConfig,
+                        ConfigCatClient configCatClient) {
         this.visaCheckeeCrawlHtmlRepository = visaCheckeeCrawlHtmlRepository;
         this.crawlConfig = crawlConfig;
+        this.configCatClient = configCatClient;
     }
 
+    protected Integer getCrawlIntervalValue() {
+        return this.configCatClient.getValue(Integer.class, "crawlInterval", this.crawlInterval);
+    }
 
     /**
      * 爬取页面增加重试机制代理机制
@@ -119,7 +129,11 @@ public class CrawlService {
             SaveHtml(content, crawlUrl, crawlDate, crawlId);
         }
 
-        Thread.sleep(2000);
+        Integer sleepTime = getCrawlIntervalValue();
+
+        log.info("Crawl url {} success, sleep {} ms", crawlUrl, sleepTime);
+
+        Thread.sleep(sleepTime);
 
         return content;
     }