From 6d43e5e730ab1fea6ac4a9ee5fd250fa8df8b7ef Mon Sep 17 00:00:00 2001 From: YuCheng Hu Date: Thu, 22 Dec 2022 09:39:15 -0500 Subject: [PATCH 1/2] =?UTF-8?q?USVT-116=20=E6=9B=B4=E6=96=B0=E5=8D=9A?= =?UTF-8?q?=E5=AE=A2=E8=A1=A8=E7=9A=84=E5=AD=97=E6=AE=B5=EF=BC=8C=E6=B7=BB?= =?UTF-8?q?=E5=8A=A0=E4=B8=80=E4=B8=AA=E6=96=B0=E7=9A=84Discourse=20Id=20?= =?UTF-8?q?=E5=AD=97=E6=AE=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../api/controller/api/ContentController.java | 54 ++++++++++++------- .../api/controller/api/CrawlController.java | 32 +++++++++++ .../visatrack/api/data/entity/Blog.java | 2 + 3 files changed, 68 insertions(+), 20 deletions(-) diff --git a/src/main/java/com/northtecom/visatrack/api/controller/api/ContentController.java b/src/main/java/com/northtecom/visatrack/api/controller/api/ContentController.java index 9f16810..ff50384 100644 --- a/src/main/java/com/northtecom/visatrack/api/controller/api/ContentController.java +++ b/src/main/java/com/northtecom/visatrack/api/controller/api/ContentController.java @@ -1,16 +1,20 @@ package com.northtecom.visatrack.api.controller.api; +import com.northtecom.visatrack.api.controller.vo.VisaCaseSearch; import com.northtecom.visatrack.api.service.impl.BlogService; +import com.northtecom.visatrack.api.service.impl.VisaCaseService; +import com.redfin.sitemapgenerator.ChangeFreq; +import com.redfin.sitemapgenerator.WebSitemapGenerator; +import com.redfin.sitemapgenerator.WebSitemapUrl; import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.tags.Tag; -import org.apache.commons.io.FileUtils; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; import java.io.File; -import java.io.IOException; -import java.util.Base64; +import java.net.MalformedURLException; +import java.util.Date; /** * The API related to website content. @@ -32,9 +36,12 @@ public class ContentController { private final BlogService blogService; + private final VisaCaseService visaCaseService; - public ContentController(BlogService blogService) { + + public ContentController(BlogService blogService, VisaCaseService visaCaseService) { this.blogService = blogService; + this.visaCaseService = visaCaseService; } @@ -43,27 +50,34 @@ public class ContentController { * * @return a File with Spring MVC */ - @GetMapping("/sitemap") + @GetMapping("/blog") @Operation(summary = "Get Sitemap xml file", description = "This API will get sitemap xml file") - public String querySitemap() { + public String querySitemap() throws MalformedURLException { byte[] bt = null; String btx = null; + VisaCaseSearch search = new VisaCaseSearch(); +// search.setCheckStartDate(new Date().mi); + + +// 38553 + + + + WebSitemapGenerator wsg = new WebSitemapGenerator("https://www.usvisatrack.com/", new File("D:\\home\\")); + for (int i = 0; i < 39553; i++) { + + + + WebSitemapUrl url = new WebSitemapUrl.Options("https://www.usvisatrack.com/visa/detail?id=" + i) + .lastMod(new Date()).priority(1.0).changeFreq(ChangeFreq.HOURLY).build(); + wsg.addUrl(url); + } + wsg.write(); + + // bt = FileUtils.readFileToByteArray(new File("D:\\home\\sitemap.xml")); - btx = "\n" + - "\n" + - "https://www.ossez.com/sitemap_recent.xml\n" + - "2022-12-15T05:44:41Z\n" + - "\n" + - "\n" + - "https://www.ossez.com/sitemap_1.xml\n" + - "2022-12-15T05:44:41Z\n" + - "\n" + - "\n" + - "https://www.ossez.com/sitemap_2.xml\n" + - "2022-12-12T15:49:36Z\n" + - "\n" + - ""; + btx = ""; // return Base64.getEncoder().withoutPadding().encodeToString(bt); diff --git a/src/main/java/com/northtecom/visatrack/api/controller/api/CrawlController.java b/src/main/java/com/northtecom/visatrack/api/controller/api/CrawlController.java index 67ea65c..3ed86b0 100644 --- a/src/main/java/com/northtecom/visatrack/api/controller/api/CrawlController.java +++ b/src/main/java/com/northtecom/visatrack/api/controller/api/CrawlController.java @@ -61,6 +61,38 @@ public class CrawlController { this.objectMapper = new ObjectMapper(); } + @PostMapping("/blog/sync") + @Operation(summary = "Sync ", description = "同步所有数据") + public Boolean syncBlogByDateRange(@RequestBody CheckeeSyncRequest checkeeSyncRequest) { + + LocalDate startDate = LocalDate.now(); + LocalDate endDate = LocalDate.now(); + try { + String action = checkeeSyncRequest.getAction(); + + // Get Start and End Date by action + switch (action) { + case "date": + if (StringUtils.isNotBlank(checkeeSyncRequest.getStartDate()) && StringUtils.isNotBlank(checkeeSyncRequest.getEndDate())) { + startDate = LocalDate.parse(checkeeSyncRequest.getStartDate()); + endDate = LocalDate.parse(checkeeSyncRequest.getEndDate()); + } + break; + default: + Period p = Period.parse(StringUtils.trimToEmpty(checkeeSyncRequest.getRange())); + startDate = endDate.minus(p); + } + + this.visaReportCheckeeService.syncDataAndReport(startDate,endDate); + + } catch (Exception e) { + throw new BaseException(Status.BAD_REQUEST, "Cannot Process Request"); + } + + return true; + } + + @PostMapping("/checkee/sync") @Operation(summary = "Sync ", description = "同步所有数据") public Boolean syncByDateRange(@RequestBody CheckeeSyncRequest checkeeSyncRequest) { diff --git a/src/main/java/com/northtecom/visatrack/api/data/entity/Blog.java b/src/main/java/com/northtecom/visatrack/api/data/entity/Blog.java index 7ba0e65..aca94ab 100644 --- a/src/main/java/com/northtecom/visatrack/api/data/entity/Blog.java +++ b/src/main/java/com/northtecom/visatrack/api/data/entity/Blog.java @@ -31,6 +31,8 @@ public class Blog extends BaseEntity { private String authorName; @Column(name = "category", columnDefinition = "varchar(50) COMMENT 'Category'") private String category; + @Column(name = "discourse_id", columnDefinition = "bigint COMMENT 'Discourse Id'") + private Long discourseId = 0L; @Column(name = "title", columnDefinition = "varchar(300) COMMENT 'Title'") private String blogTitle; @Column(name = "content", columnDefinition = "LONGTEXT COMMENT 'Content'") From 3f92bcb636aa5cf6fbe2fcecfe02381fd582e899 Mon Sep 17 00:00:00 2001 From: YuCheng Hu Date: Thu, 22 Dec 2022 17:56:42 -0500 Subject: [PATCH 2/2] =?UTF-8?q?USVT-116=20=E6=9B=B4=E6=96=B0=E5=8D=9A?= =?UTF-8?q?=E5=AE=A2=E6=95=B0=E6=8D=AE=E5=A4=84=E7=90=86=E7=9A=84=20API=20?= =?UTF-8?q?=EF=BC=8C=E4=BD=BF=E7=94=A8=E8=BF=99=E4=B8=AA=20API=20=E5=B0=86?= =?UTF-8?q?=E5=8D=9A=E5=AE=A2=E6=95=B0=E6=8D=AE=E7=9A=84=E5=86=85=E5=AE=B9?= =?UTF-8?q?=E6=9B=B4=E6=96=B0=E5=90=8C=E6=AD=A5=E5=88=B0=E7=BD=91=E7=AB=99?= =?UTF-8?q?=E4=B8=8A=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../api/controller/api/CrawlController.java | 39 ++---- .../visatrack/api/data/entity/Blog.java | 22 ++-- .../api/data/repository/BlogRepository.java | 4 + .../model/entity/discourse/DiscoursePost.java | 16 +++ .../entity/discourse/DiscourseTopic.java | 29 +++++ .../api/service/impl/BlogService.java | 122 ++++++++++++++++- .../api/service/impl/EmailService.java | 2 +- .../visatrack/api/util/AwsUtils.java | 75 +++++++++++ .../visatrack/api/util/DiscourseUtils.java | 123 ++++++++++++++++++ .../api/{base => }/util/EmailUtils.java | 2 +- .../northtecom/visatrack/api/CloudTest.java | 16 +++ .../northtecom/visatrack/api/EmailTest.java | 2 +- 12 files changed, 408 insertions(+), 44 deletions(-) create mode 100644 src/main/java/com/northtecom/visatrack/api/model/entity/discourse/DiscoursePost.java create mode 100644 src/main/java/com/northtecom/visatrack/api/model/entity/discourse/DiscourseTopic.java create mode 100644 src/main/java/com/northtecom/visatrack/api/util/AwsUtils.java create mode 100644 src/main/java/com/northtecom/visatrack/api/util/DiscourseUtils.java rename src/main/java/com/northtecom/visatrack/api/{base => }/util/EmailUtils.java (98%) diff --git a/src/main/java/com/northtecom/visatrack/api/controller/api/CrawlController.java b/src/main/java/com/northtecom/visatrack/api/controller/api/CrawlController.java index 3ed86b0..0178d78 100644 --- a/src/main/java/com/northtecom/visatrack/api/controller/api/CrawlController.java +++ b/src/main/java/com/northtecom/visatrack/api/controller/api/CrawlController.java @@ -7,9 +7,7 @@ import com.northtecom.visatrack.api.controller.vo.ImportReportRequest; import com.northtecom.visatrack.api.controller.vo.VisaCrawlRequest; import com.northtecom.visatrack.api.data.spec.DateRange; import com.northtecom.visatrack.api.model.request.api.CheckeeSyncRequest; -import com.northtecom.visatrack.api.service.impl.CrawlService; -import com.northtecom.visatrack.api.service.impl.VisaCaseService; -import com.northtecom.visatrack.api.service.impl.VisaReportCheckeeService; +import com.northtecom.visatrack.api.service.impl.*; import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.tags.Tag; import lombok.extern.slf4j.Slf4j; @@ -50,45 +48,28 @@ public class CrawlController { private final VisaReportCheckeeService visaReportCheckeeService; private final VisaCaseService visaCaseService; + private final BlogService blogService; + private final ObjectMapper objectMapper; @Autowired public CrawlController(CrawlService crawlService, VisaReportCheckeeService visaReportCheckeeService, - VisaCaseService visaCaseService) { + VisaCaseService visaCaseService, BlogService blogService) { this.crawlService = crawlService; this.visaReportCheckeeService = visaReportCheckeeService; this.visaCaseService = visaCaseService; + this.blogService = blogService; this.objectMapper = new ObjectMapper(); } @PostMapping("/blog/sync") @Operation(summary = "Sync ", description = "同步所有数据") public Boolean syncBlogByDateRange(@RequestBody CheckeeSyncRequest checkeeSyncRequest) { - - LocalDate startDate = LocalDate.now(); - LocalDate endDate = LocalDate.now(); try { - String action = checkeeSyncRequest.getAction(); - - // Get Start and End Date by action - switch (action) { - case "date": - if (StringUtils.isNotBlank(checkeeSyncRequest.getStartDate()) && StringUtils.isNotBlank(checkeeSyncRequest.getEndDate())) { - startDate = LocalDate.parse(checkeeSyncRequest.getStartDate()); - endDate = LocalDate.parse(checkeeSyncRequest.getEndDate()); - } - break; - default: - Period p = Period.parse(StringUtils.trimToEmpty(checkeeSyncRequest.getRange())); - startDate = endDate.minus(p); - } - - this.visaReportCheckeeService.syncDataAndReport(startDate,endDate); - + blogService.crawlDiscourseKB(null); } catch (Exception e) { throw new BaseException(Status.BAD_REQUEST, "Cannot Process Request"); } - return true; } @@ -115,7 +96,7 @@ public class CrawlController { startDate = endDate.minus(p); } - this.visaReportCheckeeService.syncDataAndReport(startDate,endDate); + this.visaReportCheckeeService.syncDataAndReport(startDate, endDate); } catch (Exception e) { throw new BaseException(Status.BAD_REQUEST, "Cannot Process Request"); @@ -130,7 +111,7 @@ public class CrawlController { public Boolean syncAllVisaDataFromCheckee() { LocalDate endDate = LocalDate.now(); LocalDate startDate = LocalDate.of(2018, 10, 1); - this.visaReportCheckeeService.syncDataAndReport(startDate,endDate); + this.visaReportCheckeeService.syncDataAndReport(startDate, endDate); return true; } @@ -139,7 +120,7 @@ public class CrawlController { public Boolean syncLast3monthVisaDataFromCheckee() { LocalDate endDate = LocalDate.now(); LocalDate startDate = endDate.minusMonths(3); - this.visaReportCheckeeService.syncDataAndReport(startDate,endDate); + this.visaReportCheckeeService.syncDataAndReport(startDate, endDate); return true; } @@ -148,7 +129,7 @@ public class CrawlController { public Boolean syncLast3YearsVisaDataFromCheckee() { LocalDate endDate = LocalDate.now(); LocalDate startDate = endDate.minusYears(3); - this.visaReportCheckeeService.syncDataAndReport(startDate,endDate); + this.visaReportCheckeeService.syncDataAndReport(startDate, endDate); return true; } diff --git a/src/main/java/com/northtecom/visatrack/api/data/entity/Blog.java b/src/main/java/com/northtecom/visatrack/api/data/entity/Blog.java index aca94ab..b07d427 100644 --- a/src/main/java/com/northtecom/visatrack/api/data/entity/Blog.java +++ b/src/main/java/com/northtecom/visatrack/api/data/entity/Blog.java @@ -2,12 +2,10 @@ package com.northtecom.visatrack.api.data.entity; import com.northtecom.visatrack.api.base.data.BaseEntity; import lombok.Data; +import org.checkerframework.common.aliasing.qual.Unique; import org.hibernate.annotations.Type; -import javax.persistence.Column; -import javax.persistence.Entity; -import javax.persistence.Index; -import javax.persistence.Table; +import javax.persistence.*; import java.time.LocalDateTime; import java.util.List; @@ -18,13 +16,15 @@ import java.util.List; */ @Entity @Data -@Table(name = "blog", indexes = { - @Index(name = "ix_blog_category", columnList = "category"), - @Index(name = "ix_blog_title", columnList = "title"), - @Index(name = "ix_blog_summary", columnList = "summary"), - @Index(name = "ix_blog_author_name", columnList = "author_name"), - @Index(name = "ix_blog_publish_time", columnList = "publish_time") -}) +@Table(name = "blog", + uniqueConstraints = { + @UniqueConstraint(columnNames = {"discourse_id"})}, + indexes = { + @Index(name = "ix_blog_category", columnList = "category"), + @Index(name = "ix_blog_title", columnList = "title"), + @Index(name = "ix_blog_summary", columnList = "summary"), + @Index(name = "ix_blog_author_name", columnList = "author_name"), + @Index(name = "ix_blog_publish_time", columnList = "publish_time")}) @org.hibernate.annotations.Table(appliesTo = "blog", comment = "Blog") public class Blog extends BaseEntity { @Column(name = "author_name", columnDefinition = "varchar(50) COMMENT 'Author name'") diff --git a/src/main/java/com/northtecom/visatrack/api/data/repository/BlogRepository.java b/src/main/java/com/northtecom/visatrack/api/data/repository/BlogRepository.java index af86cc1..d84c59d 100644 --- a/src/main/java/com/northtecom/visatrack/api/data/repository/BlogRepository.java +++ b/src/main/java/com/northtecom/visatrack/api/data/repository/BlogRepository.java @@ -6,6 +6,7 @@ import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.PagingAndSortingRepository; import java.util.List; +import java.util.Set; /** * @@ -20,4 +21,7 @@ public interface BlogRepository extends PagingAndSortingRepository, ".blog b group by month_key", nativeQuery = true) List reportAllYear(); + + List findAllByDiscourseIdIsIn(Set discourseIds); + } diff --git a/src/main/java/com/northtecom/visatrack/api/model/entity/discourse/DiscoursePost.java b/src/main/java/com/northtecom/visatrack/api/model/entity/discourse/DiscoursePost.java new file mode 100644 index 0000000..e225689 --- /dev/null +++ b/src/main/java/com/northtecom/visatrack/api/model/entity/discourse/DiscoursePost.java @@ -0,0 +1,16 @@ +package com.northtecom.visatrack.api.model.entity.discourse; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Data; +import lombok.experimental.Accessors; + +import java.util.List; + +@Data +@Accessors(chain = true) +public class DiscoursePost { + @JsonProperty(value = "cooked") + private String cooked; +} + + diff --git a/src/main/java/com/northtecom/visatrack/api/model/entity/discourse/DiscourseTopic.java b/src/main/java/com/northtecom/visatrack/api/model/entity/discourse/DiscourseTopic.java new file mode 100644 index 0000000..1ff2927 --- /dev/null +++ b/src/main/java/com/northtecom/visatrack/api/model/entity/discourse/DiscourseTopic.java @@ -0,0 +1,29 @@ +package com.northtecom.visatrack.api.model.entity.discourse; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Data; +import lombok.experimental.Accessors; + +import java.util.List; + +@Data +@Accessors(chain = true) +public class DiscourseTopic { + + private Long id; + + @JsonProperty(required = true) + private String title; + + @JsonProperty(value = "image_url") + private String imageUrl; + + @JsonProperty(value = "excerpt") + private String excerpt; + + @JsonProperty(value = "tags") + private List tags; +} + + diff --git a/src/main/java/com/northtecom/visatrack/api/service/impl/BlogService.java b/src/main/java/com/northtecom/visatrack/api/service/impl/BlogService.java index f4a4e30..621607d 100644 --- a/src/main/java/com/northtecom/visatrack/api/service/impl/BlogService.java +++ b/src/main/java/com/northtecom/visatrack/api/service/impl/BlogService.java @@ -1,20 +1,40 @@ package com.northtecom.visatrack.api.service.impl; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; import com.northtecom.visatrack.api.controller.vo.BlogCategoryReport; import com.northtecom.visatrack.api.controller.vo.BlogYearReport; +import com.northtecom.visatrack.api.controller.vo.VisaCrawlRequest; import com.northtecom.visatrack.api.data.entity.Blog; import com.northtecom.visatrack.api.data.repository.BlogRepository; import com.northtecom.visatrack.api.data.spec.BlogSpecification; +import com.northtecom.visatrack.api.model.entity.discourse.DiscoursePost; +import com.northtecom.visatrack.api.model.entity.discourse.DiscourseTopic; +import com.northtecom.visatrack.api.util.DiscourseUtils; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; +import org.springframework.util.NumberUtils; +import org.springframework.util.ObjectUtils; import org.springframework.util.StringUtils; +import java.io.IOException; +import java.time.LocalDateTime; import java.util.ArrayList; import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Function; +import java.util.stream.Collectors; /** * Listing Service. @@ -25,13 +45,21 @@ import java.util.List; @Service @Slf4j public class BlogService { + private final ObjectMapper om; + private final DiscourseUtils discourseUtils; private final BlogRepository blogRepository; + @Autowired - public BlogService(BlogRepository blogRepository) { + public BlogService(DiscourseUtils discourseUtils, ObjectMapper om, BlogRepository blogRepository) { + this.discourseUtils = discourseUtils; this.blogRepository = blogRepository; + + om.disable(DeserializationFeature.FAIL_ON_IGNORED_PROPERTIES); + om.disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES); + this.om = om; } public Page queryAllPageBlog(Pageable pageable, String category, String year, String search) { @@ -81,6 +109,98 @@ public class BlogService { return blogRepository.findById(blogId).orElse(null); } + public void crawlDiscourseKB(VisaCrawlRequest visaCrawlRequest) { + + + String xx = discourseUtils.getTopicsList(); + List blogList = Lists.newArrayList(); + + + try { + JsonNode topicsNode = om.readTree(xx).get("topic_list").get("topics"); + List discourseTopics = om.readerFor(new TypeReference>() { + }).readValue(topicsNode); + Set discourseIds = discourseTopics.stream().map(DiscourseTopic::getId).collect(Collectors.toSet()); + Map blogDBMap = blogRepository.findAllByDiscourseIdIsIn(discourseIds).stream().collect(Collectors.toMap(Blog::getDiscourseId, Function.identity())); + Map topicsDetailMap = discourseUtils.getTopicsDetail(discourseIds); + + for (DiscourseTopic discourseTopic : discourseTopics) { + Long discourseId = discourseTopic.getId(); + Blog blog = null; + + if (ObjectUtils.isEmpty(blogDBMap.get(discourseId))) { + blog = new Blog(); + blog.setDiscourseId(discourseId); + blog.setCategory("KB"); + blog.setVisitCount(0); + + } else { + blog = blogDBMap.get(discourseId); + } + blog.setAuthorName("USVisaTrack"); + + blog.setBlogTitle(discourseTopic.getTitle()); + blog.setBlogCover(discourseTopic.getImageUrl()); + blog.setBlogSummary(discourseTopic.getExcerpt()); + blog.setBlogContent(discourseTopic.getExcerpt()); + blog.setCategory("KB"); + blog.setTags(discourseTopic.getTags()); + blog.setPublishDatetime(LocalDateTime.now()); + + // SET CONTENT + DiscoursePost discoursePost = getx(om, topicsDetailMap, discourseId); + if (StringUtils.hasText(discoursePost.getCooked())) { + blog.setBlogContent(discoursePost.getCooked()); + } + // SET VISIT COUNT + if (ObjectUtils.isEmpty(blog.getVisitCount())) { + blog.setVisitCount(0); + } + + blogList.add(blog); + } + + log.debug(">>>>>>>>>>>>> {}", blogList.size()); + blogRepository.saveAll(blogList); +// +// List discourseTopics = om.co(topicsNode, DiscourseTopic.class); + + } catch (JsonProcessingException e) { + log.error("s", e); + } catch (IOException e) { + throw new RuntimeException(e); + } + + } + + private DiscoursePost getx(ObjectMapper om, Map topicsDetailMap, Long topicId) throws JsonProcessingException { + + DiscoursePost discoursePost = new DiscoursePost(); + String postStr = topicsDetailMap.get(topicId); + try { + if (StringUtils.hasLength(postStr)) { + + JsonNode postsNode = om.readTree(postStr).get("post_stream").get("posts"); + + List discoursePosts = om.readerFor(new TypeReference>() { + }).readValue(postsNode); + if (CollectionUtils.isNotEmpty(discoursePosts)) { + discoursePost = discoursePosts.get(0); + } + + } + + return discoursePost; + + } catch (JsonProcessingException e) { + log.error("s", e); + } catch (IOException e) { + throw new RuntimeException(e); + } + + return null; + } + @Transactional public void updateBlogVisitCount(Blog blog) { diff --git a/src/main/java/com/northtecom/visatrack/api/service/impl/EmailService.java b/src/main/java/com/northtecom/visatrack/api/service/impl/EmailService.java index 16e65e0..965f8ce 100644 --- a/src/main/java/com/northtecom/visatrack/api/service/impl/EmailService.java +++ b/src/main/java/com/northtecom/visatrack/api/service/impl/EmailService.java @@ -1,6 +1,6 @@ package com.northtecom.visatrack.api.service.impl; -import com.northtecom.visatrack.api.base.util.EmailUtils; +import com.northtecom.visatrack.api.util.EmailUtils; import com.northtecom.visatrack.api.data.entity.User; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; diff --git a/src/main/java/com/northtecom/visatrack/api/util/AwsUtils.java b/src/main/java/com/northtecom/visatrack/api/util/AwsUtils.java new file mode 100644 index 0000000..6c95519 --- /dev/null +++ b/src/main/java/com/northtecom/visatrack/api/util/AwsUtils.java @@ -0,0 +1,75 @@ +package com.northtecom.visatrack.api.util; + +import cn.hutool.core.lang.hash.Hash; +import com.amazonaws.auth.AWSCredentials; +import com.amazonaws.auth.AWSStaticCredentialsProvider; +import com.amazonaws.auth.BasicAWSCredentials; +import com.amazonaws.regions.Regions; +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.AmazonS3ClientBuilder; +import com.amazonaws.services.simplesystemsmanagement.AWSSimpleSystemsManagement; +import com.amazonaws.services.simplesystemsmanagement.AWSSimpleSystemsManagementClientBuilder; +import com.amazonaws.services.simplesystemsmanagement.model.GetParameterRequest; +import com.amazonaws.services.simplesystemsmanagement.model.GetParametersByPathRequest; +import com.amazonaws.services.simplesystemsmanagement.model.GetParametersByPathResult; +import com.amazonaws.services.simplesystemsmanagement.model.Parameter; +import com.google.common.collect.Maps; +import com.mailgun.api.v3.MailgunMessagesApi; +import com.mailgun.client.MailgunClient; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.StringUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.stereotype.Component; + +import java.util.HashMap; +import java.util.Map; + + +/** + * Created with IntelliJ IDEA. + * + * @Author: XieYang + * @Date: 2022/10/30/7:24 + * @Description: + */ +@Slf4j +@Component +public class AwsUtils { + + private final AWSSimpleSystemsManagement ssmClient; + + @Autowired + public AwsUtils(AWSSimpleSystemsManagement ssmClient) { + this.ssmClient = ssmClient; + } + + public Map getDiscourseConfigValueFromAWS(Map discourseApiConfMap) { + + Map awsKVMap = new HashMap<>(); + + try { + + GetParametersByPathRequest request = new GetParametersByPathRequest(); + request.setWithDecryption(false); + request.setRecursive(true); + request.setPath(DiscourseUtils.DISCOURSE_PATH); + + GetParametersByPathResult result = this.ssmClient.getParametersByPath(request); + awsKVMap = Maps.uniqueIndex(result.getParameters(), Parameter::getName); + + } catch (Exception ex) { + log.error("Get AWS Value for Key - [{}] Error", DiscourseUtils.DISCOURSE_PATH, ex); + } + + // Update the map + for (Map.Entry entry : discourseApiConfMap.entrySet()) { + discourseApiConfMap.put(entry.getKey(), awsKVMap.get(DiscourseUtils.DISCOURSE_PATH + "/" + entry.getKey()).getValue()); + } + + return discourseApiConfMap; + } +} diff --git a/src/main/java/com/northtecom/visatrack/api/util/DiscourseUtils.java b/src/main/java/com/northtecom/visatrack/api/util/DiscourseUtils.java new file mode 100644 index 0000000..2009761 --- /dev/null +++ b/src/main/java/com/northtecom/visatrack/api/util/DiscourseUtils.java @@ -0,0 +1,123 @@ +/* + * XMLUtils.java + * + * Created on December 6, 2001, 1:21 PM + */ +package com.northtecom.visatrack.api.util; + +import com.amazonaws.services.simplesystemsmanagement.AWSSimpleSystemsManagement; +import com.amazonaws.services.simplesystemsmanagement.model.GetParametersByPathRequest; +import com.mailgun.api.v3.MailgunMessagesApi; +import com.mailgun.model.message.Message; +import com.mailgun.model.message.MessageResponse; +import lombok.extern.slf4j.Slf4j; +import okhttp3.*; +import org.apache.commons.lang3.StringUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; + +/** + * Utilities for Email sending + * + * @author YuCheng Hu + */ +@Slf4j +@Component +public class DiscourseUtils { + + public static final String DISCOURSE_PATH = "/discourse"; + public static final String DISCOURSE_API_KEY = "api_key"; + public static final String DISCOURSE_API_USERNAME = "api_username"; + private Map discourseApiConfMap = new HashMap(); + + @Autowired + public DiscourseUtils(AwsUtils awsUtils) { + discourseApiConfMap.put(DISCOURSE_API_KEY, StringUtils.EMPTY); + discourseApiConfMap.put(DISCOURSE_API_USERNAME, StringUtils.EMPTY); + + discourseApiConfMap = awsUtils.getDiscourseConfigValueFromAWS(discourseApiConfMap); + + } + + + /** + * Send Test Email to check config and email sending API + * + * @return + */ + public String getTopicsList() { + + OkHttpClient client = new OkHttpClient(); + + String responseStr; + try { + HttpUrl.Builder urlBuilder = HttpUrl.parse("https://www.visafn.com/c/kb/6.json").newBuilder(); +// +// urlBuilder.addQueryParameter("appid", weChatAppId); +// urlBuilder.addQueryParameter("secret", weChatSecret); +// urlBuilder.addQueryParameter("code", weChatCode); +// urlBuilder.addQueryParameter("grant_type", "authorization_code"); + + + Request request = new Request.Builder().url(urlBuilder.build().toString()) + .addHeader(DISCOURSE_API_KEY, discourseApiConfMap.get(DISCOURSE_API_KEY)) + .addHeader(DISCOURSE_API_USERNAME, discourseApiConfMap.get(DISCOURSE_API_USERNAME)) + .build(); + + Call call = client.newCall(request); + + + Response response = call.execute(); + responseStr = response.body().string(); + + } catch (IOException e) { + throw new RuntimeException(e); + } + return responseStr; + + } + + public Map getTopicsDetail(Set discourseTopics) { + + Map topicMap = new HashMap<>(); + + OkHttpClient client = new OkHttpClient(); + + try { + for (Long discourseTopic : discourseTopics) { + String responseStr = StringUtils.EMPTY; + + String urlStr = "https://www.visafn.com/t/" + discourseTopic + ".json"; + HttpUrl.Builder urlBuilder = HttpUrl.parse(urlStr).newBuilder(); + + Request request = new Request.Builder().url(urlBuilder.build().toString()) + .addHeader(DISCOURSE_API_KEY, discourseApiConfMap.get(DISCOURSE_API_KEY)) + .addHeader(DISCOURSE_API_USERNAME, discourseApiConfMap.get(DISCOURSE_API_USERNAME)) + .build(); + + Call call = client.newCall(request); + + + Response response = call.execute(); + responseStr = response.body().string(); + + topicMap.put(discourseTopic, responseStr); + + } + + + } catch (IOException e) { + throw new RuntimeException(e); + } + return topicMap; + + } + + +} diff --git a/src/main/java/com/northtecom/visatrack/api/base/util/EmailUtils.java b/src/main/java/com/northtecom/visatrack/api/util/EmailUtils.java similarity index 98% rename from src/main/java/com/northtecom/visatrack/api/base/util/EmailUtils.java rename to src/main/java/com/northtecom/visatrack/api/util/EmailUtils.java index 5adaf3e..f9179a4 100644 --- a/src/main/java/com/northtecom/visatrack/api/base/util/EmailUtils.java +++ b/src/main/java/com/northtecom/visatrack/api/util/EmailUtils.java @@ -3,7 +3,7 @@ * * Created on December 6, 2001, 1:21 PM */ -package com.northtecom.visatrack.api.base.util; +package com.northtecom.visatrack.api.util; import com.amazonaws.services.simplesystemsmanagement.AWSSimpleSystemsManagement; import com.amazonaws.services.simplesystemsmanagement.model.GetParameterRequest; diff --git a/src/test/java/com/northtecom/visatrack/api/CloudTest.java b/src/test/java/com/northtecom/visatrack/api/CloudTest.java index b9bee2f..e261a84 100644 --- a/src/test/java/com/northtecom/visatrack/api/CloudTest.java +++ b/src/test/java/com/northtecom/visatrack/api/CloudTest.java @@ -4,6 +4,8 @@ package com.northtecom.visatrack.api; import com.amazonaws.services.simplesystemsmanagement.AWSSimpleSystemsManagement; import com.amazonaws.services.simplesystemsmanagement.model.GetParameterRequest; import com.amazonaws.services.simplesystemsmanagement.model.GetParameterResult; +import com.amazonaws.services.simplesystemsmanagement.model.GetParametersByPathRequest; +import com.amazonaws.services.simplesystemsmanagement.model.GetParametersByPathResult; import lombok.extern.slf4j.Slf4j; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; @@ -50,4 +52,18 @@ class CloudTest { } + @Test + public void testGetParameterStoreByPath() { + + GetParametersByPathRequest request = new GetParametersByPathRequest(); + request.setWithDecryption(false); + request.setRecursive(true); + request.setPath("/discourse"); + + GetParametersByPathResult result = ssmClient.getParametersByPath(request); + + System.out.println(result.getNextToken()); + + } + } diff --git a/src/test/java/com/northtecom/visatrack/api/EmailTest.java b/src/test/java/com/northtecom/visatrack/api/EmailTest.java index 5ba6c30..81ca222 100644 --- a/src/test/java/com/northtecom/visatrack/api/EmailTest.java +++ b/src/test/java/com/northtecom/visatrack/api/EmailTest.java @@ -1,6 +1,6 @@ package com.northtecom.visatrack.api; -import com.northtecom.visatrack.api.base.util.EmailUtils; +import com.northtecom.visatrack.api.util.EmailUtils; import lombok.extern.slf4j.Slf4j; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInstance;