honeymoose/Framework-Cli
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/master'

Browse Source
This commit is contained in:
YuCheng Hu 2018-12-26 18:07:35 -05:00
commit 4e537e67e2
10 changed files with 405 additions and 95 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

59
core/src/main/java/com/usvisatrack/core/dao/model/CmsArticle.java Normal file
View File

@ -0,0 +1,59 @@
package com.usvisatrack.core.dao.model;
import java.util.Date;
import com.usvisatrack.core.common.DataObject;
public class CmsArticle extends DataObject {
private String rssArticleId;
private String title;
private String summary;
private String content;
private Date rssDateUpdate;
public CmsArticle() {
this.setCreateDate(new Date());
}
public String getRssArticleId() {
return rssArticleId;
}
public void setRssArticleId(String rssArticleId) {
this.rssArticleId = rssArticleId;
}
public String getTitle() {
return title;
}
public void setTitle(String title) {
this.title = title;
}
public String getSummary() {
return summary;
}
public void setSummary(String summary) {
this.summary = summary;
}
public String getContent() {
return content;
}
public void setContent(String content) {
this.content = content;
}
public Date getRssDateUpdate() {
return rssDateUpdate;
}
public void setRssDateUpdate(Date rssDateUpdate) {
this.rssDateUpdate = rssDateUpdate;
}
}

26
core/src/main/java/com/usvisatrack/core/dao/model/CmsArticleOssezContent.java Normal file
View File

@ -0,0 +1,26 @@
package com.usvisatrack.core.dao.model;
import com.usvisatrack.core.common.DataObject;
public class CmsArticleOssezContent extends DataObject {
private Long aid;
private String content;
public Long getAid() {
return aid;
}
public void setAid(Long aid) {
this.aid = aid;
}
public String getContent() {
return content;
}
public void setContent(String content) {
this.content = content;
}
}

70
core/src/main/java/com/usvisatrack/core/factories/DataFactory.java
View File

@ -1,8 +1,20 @@
package com.usvisatrack.core.factories;
import java.util.Date;
import org.hibernate.Criteria;
import org.hibernate.criterion.Restrictions;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.usvisatrack.core.common.DataObject;
import com.usvisatrack.core.common.Factory;
import com.usvisatrack.core.dao.model.CmsArticle;
import com.usvisatrack.core.dao.model.CmsArticleOssezContent;
import com.usvisatrack.core.dao.model.SocialMediaLog;
import com.usvisatrack.core.dao.model.UscisCase;
import com.usvisatrack.core.dao.model.VisaStatusLog;
/**
*
*
@ -14,5 +26,63 @@ public class DataFactory {
public static final int DEFAULT_AUTH_EXPIRATION = 10;
private static Logger logger = LoggerFactory.getLogger(DataFactory.class);
/**
*
* @param id
* @return
*/
public static CmsArticleOssezContent getCmsArticleOssezContent(Long aid) {
CmsArticleOssezContent cmsArticleOssezContent = null;
try {
Factory.beginTransaction();
Criteria criteria = Factory.createCriteria(CmsArticleOssezContent.class);
criteria.add(Restrictions.eq("aid", aid));
criteria.setMaxResults(1);
criteria.uniqueResult();
cmsArticleOssezContent = (CmsArticleOssezContent) criteria.uniqueResult();
} catch (Exception ex) {
cmsArticleOssezContent = null;
logger.error("Get Visa by item ID error", ex);
} finally {
Factory.commitTransaction();
}
return cmsArticleOssezContent;
}
public static CmsArticle getCmsArticle(String rssArticleId) {
CmsArticle cmsArticle = null;
try {
Factory.beginTransaction();
Criteria criteria = Factory.createCriteria(CmsArticle.class);
criteria.add(Restrictions.eq("rssArticleId", rssArticleId));
criteria.setMaxResults(1);
criteria.uniqueResult();
cmsArticle = (CmsArticle) criteria.uniqueResult();
} catch (Exception ex) {
logger.error("Get Visa by item ID error", ex);
} finally {
Factory.commitTransaction();
}
return cmsArticle;
}
public static void save(DataObject obj) {
obj.setModifyDate(new Date());
try {
Factory.beginTransaction();
Factory.saveOrUpdate(obj);
Factory.commitTransaction();
} catch (Exception ex) {
logger.error("Save CheckeeVisa OBJ ERROR", ex);
Factory.rollbackTransaction();
}
}
}

42
core/src/main/resources/hbm/CmsArticle.hbm.xml Normal file
View File

@ -0,0 +1,42 @@
<?xml version="1.0"?>
<!DOCTYPE hibernate-mapping PUBLIC "-//Hibernate/Hibernate Mapping DTD 3.0//EN"
"http://www.hibernate.org/dtd/hibernate-mapping-3.0.dtd">
<hibernate-mapping package="com.usvisatrack.core.dao.model">
<class name="CmsArticle" table="cms_article" catalog="northtecom_usvisatrack">
<id name="id" type="java.lang.Long">
<column name="cms_article_id" />
<generator class="identity" />
</id>
<property name="rssArticleId" type="string">
<column name="rss_article_id" length="128" />
</property>
<property name="title" type="string">
<column name="title" length="256" />
</property>
<property name="summary" type="string">
<column name="summary" length="1024" />
</property>
<property name="content" type="string">
<column name="content" length="2048" />
</property>
<property name="rssDateUpdate" type="timestamp">
<column name="rss_date_upd" />
</property>
<property name="createDate" type="timestamp">
<column name="date_add" />
</property>
<property name="modifyDate" type="timestamp">
<column name="date_upd" />
</property>
</class>
</hibernate-mapping>

20
core/src/main/resources/hbm/CmsArticleOssezContent.hbm.xml Normal file
View File

@ -0,0 +1,20 @@
<?xml version="1.0"?>
<!DOCTYPE hibernate-mapping PUBLIC "-//Hibernate/Hibernate Mapping DTD 3.0//EN"
"http://www.hibernate.org/dtd/hibernate-mapping-3.0.dtd">
<hibernate-mapping package="com.usvisatrack.core.dao.model">
<class name="CmsArticleOssezContent" table="bbsossez_portal_article_content" catalog="ossez_bbs">
<id name="id" type="java.lang.Long">
<column name="cid" />
<generator class="identity" />
</id>
<property name="aid" type="long">
<column name="aid" />
</property>
<property name="content" type="string">
<column name="content" length="2068" />
</property>
</class>
</hibernate-mapping>

2
core/src/main/resources/hibernate.cfg.xml
View File

@ -29,6 +29,8 @@
<!-- HBMs -->
<mapping resource="hbm/CheckeeVisa.hbm.xml" />
<mapping resource="hbm/CmsArticle.hbm.xml" />
<mapping resource="hbm/CmsArticleOssezContent.hbm.xml" />
<mapping resource="hbm/ConfApp.hbm.xml" />
<mapping resource="hbm/SocialMedia.hbm.xml" />
<mapping resource="hbm/SocialMediaLog.hbm.xml" />

14
services/pom.xml
View File

@ -43,11 +43,6 @@
<!-- AOP -->
<aspectj.version>1.7.4</aspectj.version>
<!-- RSS -->
<rome.version>1.0</rome.version>
<hamcrest.version>1.3</hamcrest.version>
<!-- Web dependencies -->
<webjars-bootstrap.version>2.3.0</webjars-bootstrap.version>
<webjars-jquery-ui.version>1.10.3</webjars-jquery-ui.version>
@ -84,7 +79,7 @@
<dependency>
<groupId>org.hamcrest</groupId>
<artifactId>hamcrest-library</artifactId>
<version>${hamcrest.version}</version>
<version>1.3</version>
<scope>test</scope>
</dependency>
@ -151,6 +146,13 @@
<version>4.0.6</version>
</dependency>
<!-- RSS -->
<dependency>
<groupId>com.rometools</groupId>
<artifactId>rome</artifactId>
<version>1.8.0</version>
</dependency>
</dependencies>
<dependencyManagement>

185
services/src/main/java/com/usvisatrack/services/VisaImporter.java
View File

@ -1,6 +1,7 @@
package com.usvisatrack.services;
import java.io.IOException;
import java.net.URL;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
@ -19,7 +20,13 @@ import org.jsoup.select.Elements;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.rometools.rome.feed.synd.SyndEntry;
import com.rometools.rome.feed.synd.SyndFeed;
import com.rometools.rome.io.SyndFeedInput;
import com.rometools.rome.io.XmlReader;
import com.usvisatrack.core.dao.model.CheckeeVisa;
import com.usvisatrack.core.dao.model.CmsArticle;
import com.usvisatrack.core.dao.model.CmsArticleOssezContent;
import com.usvisatrack.core.dao.model.ConfApp;
import com.usvisatrack.core.dao.model.SocialMedia;
import com.usvisatrack.core.dao.model.SocialMediaLog;
@ -30,6 +37,7 @@ import com.usvisatrack.core.dao.model.VisaStatusLog;
import com.usvisatrack.core.dao.model.data.VisaEntry;
import com.usvisatrack.core.dao.model.data.VisaStatus;
import com.usvisatrack.core.factories.ConfFactory;
import com.usvisatrack.core.factories.DataFactory;
import com.usvisatrack.core.factories.LogFactory;
import com.usvisatrack.core.factories.SocialMediaFactory;
import com.usvisatrack.core.factories.USFactory;
@ -53,6 +61,8 @@ public class VisaImporter extends DataCrawl {
private static final String CONF_LAST_RUNTIME = VisaImporter.class.getName() + "-lastRuntime";
private static final String CONF_TWITTER_COUNT = VisaImporter.class.getName() + "-twitterCount";
private static final String RSS_URL = "http://rss.ossez.com/public.php?op=rss&id=-2&view-mode=all_articles&key=ofclrs5a01d7c3bdab2";
private static final String TWITTER = "Twitter";
private static final String URL_CHECKEE = "https://www.checkee.info/main.php?dispdate=";
@ -103,8 +113,8 @@ public class VisaImporter extends DataCrawl {
logger.debug("NOT RUN");
}
// SOCIAL MEDIA
SocialMedia();
socialMedia();// SOCIAL MEDIA
rssArticle(); // RSS
processCleanup();
}
@ -204,30 +214,30 @@ public class VisaImporter extends DataCrawl {
String visaClassName = StringUtils.trimToEmpty(tds.get(2).text());
if (StringUtils.isNotBlank(visaClassName)) {
switch (visaClassName) {
case "B1" :
visaClassName = "B-1";
break;
case "B2" :
visaClassName = "B-2";
break;
case "H1" :
visaClassName = "H1-B";
break;
case "H4" :
visaClassName = "H-4";
break;
case "F1" :
visaClassName = "F-1";
break;
case "F2" :
visaClassName = "F-2";
break;
case "J1" :
visaClassName = "J-1";
break;
case "L1" :
visaClassName = "L-1";
break;
case "B1":
visaClassName = "B-1";
break;
case "B2":
visaClassName = "B-2";
break;
case "H1":
visaClassName = "H1-B";
break;
case "H4":
visaClassName = "H-4";
break;
case "F1":
visaClassName = "F-1";
break;
case "F2":
visaClassName = "F-2";
break;
case "J1":
visaClassName = "J-1";
break;
case "L1":
visaClassName = "L-1";
break;
}
visa.setVisaClass(visaClassMap.get(visaClassName));
@ -237,12 +247,12 @@ public class VisaImporter extends DataCrawl {
String visaEntryName = StringUtils.upperCase(StringUtils.trimToEmpty(tds.get(3).text()));
if (StringUtils.isNotBlank(visaEntryName)) {
switch (visaEntryName) {
case "NEW" :
visa.setVisaEntry(VisaEntry.NEW);
break;
case "RENEWAL" :
visa.setVisaEntry(VisaEntry.RENEWAL);
break;
case "NEW":
visa.setVisaEntry(VisaEntry.NEW);
break;
case "RENEWAL":
visa.setVisaEntry(VisaEntry.RENEWAL);
break;
}
}
@ -251,27 +261,27 @@ public class VisaImporter extends DataCrawl {
String usEmbassyName = StringUtils.upperCase(StringUtils.trimToEmpty(tds.get(4).text()));
if (StringUtils.isNotBlank(usEmbassyName)) {
switch (usEmbassyName) {
case "BEIJING" :
visa.setUsEmbassy(USFactory.searchUSEmbassy("China", "BeiJing"));
break;
case "CHENGDU" :
visa.setUsEmbassy(USFactory.searchUSEmbassy("China", "Chengdu"));
break;
case "GUANGZHOU" :
visa.setUsEmbassy(USFactory.searchUSEmbassy("China", "Guangzhou"));
break;
case "SHANGHAI" :
visa.setUsEmbassy(USFactory.searchUSEmbassy("China", "Shanghai"));
break;
case "SHENYANG" :
visa.setUsEmbassy(USFactory.searchUSEmbassy("China", "Shenyang"));
break;
case "WUHAN" :
visa.setUsEmbassy(USFactory.searchUSEmbassy("China", "Wuhan"));
break;
case "HONGKONG" :
visa.setUsEmbassy(USFactory.searchUSEmbassy("China", "HongKong"));
break;
case "BEIJING":
visa.setUsEmbassy(USFactory.searchUSEmbassy("China", "BeiJing"));
break;
case "CHENGDU":
visa.setUsEmbassy(USFactory.searchUSEmbassy("China", "Chengdu"));
break;
case "GUANGZHOU":
visa.setUsEmbassy(USFactory.searchUSEmbassy("China", "Guangzhou"));
break;
case "SHANGHAI":
visa.setUsEmbassy(USFactory.searchUSEmbassy("China", "Shanghai"));
break;
case "SHENYANG":
visa.setUsEmbassy(USFactory.searchUSEmbassy("China", "Shenyang"));
break;
case "WUHAN":
visa.setUsEmbassy(USFactory.searchUSEmbassy("China", "Wuhan"));
break;
case "HONGKONG":
visa.setUsEmbassy(USFactory.searchUSEmbassy("China", "HongKong"));
break;
}
}
@ -284,15 +294,15 @@ public class VisaImporter extends DataCrawl {
String visaStatusName = StringUtils.upperCase(StringUtils.trimToEmpty(tds.get(6).text()));
if (StringUtils.isNotBlank(visaStatusName)) {
switch (visaStatusName) {
case "CLEAR" :
visa.setVisaStatus(VisaStatus.ISSUED);
break;
case "PENDING" :
visa.setVisaStatus(VisaStatus.ADMINISTRATIVE_PROCESSING);
break;
case "REJECT" :
visa.setVisaStatus(VisaStatus.REFUSED);
break;
case "CLEAR":
visa.setVisaStatus(VisaStatus.ISSUED);
break;
case "PENDING":
visa.setVisaStatus(VisaStatus.ADMINISTRATIVE_PROCESSING);
break;
case "REJECT":
visa.setVisaStatus(VisaStatus.REFUSED);
break;
}
}
@ -334,7 +344,7 @@ public class VisaImporter extends DataCrawl {
}
private void SocialMedia() {
private void socialMedia() {
List<VisaStatusLog> visaStatusLogList = LogFactory.getVisaStatusLogAfter(new DateTime().minusHours(2));
// Twitter
@ -353,6 +363,55 @@ public class VisaImporter extends DataCrawl {
}
private void rssArticle() {
try {
URL urls = new URL(RSS_URL);
SyndFeedInput input = new SyndFeedInput();
SyndFeed feed = input.build(new XmlReader(urls));
for (SyndEntry entry : feed.getEntries()) {
CmsArticle cmsArticle = null;
String aid = StringUtils.substringBetween(entry.getLink(), "article-", "-");
logger.debug("article Id/Title - [{}]/[{}]", aid, entry.getTitle());
DateTime dt = new DateTime(entry.getUpdatedDate());
CmsArticleOssezContent cmsArticleOssezContent = DataFactory.getCmsArticleOssezContent(NumberUtils.toLong(aid));
cmsArticle = DataFactory.getCmsArticle(aid);
if (cmsArticle == null) {
cmsArticle = new CmsArticle();
cmsArticle.setRssArticleId(aid);
} else if (dt.compareTo(new DateTime(cmsArticle.getRssDateUpdate())) == 0) {
continue;
}
cmsArticle.setTitle(entry.getTitle());
cmsArticle.setSummary(entry.getContents().get(0).getValue());
cmsArticle.setContent(cmsArticleOssezContent.getContent());
cmsArticle.setRssDateUpdate(dt.toDate());
DataFactory.save(cmsArticle);
// logger.debug("SAVE");
// break;
}
} catch (Exception ex) {
logger.error("RSS data update error.", ex);
//
}
// System.out.println(feed);
}
private void publishTwitter(VisaStatusLog visaStatusLog) {
SocialMedia socialMedia = SocialMediaFactory.get(TWITTER);

74
services/src/test/java/com/usvisatrack/services/test/DataTest.java
View File

@ -1,8 +1,21 @@
package com.usvisatrack.services.test;
import java.net.URL;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.math.NumberUtils;
import org.joda.time.DateTime;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.rometools.rome.feed.synd.SyndEntry;
import com.rometools.rome.feed.synd.SyndFeed;
import com.rometools.rome.io.SyndFeedInput;
import com.rometools.rome.io.XmlReader;
import com.usvisatrack.core.dao.model.CmsArticle;
import com.usvisatrack.core.dao.model.CmsArticleOssezContent;
import com.usvisatrack.core.factories.DataFactory;
/**
* TEST Data API
@ -13,31 +26,48 @@ import org.slf4j.LoggerFactory;
public class DataTest {
private static Logger logger = LoggerFactory.getLogger(DataTest.class);
public final static String APIDATA_CDN_FOLDER = "/home/cdn/com-bcodepot-www/data/apidata";
public final static String MERCHANDISE_CDN_FOLDER = "/home/cdn/com-bcodepot-www/data/item";
public final static String RSS_URL = "http://rss.ossez.com/public.php?op=rss&id=-2&view-mode=all_articles&key=ofclrs5a01d7c3bdab2";
/**
* Your AWS Access Key ID, as taken from the AWS Your Account page.
*/
private static final String AWS_NAME = "AmazonMerchandiseData";
private static String AWS_ACCESS_KEY_ID = "";
/*
* Your AWS Secret Key corresponding to the above ID, as taken from the AWS Your Account page.
*/
private static String AWS_SECRET_KEY = "";
@Test
public void testRSS() throws Exception {
URL urls = new URL(RSS_URL);
/*
* Use one of the following end-points, according to the region you are interested in:
*
* US: ecs.amazonaws.com CA: ecs.amazonaws.ca UK: ecs.amazonaws.co.uk DE: ecs.amazonaws.de FR: ecs.amazonaws.fr JP: ecs.amazonaws.jp
*/
private static String ENDPOINT = "ecs.amazonaws.com";
SyndFeedInput input = new SyndFeedInput();
SyndFeed feed = input.build(new XmlReader(urls));
/*
* The Item ID to lookup. The value below was selected for the US locale. You can choose a different value if this value does not work
* in the locale of your choice.
*/
private static final String ITEM_ID = "014100087847";
// System.out.println(feed);
for (SyndEntry entry : feed.getEntries()) {
CmsArticle cmsArticle = null;
String aid = StringUtils.substringBetween(entry.getLink(), "article-", "-");
logger.debug("article Id/Title - [{}]/[{}]", aid, entry.getTitle());
DateTime dt = new DateTime(entry.getUpdatedDate());
CmsArticleOssezContent cmsArticleOssezContent = DataFactory.getCmsArticleOssezContent(NumberUtils.toLong(aid));
cmsArticle = DataFactory.getCmsArticle(aid);
if (cmsArticle == null) {
cmsArticle = new CmsArticle();
cmsArticle.setRssArticleId(aid);
} else if (dt.compareTo(new DateTime(cmsArticle.getRssDateUpdate())) == 0) {
continue;
}
cmsArticle.setTitle(entry.getTitle());
cmsArticle.setSummary(entry.getContents().get(0).getValue());
cmsArticle.setContent(cmsArticleOssezContent.getContent());
cmsArticle.setRssDateUpdate(dt.toDate());
DataFactory.save(cmsArticle);
// logger.debug("SAVE");
// break;
}
}
}

8
services/src/test/resources/hibernate.properties
View File

@ -1,11 +1,11 @@
# JDBC Connection
hibernate.connection.driver_class=com.mysql.jdbc.Driver
hibernate.connection.driver_class=org.mariadb.jdbc.Driver
hibernate.dialect=org.hibernate.dialect.MySQL5Dialect
hibernate.connection.url=jdbc:mysql://db.upcex.com/
hibernate.connection.url=jdbc:mysql://db.usvisatrack.com/northtecom_usvisatrack?serverTimezone=EST&useUnicode=true&characterEncoding=UTF8MB64&autoReconnect=true
# Username and Password
hibernate.connection.username=upcex.web
hibernate.connection.password=026l64e4rtfg
hibernate.connection.username=usvisatrack.web
hibernate.connection.password=rOhAz8j6SAjObag3ve5oYew2d5d4Va
# Performance
#hibernate.connection.pool.size=20