honeymoose/Framework-Cli
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/master'

Browse Source
This commit is contained in:
Yucheng Hu 2017-07-30 00:01:03 -04:00
commit c3fc377bb8
13 changed files with 294 additions and 191 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
core/pom.xml
View File

@ -56,7 +56,7 @@
<dependency>
<groupId>org.apache.solr</groupId>
<artifactId>solr-solrj</artifactId>
<artifactId>solr-solrj</artifactId>
<version>5.5.3</version>
</dependency>
<!-- /DATABASE AND HIBERNATE -->
@ -142,22 +142,10 @@
<!--/CLOUDS -->
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk</artifactId>
<version>1.11.114</version>
<artifactId>aws-java-sdk-s3</artifactId>
<version>1.11.166</version>
</dependency>
<dependency>
<groupId>com.paypal.sdk</groupId>
<artifactId>paypal-core</artifactId>
<version>1.7.2</version>
</dependency>
<dependency>
<groupId>com.paypal.sdk</groupId>
<artifactId>rest-api-sdk</artifactId>
<version>1.13.0</version>
</dependency>
<!--/CLOUDS -->
<dependency>

27
core/src/main/java/com/usvisatrack/core/dao/model/CheckeeVisa.java Normal file
View File

@ -0,0 +1,27 @@
package com.usvisatrack.core.dao.model;
import com.usvisatrack.core.common.DataObject;
/**
* USEmbassy ORM
*
* @author YuCheng Hu
*
*/
public class CheckeeVisa extends DataObject {
private Long visaID;
public CheckeeVisa() {
}
public Long getVisaID() {
return visaID;
}
public void setVisaID(Long visaID) {
this.visaID = visaID;
}
}

75
core/src/main/java/com/usvisatrack/core/dao/model/User.java
View File

@ -1,42 +1,25 @@
package com.usvisatrack.core.dao.model;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import com.usvisatrack.core.common.DataObject;
/**
* Merchant generated by hbm2java
* User ORM
*
* @author YuCheng
*
*/
public class User extends DataObject implements java.io.Serializable {
public class User extends DataObject {
/**
*
*/
private static final long serialVersionUID = -8094138967079219902L;
private String email;
private String userName;
private String password;
private String apiKey;
private String apiToken;
private String firstName;
private String lirstName;
private Date dateReg;
private String email;
private Date dateRegistered;
public User() {
}
public String getEmail() {
return email;
}
public void setEmail(String email) {
this.email = email;
this.dateRegistered = new Date();
}
public String getUserName() {
@ -55,46 +38,20 @@ public class User extends DataObject implements java.io.Serializable {
this.password = password;
}
public String getApiKey() {
return apiKey;
public String getEmail() {
return email;
}
public void setApiKey(String apiKey) {
this.apiKey = apiKey;
public void setEmail(String email) {
this.email = email;
}
public String getApiToken() {
return apiToken;
public Date getDateRegistered() {
return dateRegistered;
}
public void setApiToken(String apiToken) {
this.apiToken = apiToken;
public void setDateRegistered(Date dateRegistered) {
this.dateRegistered = dateRegistered;
}
public String getFirstName() {
return firstName;
}
public void setFirstName(String firstName) {
this.firstName = firstName;
}
public String getLirstName() {
return lirstName;
}
public void setLirstName(String lirstName) {
this.lirstName = lirstName;
}
public Date getDateReg() {
return dateReg;
}
public void setDateReg(Date dateReg) {
this.dateReg = dateReg;
}
}

11
core/src/main/java/com/usvisatrack/core/dao/model/Visa.java
View File

@ -14,9 +14,10 @@ import com.usvisatrack.core.dao.model.data.VisaStatus;
*/
public class Visa extends DataObject {
private String ds160Code;
private User user;
private USEmbassy usEmbassy;
private VisaClass visaClass;
private String ds160Code;
private VisaEntry visaEntry;
private VisaStatus visaStatus;
private String major;
@ -31,6 +32,14 @@ public class Visa extends DataObject {
super.setModifyDate(date);
}
public User getUser() {
return user;
}
public void setUser(User user) {
this.user = user;
}
public String getDs160Code() {
return ds160Code;
}

60
core/src/main/java/com/usvisatrack/core/factories/UserFactory.java
View File

@ -1,16 +1,14 @@
package com.usvisatrack.core.factories;
import java.util.List;
import org.hibernate.Criteria;
import org.hibernate.Transaction;
import org.hibernate.criterion.Order;
import org.hibernate.criterion.Restrictions;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.usvisatrack.core.common.Factory;
import com.usvisatrack.core.dao.model.USEmbassy;
import com.usvisatrack.core.dao.model.User;
import com.usvisatrack.core.dao.model.Visa;
public class UserFactory {
private static final int LISTING_PAGE_SIZE = 20;
@ -18,6 +16,58 @@ public class UserFactory {
private static Logger logger = LoggerFactory.getLogger(UserFactory.class);
public static User get(Long id) {
logger.debug("Get User OBJ by ID - [{}]", id);
Factory.beginTransaction();
User user = null;
try {
user = Factory.get(User.class, id);
Factory.commitTransaction();
} catch (Exception ex) {
logger.error("Get User by user ID error", ex);
} finally {
Factory.rollbackTransaction();
}
return user;
}
/**
* Get user by userName
*
* @param userName
* @return
*/
public static User get(String userName) {
logger.debug("Search USEmbassy - [{}]/[{}]", userName);
try {
Factory.beginTransaction();
Criteria criteria = Factory.createCriteria(User.class);
criteria.add(Restrictions.eq("userName", userName));
criteria.setMaxResults(1);
return (User) criteria.uniqueResult();
} catch (Exception ex) {
logger.error("Search USEmbassy data error", ex);
} finally {
Factory.commitTransaction();
}
return null;
}
/**
*
* @param user
*/
public static void save(User user) {
try {
Factory.beginTransaction();
Factory.saveOrUpdate(user);
Factory.commitTransaction();
} catch (Exception ex) {
logger.error("Save USER OBJ ERROR", ex);
Factory.rollbackTransaction();
}
}
}

40
core/src/main/java/com/usvisatrack/core/factories/VisaFactory.java
View File

@ -7,6 +7,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.usvisatrack.core.common.Factory;
import com.usvisatrack.core.dao.model.CheckeeVisa;
import com.usvisatrack.core.dao.model.Visa;
import com.usvisatrack.core.dao.model.VisaClass;
@ -45,6 +46,34 @@ public class VisaFactory extends Factory {
return item;
}
/**
* Get Visa OBJ
*
* @param checkkeID
* @return
*/
public static Visa getVisaFromCheckee(Long checkkeID) {
logger.debug("Get Visa OBJ by checkkeID - [{}]", checkkeID);
Visa visa = null;
CheckeeVisa checkeeVisa = null;
Factory.beginTransaction();
try {
checkeeVisa = Factory.get(CheckeeVisa.class, checkkeID);
if (checkeeVisa != null) {
visa = get(checkeeVisa.getVisaID());
}
Factory.commitTransaction();
} catch (Exception ex) {
logger.error("Get Visa by item ID error", ex);
} finally {
Factory.rollbackTransaction();
}
return visa;
}
/**
*
* @return
@ -80,4 +109,15 @@ public class VisaFactory extends Factory {
}
}
public static void save(CheckeeVisa checkeeVisa) {
try {
Factory.beginTransaction();
Factory.saveOrUpdate(checkeeVisa);
Factory.commitTransaction();
} catch (Exception ex) {
logger.error("Save CheckeeVisa OBJ ERROR", ex);
Factory.rollbackTransaction();
}
}
}

16
core/src/main/resources/hbm/CheckeeVisa.hbm.xml Normal file
View File

@ -0,0 +1,16 @@
<?xml version="1.0"?>
<!DOCTYPE hibernate-mapping PUBLIC "-//Hibernate/Hibernate Mapping DTD 3.0//EN"
"http://www.hibernate.org/dtd/hibernate-mapping-3.0.dtd">
<hibernate-mapping package="com.usvisatrack.core.dao.model">
<class name="CheckeeVisa" table="checkee_visa" catalog="northtecom_usvisatrack">
<id name="id" type="long">
<column name="checkee_visa_id" />
<generator class="assigned" />
</id>
<property name="visaID" type="long">
<column name="visa_id" />
</property>
</class>
</hibernate-mapping>

26
core/src/main/resources/hbm/User.hbm.xml
View File

@ -3,34 +3,26 @@
"http://www.hibernate.org/dtd/hibernate-mapping-3.0.dtd">
<hibernate-mapping package="com.usvisatrack.core.dao.model">
<class name="User" table="user" catalog="ntc_usvisatrack">
<id name="id" type="java.lang.Long">
<column name="uid" />
<class name="User" table="user" catalog="northtecom_usvisatrack">
<id name="id" type="long">
<column name="user_id" />
<generator class="identity" />
</id>
<property name="email" type="string">
<column name="email" />
</property>
<property name="userName" type="string">
<column name="username" />
<column name="username" length="64" />
</property>
<property name="password" type="string">
<column name="password" />
<column name="password" length="255" />
</property>
<property name="apiKey" type="string">
<column name="api_key" />
<property name="email" type="string">
<column name="email" length="255" />
</property>
<property name="apiToken" type="string">
<column name="api_token" />
</property>
<property name="dateReg" type="timestamp">
<column name="date_reg" />
<property name="dateRegistered" type="timestamp">
<column name="date_registered" />
</property>
</class>

11
core/src/main/resources/hbm/Visa.hbm.xml
View File

@ -9,13 +9,18 @@
<generator class="identity" />
</id>
<property name="ds160Code" type="string">
<column name="ds160_code" length="16" />
</property>
<many-to-one class="com.usvisatrack.core.dao.model.User" fetch="select" insert="true" name="user" update="true" lazy="proxy">
<column name="user_fk" not-null="true" />
</many-to-one>
<many-to-one class="com.usvisatrack.core.dao.model.USEmbassy" name="usEmbassy" column="us_embassy_fk" fetch="join" />
<many-to-one class="com.usvisatrack.core.dao.model.VisaClass" name="visaClass" column="visa_class_fk" fetch="join" />
<property name="ds160Code" type="string">
<column name="ds160_code" length="16" />
</property>
<property name="visaStatus" column="visa_status">
<type name="org.hibernate.type.EnumType">
<param name="enumClass">com.usvisatrack.core.common.data.VisaStatus</param>

1
core/src/main/resources/hibernate.cfg.xml
View File

@ -28,6 +28,7 @@
<property name="hibernate.c3p0.validate">true</property>
<!-- HBMs -->
<mapping resource="hbm/CheckeeVisa.hbm.xml" />
<mapping resource="hbm/USEmbassy.hbm.xml" />
<mapping resource="hbm/User.hbm.xml" />
<mapping resource="hbm/Visa.hbm.xml" />

64
services/pom.xml
View File

@ -23,11 +23,6 @@
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
<!-- Spring -->
<spring-framework.version>4.0.3.RELEASE</spring-framework.version>
<spring-data-jpa.version>1.5.1.RELEASE</spring-data-jpa.version>
<!-- Java EE / Java SE dependencies -->
<jsp.version>2.2</jsp.version>
<jstl.version>1.2</jstl.version>
@ -90,13 +85,6 @@
<version>1.4</version>
</dependency>
<!-- Test Artifacts -->
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-test</artifactId>
<version>${spring-framework.version}</version>
<scope>test</scope>
</dependency>
<!-- used by Spring MVC Test framework -->
<dependency>
<groupId>org.hamcrest</groupId>
@ -141,12 +129,6 @@
<artifactId>geronimo-activation_1.1_spec</artifactId>
<version>1.1</version>
</dependency>
<dependency>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
<version>1.1.3</version>
</dependency>
<dependency>
<groupId>org.apache.ws.commons.schema</groupId>
@ -167,56 +149,10 @@
<version>1.2.3</version>
</dependency>
<dependency>
<groupId>commons-httpclient</groupId>
<artifactId>commons-httpclient</artifactId>
<version>3.1</version>
</dependency>
<dependency>
<groupId>org.apache.jclouds.labs</groupId>
<artifactId>rackspace-cloudfiles-us</artifactId>
<version>1.9.0</version>
</dependency>
</dependencies>
<dependencyManagement>
<dependencies>
<!-- Force the version of all the spring jars (core, beans, context, ...) pulled by spring-data-jpa:1.3.4.RELEASE to 3.2.x when spring-data
pulls the 3.1.x versions to prevent some misbehaviors of maven which sometimes pulls both 3.2.x and 3.1.x versions of spring-core, spring-beans
and spring-context -->
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-core</artifactId>
<version>${spring-framework.version}</version>
</dependency>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-beans</artifactId>
<version>${spring-framework.version}</version>
</dependency>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-context</artifactId>
<version>${spring-framework.version}</version>
</dependency>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-orm</artifactId>
<version>${spring-framework.version}</version>
</dependency>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-aop</artifactId>
<version>${spring-framework.version}</version>
</dependency>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-tx</artifactId>
<version>${spring-framework.version}</version>
</dependency>
<dependency>
<groupId>jaxen</groupId>
<artifactId>jaxen</artifactId>

132
services/src/main/java/com/usvisatrack/services/VisaImporter.java
View File

@ -1,14 +1,15 @@
package com.usvisatrack.services;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
import org.apache.commons.cli.GnuParser;
import org.apache.commons.cli.DefaultParser;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.math.NumberUtils;
import org.joda.time.DateTime;
import org.joda.time.format.DateTimeFormat;
import org.jsoup.Jsoup;
@ -20,10 +21,12 @@ import org.slf4j.LoggerFactory;
import com.usvisatrack.core.common.data.VisaEntry;
import com.usvisatrack.core.common.data.VisaStatus;
import com.usvisatrack.core.dao.model.USEmbassy;
import com.usvisatrack.core.dao.model.CheckeeVisa;
import com.usvisatrack.core.dao.model.User;
import com.usvisatrack.core.dao.model.Visa;
import com.usvisatrack.core.dao.model.VisaClass;
import com.usvisatrack.core.factories.USFactory;
import com.usvisatrack.core.factories.UserFactory;
import com.usvisatrack.core.factories.VisaFactory;
import com.usvisatrack.services.common.DataCrawl;
@ -36,9 +39,7 @@ import com.usvisatrack.services.common.DataCrawl;
public class VisaImporter extends DataCrawl {
private static final Logger logger = LoggerFactory.getLogger(VisaImporter.class);
public final static String ITEM_FTP_FOLDER = "/home/ftp/com-bcodepot/item";
public final static String ITEM_PROCESSED_FOLDER = "/home/data/origin/com-bcodepot-www/processed/item";
public final static String APIDATA_CDN_FOLDER = "/home/cdn/com-bcodepot-www/data/apidata";
public final static String URL_CHECKEE = "https://www.checkee.info/main.php?dispdate=";
public static HashMap<String, VisaClass> visaClassMap = new HashMap<String, VisaClass>();
@ -50,7 +51,7 @@ public class VisaImporter extends DataCrawl {
options.addOption("l", true, "Clean all agents input information");
options.addOption("h", true, "Clean one agent by input email address");
CommandLineParser parser = new GnuParser();
CommandLineParser parser = new DefaultParser();
// parse command line
try {
@ -72,43 +73,70 @@ public class VisaImporter extends DataCrawl {
@Override
public void run() {
initData(); // Init data from database
crawlWebVisa();
crawlWebVisa(URL_CHECKEE + DateTimeFormat.forPattern("yyyy-MM").print(new DateTime()));
crawlWebVisa(URL_CHECKEE + DateTimeFormat.forPattern("yyyy-MM").print(new DateTime().minusMonths(1)));
}
/**
* crawlWebItem by parse web page
*/
private void crawlWebVisa() {
private void crawlWebVisa(String cURL) {
logger.error("Crawl Web Data to load item info.");
List<Visa> visaList = new ArrayList<Visa>();
boolean isNewVisa = false;
Document doc = null;
try {
doc = Jsoup.connect("https://www.checkee.info/main.php?dispdate=2017-07").get();
doc = Jsoup.connect(cURL).get();
Elements newsHeadlines = doc.select("table");
Element table = newsHeadlines.get(6);
int i = 0;
// int i = 0;
for (Element row : table.select("tr")) {
i++;
// i++;
Elements tds = row.select("td");
if (!StringUtils.equalsIgnoreCase("ID", tds.get(1).text())) {
Visa visa = new Visa();
String checkeeCaseNumber = getCheckeeCaseNumber(tds.get(0));
Visa visa = VisaFactory.getVisaFromCheckee(NumberUtils.toLong(checkeeCaseNumber));
if (visa == null) {
visa = new Visa();
isNewVisa = true;
}
// SET VISA CLASS
String visaClassName = StringUtils.trimToEmpty(tds.get(2).text());
if (StringUtils.isNotBlank(visaClassName)) {
switch (visaClassName) {
case "B1":
visaClassName = "B-1";
break;
case "B2":
visaClassName = "B-2";
break;
case "H1":
visaClassName = "H1-B";
break;
case "H4":
visaClassName = "H-4";
break;
case "F1":
visaClassName = "F-1";
break;
case "F2":
visaClassName = "F-2";
break;
case "J1":
visaClassName = "J-1";
break;
case "L1":
visaClassName = "L-1";
break;
}
visa.setVisaClass(visaClassMap.get(visaClassName));
@ -132,11 +160,26 @@ public class VisaImporter extends DataCrawl {
String usEmbassyName = StringUtils.upperCase(StringUtils.trimToEmpty(tds.get(4).text()));
if (StringUtils.isNotBlank(usEmbassyName)) {
switch (usEmbassyName) {
case "BEIJING":
visa.setUsEmbassy(USFactory.searchUSEmbassy("China", "BeiJing"));
break;
case "CHENGDU":
visa.setUsEmbassy(USFactory.searchUSEmbassy("China", "Chengdu"));
break;
case "GUANGZHOU":
visa.setUsEmbassy(USFactory.searchUSEmbassy("China", "Guangzhou"));
break;
case "RENEWAL":
visa.setVisaEntry(VisaEntry.RENEWAL);
case "SHANGHAI":
visa.setUsEmbassy(USFactory.searchUSEmbassy("China", "Shanghai"));
break;
case "SHENYANG":
visa.setUsEmbassy(USFactory.searchUSEmbassy("China", "Shenyang"));
break;
case "WUHAN":
visa.setUsEmbassy(USFactory.searchUSEmbassy("China", "Wuhan"));
break;
case "HONGKONG":
visa.setUsEmbassy(USFactory.searchUSEmbassy("China", "HongKong"));
break;
}
@ -166,21 +209,25 @@ public class VisaImporter extends DataCrawl {
if (StringUtils.isNotBlank(dateVisaInterview)) {
visa.setDateVisaInterview(DateTimeFormat.forPattern("yyyy-MM-dd").parseDateTime(dateVisaInterview).toDate());
}
if (StringUtils.isNotBlank(dateVisaIssued)) {
if (StringUtils.isNotBlank(dateVisaIssued) && !StringUtils.equals(dateVisaIssued, "0000-00-00")) {
visa.setDateVisaIssued(DateTimeFormat.forPattern("yyyy-MM-dd").parseDateTime(dateVisaIssued).toDate());
}
Element link = tds.get(10).select("a").first();
logger.debug(">>>>>>>>>>>>>>>[{}]", link.attr("href"));
updateVisaNote(visa, checkeeCaseNumber);
visa.setModifyDate(new Date());
visa.setUser(getUser(checkeeCaseNumber));
VisaFactory.save(visa);
if (isNewVisa) {
CheckeeVisa checkeeVisa = new CheckeeVisa();
checkeeVisa.setId(NumberUtils.toLong(checkeeCaseNumber));
checkeeVisa.setVisaID(visa.getId());
VisaFactory.save(checkeeVisa);
}
}
if (i == 2)
break;
// if (i == 2)
// break;
}
@ -213,13 +260,48 @@ public class VisaImporter extends DataCrawl {
*/
private String getCheckeeCaseNumber(Element element) {
String checkeeCaseNumber = null;
Element link = element.select("a").first();
checkeeCaseNumber = StringUtils.substringAfterLast(link.attr("href"), "casenum=");
return checkeeCaseNumber;
}
/**
*
* @param userName
* @return
*/
private User getUser(String checkeeCaseNumber) {
String userName = null;
String userEmail = null;
User user = null;
Document doc = null;
try {
doc = Jsoup.connect("https://www.checkee.info/update.php?casenum=" + checkeeCaseNumber).get();
Elements elements = doc.select("input[name=email_dis]");
userEmail = elements.first().val();
elements = doc.select("b");
userName = StringUtils.trim(StringUtils.substringAfter(elements.get(7).text(), "ID:"));
user = UserFactory.get(userName);
if (user == null) {
user = new User();
user.setUserName(userName);
user.setEmail(userEmail);
UserFactory.save(user);
}
} catch (Exception e) {
// TODO: handle exception
e.printStackTrace();
}
return user;
}
private void updateVisaNote(Visa visa, String checkeeCaseNumber) {
Document doc = null;

2
services/src/main/resources/hibernate.properties
View File

@ -1,7 +1,7 @@
# JDBC Connection
hibernate.connection.driver_class=org.mariadb.jdbc.Driver
hibernate.dialect=org.hibernate.dialect.MySQL5Dialect
hibernate.connection.url=jdbc:mysql://db.usvisatrack.com/northtecom_usvisatrack?serverTimezone=EST&useUnicode=true&characterEncoding=UTF8&autoReconnect=true
hibernate.connection.url=jdbc:mysql://db.usvisatrack.com/northtecom_usvisatrack?serverTimezone=EST&useUnicode=true&characterEncoding=UTF8MB64&autoReconnect=true
# Username and Password
hibernate.connection.username=usvisatrack.web