iSharkFly-Open
/
covid-19
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

init submit the project file

This commit is contained in:
YuCheng Hu 2020-04-13 10:05:46 -04:00
parent 9f09b043d8
commit 85f1a62e61
31 changed files with 3549 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
covid-19-service/README.adoc Normal file
View File

@ -0,0 +1,13 @@
== REoC MLS Batch
MLS Batch process
== What you'll need
FEEDS project to send listing data to thirty party company.
mvn install:install-file -Dfile=D:\workdir\Verani-Realty\src\svn-verani\references\RETS\retsiq-core-api.jar -DgroupId=retsiq -DartifactId=core-api -Dversion=1.0 -Dpackaging=jar
mvn install:install-file -Dfile=D:\workdir\Verani-Realty\src\svn-verani\references\RETS\retsiq-core-client.jar -DgroupId=retsiq -DartifactId=core-client -Dversion=1.0 -Dpackaging=jar
mvn install:install-file -Dfile=D:\workdir\Verani-Realty\src\svn-verani\references\RETS\retsiq-core-metadata.jar -DgroupId=retsiq -DartifactId=core-metadata -Dversion=1.0 -Dpackaging=jar
mvn install:install-file -Dfile=D:\workdir\Verani-Realty\src\svn-verani\references\RETS\retsiq-core-simpleclient.jar -DgroupId=retsiq -DartifactId=core-simpleclient -Dversion=1.0 -Dpackaging=jar
mvn install:install-file -Dfile=D:\workdir\Verani-Realty\src\svn-verani\references\RETS\retsiq-core-util.jar -DgroupId=retsiq -DartifactId=core-util -Dversion=1.0 -Dpackaging=jar

2
covid-19-service/build.gradle
View File

@ -11,8 +11,8 @@ repositories {
}
dependencies {
implementation project(":covid-19-common")
// COMMONS
implementation 'org.apache.commons:commons-lang3:3.9'
implementation 'org.apache.commons:commons-vfs2:2.0'

21
covid-19-service/src/assembly/zipAssembly.xml Normal file
View File

@ -0,0 +1,21 @@
<assembly xmlns="http://maven.apache.org/ASSEMBLY/2.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/ASSEMBLY/2.0.0 http://maven.apache.org/xsd/assembly-2.0.0.xsd">
<id>zip-with-dependencies</id>
<formats>
<format>zip</format>
</formats>
<files>
<file>
<source>target/${project.artifactId}-${project.version}.jar</source>
<outputDirectory>./</outputDirectory>
<destName>${project.artifactId}.jar</destName>
</file>
</files>
<dependencySets>
<dependencySet>
<outputDirectory>lib/</outputDirectory>
<useProjectArtifact>false</useProjectArtifact>
<scope>runtime</scope>
</dependencySet>
</dependencySets>
</assembly>

262
covid-19-service/src/main/java/com/ossez/covid19/service/Main.java Normal file
View File

@ -0,0 +1,262 @@
package com.ossez.covid19.service;
import com.ossez.covid19.common.Factory;
import com.ossez.covid19.common.mls.feeds.FeedStep;
import com.ossez.covid19.common.mls.feeds.ListingFeed;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
import org.apache.commons.cli.GnuParser;
import org.apache.commons.cli.Options;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.FileNotFoundException;
import java.util.*;
//import org.apache.log4j.Level;
public class Main {
private static final Logger logger = LoggerFactory.getLogger(Main.class);
private static Options options = new Options();
private static Properties properties = new Properties();
private static CommandLine cl = null;
private static Map<String, ListingFeed> feeds = new HashMap<String, ListingFeed>();
private static boolean dryRun = false;
private static List<ListingFeed> feedsToRun = new ArrayList<ListingFeed>();
private static List<FeedStep> feedSteps = new ArrayList<FeedStep>();
private static int limit = 0;
private static boolean force = false;
private static List<Integer> mlsNumbers = new ArrayList<Integer>();
public static void main(String[] args) {
// get the idx feed properties file
Main.parseProperties();
// load console options
Main.parseCommandLine(args);
logger.debug("Starting feeds...");
System.out.println("starting feeds...");
// execute the feeds
Main.executeFeeds();
Factory.close();
}
/**
* Executes the feeds specified in the feeds.properties file.
*/
private static void executeFeeds() {
// run through all feeds specified
for (ListingFeed feed : Main.feedsToRun) {
feed.setDryRun(Main.dryRun);
feed.setLimit(Main.limit);
feed.setMlsNumbers(Main.mlsNumbers);
// load the properties file for the feed
Properties properties = new Properties();
logger.debug("Loading: " + feed.getName() + ".properties");
// try {
// properties.load(Main.class.getClassLoader().getResourceAsStream(feed.getName() + ".properties"));
// } catch (IOException ex) {
// logger.error("Could not load " + feed.getName() + ".properties");
// continue;
// }
// feed.setProperties(properties);
feed.setForce(Main.force);
// run the feed
try {
feed.run(Main.feedSteps, null);
} catch (Exception ex) {
logger.error("Error proccessing " + feed.getName() + " feed", ex);
}
}
}
/**
* Parses the properties file to get a list of all feeds.
*/
private static void parseProperties() {
try {
// load the properties file
logger.debug("Parsing properties");
Main.properties.load(Main.class.getClassLoader().getResourceAsStream("mls.properties"));
// load the feeds
logger.debug("Loading feeds");
Set<Object> keys = Main.properties.keySet();
for (Object k : keys) {
String name = (String) k;
String feedClass = Main.properties.getProperty(name);
logger.trace("Loading feed class: " + k);
// load the class
Class<?> clazz = Class.forName(feedClass);
// make sure the class implements ListingFeed
if (ListingFeed.class.isAssignableFrom(clazz)) {
logger.trace("Adding " + feedClass + " to list of feeds.");
// instantiate the class
Main.feeds.put(name, (ListingFeed) clazz.newInstance());
} else {
logger.error("Class " + feedClass + " does not implement the com.verani.common.mls.idx.ListingFeed interface.");
}
}
// String[] feedClasses = Main.properties.getProperty("feeds").split(",");
//
// for (String feed : feedClasses) {
// logger.trace("Loading feed class: " + feed);
//
// String name = feed.substring(0, feed.indexOf(";"));
// String feedClass = feed.substring(feed.indexOf(";") + 1);
//
// // load the class
// Class<?> clazz = Class.forName(feedClass);
//
// // make sure the class implements ListingFeed
// if (ListingFeed.class.isAssignableFrom(clazz)) {
// logger.trace("Adding " + feedClass + " to list of feeds.");
// // instantiate the class
// Main.feeds.put(name, (ListingFeed) clazz.newInstance());
// } else {
// logger.error("Class " + feedClass + " does not implement the com.verani.common.mls.idx.ListingFeed interface.");
// }
// }
} catch (Exception ex) {
ex.printStackTrace();
logger.error("Could not parse feed properties", ex);
}
}
/**
* Handles creation of console options.
*/
private static void parseCommandLine(String[] args) {
// configure command line options
Main.options.addOption("f", true, "List of which feeds to run.");
Main.options.addOption("s", true, "List of which steps in the feed should be run.");
Main.options.addOption("d", false, "Instructs the process to preform a \"dry\" run, meaning no data will actually be updated.");
Main.options.addOption("l", true, "Max limit of listings to process.");
Main.options.addOption("u", false, "Forces an update of any listings processed.");
Main.options.addOption("i", true, "One or more MLS IDs used when updating a single listings (UpdateListing step). Comma separated");
Main.options.addOption("v", false, "Enables verbose logging.");
Main.options.addOption("m", true, "Name of the FLAT TXT file that will be used to load spacific MLSNumbers");
String flatFileName = "";
// parse command line options
CommandLineParser parser = new GnuParser();
try {
Main.cl = parser.parse(Main.options, args);
// make sure feeds were specified
if (Main.cl.hasOption("f")) {
// get the feeds to run
for (String feed : Main.cl.getOptionValue("f").split(",")) {
logger.debug("Adding feed " + feed + " to the list of feeds to run.");
Main.feedsToRun.add(Main.feeds.get(feed));
}
} else {
logger.error("No feeds have been specified.");
}
// make sure feed steps were specified
if (Main.cl.hasOption("s")) {
// get the feed steps
for (String feedStep : Main.cl.getOptionValue("s").split(",")) {
logger.trace("Adding feed step: " + feedStep);
Main.feedSteps.add(FeedStep.valueOf(feedStep));
}
} else {
logger.info(
"No feed steps have been specified. [Defualt run sequence: UpdateListings -> UpdateImages -> PruneListings -> UpdateSoldListings]");
Main.feedSteps.add(FeedStep.valueOf("UpdateListings"));
Main.feedSteps.add(FeedStep.valueOf("UpdateImages"));
Main.feedSteps.add(FeedStep.valueOf("PruneListings"));
Main.feedSteps.add(FeedStep.valueOf("UpdateSoldListings"));
}
// if we want to load from FILE, the look for the -f option and get name of file
if (Main.cl.hasOption("m"))
flatFileName = Main.cl.getOptionValue("m");
// add mls nos via input or flat file
if (Main.cl.hasOption("i") || Main.cl.hasOption("m")) {
String[] mlsNos = null;
// do logic to see if flat file is going to be used.
if (flatFileName != null && !flatFileName.isEmpty()) {
File file = new File(flatFileName);
if (file == null)
throw new FileNotFoundException();
String mlsstring = null;//MlsUtility.getMlsNumbersStringFromFile(file);
mlsNos = mlsstring.split(",");
} else if (Main.cl.hasOption("i")) {
mlsNos = Main.cl.getOptionValue("i", "").split(",");
} // End if/else
int maxLen = 60001; // Hardcoded for safety. Limit the maximum number of listings that can be updated from the CLI to
// prevent performance-expensive subqueries on the MLS provider's server.
if (mlsNos.length > maxLen)
throw new Exception("Maximum length of " + maxLen
+ " numbers in the single update list exceeded. Do not pass more than this number of listing numbers. This is a safety measure.");
Main.mlsNumbers = new ArrayList<Integer>();
for (String mlsNo : mlsNos) {
try {
Main.mlsNumbers.add(Integer.parseInt(mlsNo.trim()));
} catch (Exception e) {
// Throw exception, but continue.
logger.error("An error ocurred parsing list of single MLS numbers to update", e);
}
}
logger.trace("Value of mls numbers: " + Main.mlsNumbers);
}
// if the verbose option is specified, change logging
if (Main.cl.hasOption("v")) {
// org.apache.log4j.Logger.getLogger(NH.class).setLevel(Level.TRACE);
// org.apache.log4j.Logger.getLogger(MA.class).setLevel(Level.TRACE);
// org.apache.log4j.Logger.getLogger(MreisRetsFeed.class).setLevel(Level.TRACE);
// org.apache.log4j.Logger.getLogger(Main.class).setLevel(Level.TRACE);
// org.apache.log4j.Logger.getLogger(NnerenRetsFeed.class).setLevel(Level.TRACE);
// org.apache.log4j.Logger.getLogger(MlsPinRetsFeed.class).setLevel(Level.TRACE);
// org.apache.log4j.Logger.getLogger(NnerenRetsSoldFeed.class).setLevel(Level.TRACE);
}
// get the dry run option
Main.dryRun = Main.cl.hasOption("d");
logger.trace("Value of dryRun: " + dryRun);
// get the limit option
// Main.limit = Utility.parseInt(Main.cl.getOptionValue("l", "0"));
logger.trace("Value of limit: " + Main.limit);
// get the force option
Main.force = Main.cl.hasOption("u");
logger.trace("Value of force: " + Main.force);
} catch (Exception ex) {
logger.error("An error ocurred parsing command line arguments", ex);
}
}
}

1442
covid-19-service/src/main/java/com/ossez/covid19/service/NH.java Normal file
View File

File diff suppressed because it is too large Load Diff

51
covid-19-service/src/main/java/com/ossez/covid19/service/batch/Application.java Normal file
View File

@ -0,0 +1,51 @@
package com.ossez.covid19.service.batch;
import org.springframework.batch.core.Job;
import org.springframework.batch.core.JobParameters;
import org.springframework.batch.core.JobParametersBuilder;
import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing;
import org.springframework.batch.core.launch.JobLauncher;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.CommandLineRunner;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
/**
* @author YuCheng Hu
*/
@SpringBootApplication
@EnableBatchProcessing
public class Application implements CommandLineRunner {
@Autowired
JobLauncher jobLauncher;
@Autowired
Job cloudClean;
/**
* Main function for service application
*
* @param args
* @throws Exception
*/
public static void main(String... args) throws Exception {
SpringApplication.run(Application.class, args);
// SpringApplication app = new SpringApplication(Application.class);
// app.setWebEnvironment(false);
// ApplicationContext ctx= app.run(args);
}
@Override
public void run(String... args) throws Exception {
for (String arg: args) {
System.out.println(arg);
}
JobParameters jobParameters =
new JobParametersBuilder().addString("JobID", String.valueOf(System.currentTimeMillis())).toJobParameters();
jobLauncher.run(cloudClean, jobParameters);
}
}

33
covid-19-service/src/main/java/com/ossez/covid19/service/batch/JobCompletionNotificationListener.java Normal file
View File

@ -0,0 +1,33 @@
package com.ossez.covid19.service.batch;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.batch.core.BatchStatus;
import org.springframework.batch.core.JobExecution;
import org.springframework.batch.core.listener.JobExecutionListenerSupport;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.stereotype.Component;
@Component
public class JobCompletionNotificationListener extends JobExecutionListenerSupport {
private static final Logger log = LoggerFactory.getLogger(JobCompletionNotificationListener.class);
private final JdbcTemplate jdbcTemplate;
@Autowired
public JobCompletionNotificationListener(JdbcTemplate jdbcTemplate) {
this.jdbcTemplate = jdbcTemplate;
}
@Override
public void afterJob(JobExecution jobExecution) {
if(jobExecution.getStatus() == BatchStatus.COMPLETED) {
log.info("!!! JOB FINISHED! Time to verify the results");
}
}
}

37
covid-19-service/src/main/java/com/ossez/covid19/service/batch/Person.java Normal file
View File

@ -0,0 +1,37 @@
package com.ossez.covid19.service.batch;
public class Person {
private String lastName;
private String firstName;
public Person() {
}
public Person(String firstName, String lastName) {
this.firstName = firstName;
this.lastName = lastName;
}
public void setFirstName(String firstName) {
this.firstName = firstName;
}
public String getFirstName() {
return firstName;
}
public String getLastName() {
return lastName;
}
public void setLastName(String lastName) {
this.lastName = lastName;
}
@Override
public String toString() {
return "firstName: " + firstName + ", lastName: " + lastName;
}
}

23
covid-19-service/src/main/java/com/ossez/covid19/service/batch/PersonItemProcessor.java Normal file
View File

@ -0,0 +1,23 @@
package com.ossez.covid19.service.batch;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.batch.item.ItemProcessor;
public class PersonItemProcessor implements ItemProcessor<Person, Person> {
private static final Logger log = LoggerFactory.getLogger(PersonItemProcessor.class);
@Override
public Person process(final Person person) throws Exception {
final String firstName = person.getFirstName().toUpperCase();
final String lastName = person.getLastName().toUpperCase();
final Person transformedPerson = new Person(firstName, lastName);
log.info("Converting (" + person + ") into (" + transformedPerson + ")");
return transformedPerson;
}
}

67
covid-19-service/src/main/java/com/ossez/covid19/service/batch/jobs/BatchConfiguration.java Normal file
View File

@ -0,0 +1,67 @@
package com.ossez.covid19.service.batch.jobs;
import com.ossez.covid19.service.batch.steps.LinesWriter;
import com.ossez.covid19.common.models.Listing;
import com.ossez.covid19.service.batch.steps.LineProcessor;
import com.ossez.covid19.service.batch.steps.LineReader;
import org.springframework.amqp.rabbit.core.RabbitTemplate;
import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing;
import org.springframework.batch.core.configuration.annotation.JobBuilderFactory;
import org.springframework.batch.core.configuration.annotation.StepBuilderFactory;
import org.springframework.batch.item.ItemProcessor;
import org.springframework.batch.item.ItemReader;
import org.springframework.batch.item.ItemWriter;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
@Configuration
@EnableBatchProcessing
public class BatchConfiguration {
@Autowired
public JobBuilderFactory jobBuilderFactory;
@Autowired
public StepBuilderFactory stepBuilderFactory;
@Autowired
private RabbitTemplate rabbitTemplate;
@Bean
public ItemReader<Listing> itemReader() {
return new LineReader();
}
@Bean
public ItemProcessor<Listing, Listing> itemProcessor() {
return new LineProcessor();
}
@Bean
public ItemWriter<Listing> itemWriter() {
return new LinesWriter();
}
// @Bean
// public Job importUserJob(JobCompletionNotificationListener listener, Step step1) {
// return jobBuilderFactory.get("importUserJob")
// .incrementer(new RunIdIncrementer())
// .listener(listener)
// .flow(step1)
// .end()
// .build();
// }
//
// @Bean
// public Step step1(ItemReader<Listing> itemReader, ItemProcessor<Listing, Listing> itemProcessor, ItemWriter<Listing> itemWriter) {
// return stepBuilderFactory.get("step1")
// .<Listing, Listing>chunk(100)
// .reader(itemReader)
// .processor(itemProcessor)
// .writer(itemWriter)
// .build();
// }
}

60
covid-19-service/src/main/java/com/ossez/covid19/service/batch/jobs/CloudJobConf.java Normal file
View File

@ -0,0 +1,60 @@
package com.ossez.covid19.service.batch.jobs;
import com.ossez.covid19.service.batch.tasklet.AwsTasklet;
import com.ossez.covid19.common.models.Listing;
import com.ossez.covid19.service.batch.JobCompletionNotificationListener;
import org.springframework.batch.core.Job;
import org.springframework.batch.core.Step;
import org.springframework.batch.core.configuration.annotation.JobBuilderFactory;
import org.springframework.batch.core.configuration.annotation.StepBuilderFactory;
import org.springframework.batch.core.launch.support.RunIdIncrementer;
import org.springframework.batch.item.ItemProcessor;
import org.springframework.batch.item.ItemReader;
import org.springframework.batch.item.ItemWriter;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.core.io.FileSystemResource;
@Configuration
public class CloudJobConf {
@Autowired
public JobBuilderFactory jobBuilderFactory;
@Autowired
public StepBuilderFactory stepBuilderFactory;
@Bean
public Job cloudClean(JobCompletionNotificationListener listener, Step stepAws, Step deleteFilesStep) {
return jobBuilderFactory.get("cloudClean")
.incrementer(new RunIdIncrementer())
.listener(listener)
.flow(stepAws).next(deleteFilesStep)
.end()
.build();
}
@Bean
public Step stepAws(ItemReader<Listing> itemReader, ItemProcessor<Listing, Listing> itemProcessor, ItemWriter<Listing> itemWriter) {
return stepBuilderFactory.get("stepAws")
.<Listing, Listing>chunk(100)
.reader(itemReader)
.processor(itemProcessor)
.writer(itemWriter)
.build();
}
@Bean
public Step deleteFilesStep(StepBuilderFactory stepBuilders) {
return stepBuilders.get("deleteFilesStep")
.tasklet(fileDeletingTasklet()).build();
}
@Bean
public AwsTasklet fileDeletingTasklet() {
return new AwsTasklet(
new FileSystemResource("target/test-inputs"));
}
}

33
covid-19-service/src/main/java/com/ossez/covid19/service/batch/steps/LineProcessor.java Normal file
View File

@ -0,0 +1,33 @@
package com.ossez.covid19.service.batch.steps;
import com.ossez.covid19.common.models.Listing;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.batch.core.ExitStatus;
import org.springframework.batch.core.StepExecution;
import org.springframework.batch.core.StepExecutionListener;
import org.springframework.batch.item.ItemProcessor;
public final class LineProcessor implements ItemProcessor<Listing, Listing>, StepExecutionListener {
private final Logger logger = LoggerFactory.getLogger(LineProcessor.class);
@Override
public void beforeStep(StepExecution stepExecution) {
logger.debug("Line Processor initialized.");
}
@Override
public Listing process(Listing listing) throws Exception {
// long age = ChronoUnit.YEARS.between(line.getDob(), LocalDate.now());
// logger.debug("Calculated age " + age + " for line " + line.toString());
// line.setAge(age);
logger.debug("MLSNumber : [{}]", listing.getMlsNumber());
return listing;
}
@Override
public ExitStatus afterStep(StepExecution stepExecution) {
logger.debug("Line Processor ended.");
return ExitStatus.COMPLETED;
}
}

52
covid-19-service/src/main/java/com/ossez/covid19/service/batch/steps/LineReader.java Normal file
View File

@ -0,0 +1,52 @@
package com.ossez.covid19.service.batch.steps;
import com.ossez.covid19.common.models.Listing;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.batch.core.ExitStatus;
import org.springframework.batch.core.StepExecution;
import org.springframework.batch.core.StepExecutionListener;
import org.springframework.batch.item.ItemReader;
import java.util.ArrayList;
import java.util.List;
public class LineReader implements ItemReader<Listing>, StepExecutionListener {
private final Logger logger = LoggerFactory.getLogger(LineReader.class);
private int nextStudentIndex;
private List<Listing> listingList = new ArrayList<Listing>();
@Override
public void beforeStep(StepExecution stepExecution) {
// fu = new FileUtils("taskletsvschunks/input/tasklets-vs-chunks.csv");
Listing listing = new Listing();
listing.setMlsNumber("0");
listingList.add(listing);
logger.debug("Line Reader initialized.");
}
@Override
public Listing read() throws Exception {
// Line line = fu.readLine();
// if (line != null) logger.debug("Read line: " + line.toString());
Listing nextListing = null;
if (nextStudentIndex <1) {
nextListing = listingList.get(nextStudentIndex);
nextStudentIndex++;
}
return nextListing;
}
@Override
public ExitStatus afterStep(StepExecution stepExecution) {
// fu.closeReader();
logger.debug("Line Reader ended.");
return ExitStatus.COMPLETED;
}
}

67
covid-19-service/src/main/java/com/ossez/covid19/service/batch/steps/LinesWriter.java Normal file
View File

@ -0,0 +1,67 @@
package com.ossez.covid19.service.batch.steps;
import com.ossez.covid19.common.models.Listing;
import org.joda.time.DateTime;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.amqp.core.Queue;
import org.springframework.amqp.rabbit.core.RabbitTemplate;
import org.springframework.batch.core.ExitStatus;
import org.springframework.batch.core.StepExecution;
import org.springframework.batch.core.StepExecutionListener;
import org.springframework.batch.item.ItemWriter;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Bean;
import org.springframework.amqp.rabbit.connection.CachingConnectionFactory;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
public class LinesWriter implements ItemWriter<Listing>, StepExecutionListener {
private final Logger logger = LoggerFactory.getLogger(LinesWriter.class);
// private FileUtils fu;
public static final String MY_QUEUE_NAME = "com.ossez.real.estate";
@Autowired
private RabbitTemplate rabbitTemplate;
private CachingConnectionFactory cachingConnectionFactory;
@Bean
Queue myQueue() {
Map<String, Object> args = new HashMap<>();
args.put("x-queue-type", "classic");
return new Queue(MY_QUEUE_NAME, true, false, false, args);
}
@Override
public void beforeStep(StepExecution stepExecution) {
rabbitTemplate.setRoutingKey(MY_QUEUE_NAME);
rabbitTemplate.setDefaultReceiveQueue(MY_QUEUE_NAME);
}
@Override
public ExitStatus afterStep(StepExecution stepExecution) {
cachingConnectionFactory = (CachingConnectionFactory) rabbitTemplate.getConnectionFactory();
cachingConnectionFactory.destroy();
return ExitStatus.COMPLETED;
}
@Override
public void write(List<? extends Listing> items) throws Exception {
DateTime dateTime = new DateTime();
items.parallelStream().forEach(item -> rabbitTemplate.convertAndSend(item));
// for (Listing item : items) {
// rabbitTemplate.convertAndSend(item);
// }
logger.debug(">>>{}", new DateTime().getMillis() - dateTime.getMillis());
// rabbitTemplate.stop();
}
}

105
covid-19-service/src/main/java/com/ossez/covid19/service/batch/tasklet/AwsTasklet.java Normal file
View File

@ -0,0 +1,105 @@
package com.ossez.covid19.service.batch.tasklet;
import com.amazonaws.AmazonServiceException;
import com.amazonaws.auth.AWSCredentials;
import com.amazonaws.auth.AWSStaticCredentialsProvider;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.regions.Regions;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
import com.amazonaws.services.s3.model.*;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.math.NumberUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.batch.core.StepContribution;
import org.springframework.batch.core.scope.context.ChunkContext;
import org.springframework.batch.core.step.tasklet.Tasklet;
import org.springframework.batch.repeat.RepeatStatus;
import org.springframework.core.io.Resource;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.List;
/**
*
*/
public class AwsTasklet implements Tasklet {
private final Logger logger = LoggerFactory.getLogger(AwsTasklet.class);
private Resource directory;
public AwsTasklet(Resource directory) {
this.directory = directory;
}
@Override
public RepeatStatus execute(StepContribution contribution, ChunkContext chunkContext) throws Exception {
logger.info("Download Listing Photo from AWS Bucket: com.ossez.real.estate");
AWSCredentials awsCreds = new BasicAWSCredentials("AKIAIJDKPNI3PL7E6QXQ", "Jfaq2mGcogodLKJa7wae9dgd+M3bQ6g5XjkOt1ZV");
final AmazonS3 s3 =
AmazonS3ClientBuilder.standard().withCredentials(new AWSStaticCredentialsProvider(awsCreds)).withRegion(Regions.US_EAST_1).build();
Bucket named_bucket = null;
List<Bucket> buckets = s3.listBuckets();
for (Bucket b : buckets) {
if (b.getName().equals("com.ossez.real.estate")) {
named_bucket = b;
}
}
try {
ListObjectsV2Result result = s3.listObjectsV2("com.ossez.real.estate", "verani-listing-img");
List<S3ObjectSummary> objects = result.getObjectSummaries();
for (S3ObjectSummary os : objects) {
String fileName = StringUtils.substringAfterLast(os.getKey(), "/");
System.out.println("* " + fileName);
String folderName = "" + (NumberUtils.toLong(StringUtils.substringBefore(fileName, "-")) / 10000);
if (!(StringUtils.equalsIgnoreCase("RESI-ALL.xml", fileName) || StringUtils.equalsIgnoreCase("COMM-ALL.xml", fileName))) {
S3Object o = s3.getObject("com.ossez.real.estate", os.getKey());
S3ObjectInputStream s3is = o.getObjectContent();
FileOutputStream fos =
FileUtils.openOutputStream(new File("/home/vhosts/ossez.com/repo.ossez.com/httpdocs/real-estate/listing-photo/" + folderName + "/" + StringUtils.substringBefore(fileName,
"-") + "/" + fileName));
byte[] read_buf = new byte[1024];
int read_len = 0;
while ((read_len = s3is.read(read_buf)) > 0) {
fos.write(read_buf, 0, read_len);
}
s3is.close();
fos.close();
}
s3.deleteObject("com.ossez.real.estate", os.getKey());
}
} catch (AmazonServiceException e) {
System.err.println(e.getErrorMessage());
System.exit(1);
} catch (FileNotFoundException e) {
System.err.println(e.getMessage());
System.exit(1);
} catch (IOException e) {
System.err.println(e.getMessage());
System.exit(1);
}
return RepeatStatus.FINISHED;
}
}

396
covid-19-service/src/main/java/com/ossez/covid19/service/cloud/CloudPhotos.java Normal file
View File

@ -0,0 +1,396 @@
package com.ossez.covid19.service.cloud;
//package com.verani.feeds.mls.cloud;
//
//import java.io.BufferedReader;
//import java.io.File;
//import java.io.FileReader;
//import java.io.FileWriter;
//import java.io.IOException;
//import java.text.NumberFormat;
//import java.util.ArrayList;
//import java.util.HashMap;
//import java.util.Iterator;
//import java.util.List;
//import java.util.Locale;
//import java.util.Map;
//import java.util.Properties;
//import java.util.UUID;
//
//import org.apache.log4j.Logger;
//
//import com.bigllc.retsiq.simpleclient.ObjectRecord;
//import com.bigllc.retsiq.simpleclient.RETSConnection;
//import com.bigllc.retsiq.simpleclient.RETSUserSession;
//import com.verani.common.Factory;
//import com.verani.common.backPage.factories.ConfigSettingFactory;
//import com.verani.common.cloud.CloudFiles;
//import com.verani.common.mls.Listing;
//import com.verani.common.mls.ListingType;
//import com.verani.common.mls.factories.FeedFactory;
//import com.verani.common.mls.factories.ListingFactory;
//import com.ossez.reoc.mls.rets.NnerenRetsFeed;
//
///**
// * This class will bring us up-to-speed with listing photos in the Cloud.
// * The photos will be pulled directly from corresponding RETS feed and then
// * stored in Rackspace's Cloud Files CDN (and we will use SSL URL to avoid SSL mixed content).
// *
// * @author Mark Kelleher (mark.kelleher@verani.com)
// * @date Sep 24, 2014
// * @jira BVR-190
// *
// */
//public class CloudPhotos {
//
// private static Logger log = Logger.getLogger(CloudPhotos.class);
// private static final String PHOTO_PROCESS_LOG_FILE_NAME = "processed-listings.log";
// private static Properties properties = new Properties();
//
// /**
// * Main command line method that will do all the dirty work.
// *
// * @author Mark Kelleher (mark.kelleher@verani.com)
// * @date Sep 24, 2014
// *
// * @param args
// */
// public static void main(String[] args) {
//
// String tmpImageStorageLocationOnServer = "", path = "", imageStorageLocationInCloudFiles = "";
// Map<Integer, Integer> listingPhotos = new HashMap<Integer, Integer>();
// List<Integer> mlsNumbersAlreadyProcessed = null;
//
// String feedTypeToProcess = "NNEREN";
// int processLimit = 0;
//
// // if passing argument, then set feed type
// if( args[0]!=null && !args[0].isEmpty() )
// feedTypeToProcess = args[0];
// // if passing LIMIT number to process, then set it.
// if( args[1]!=null && !args[1].isEmpty() )
// processLimit = Integer.parseInt(args[1]);
//
// log.info("Beginning pulling "+(processLimit==0?"ALL":String.valueOf(processLimit))+" active "+feedTypeToProcess+" images from CDN to host in Rackspace Cloud Files.");
//
// try{
//
// // load properties file
// properties.load(CloudPhotos.class.getClassLoader().getResourceAsStream(feedTypeToProcess+".properties"));
//
// // Setup some vars
// Listing listing = null;
// String listingImagePath = "";
// int totalPhotoCount = 0;
// int counter = 0;
//
// String loginurl = properties.getProperty("rets.url");
// String username = properties.getProperty("rets.user");
// String password = properties.getProperty("rets.password");
// String objectPath = properties.getProperty("rets.objectPath");
// String requestPath = "";
// File directory = null;
// List<ObjectRecord> objects = null;
//
// // open database connection
// Factory.beginTransaction();
//
// // set the temp location where to store the files on server
// path = ConfigSettingFactory.getValue("tmpListingImagePath", "/tmp/cloud-images/");
// tmpImageStorageLocationOnServer = path + UUID.randomUUID().toString()+"/";
//
// // set the destination Cloud location
// imageStorageLocationInCloudFiles = ConfigSettingFactory.getValue("listingImagePathCloudFiles", "localhost-images");
//
// // get list of all active or active w/ contract NNEREN and MLSPin listings from db
// List<Integer> mlsNumbers = null;
//
// // go get the mlsnumbers for the correct feed
// mlsNumbers = ListingFactory.getMlsNumbersByFeed(FeedFactory.get(feedTypeToProcess),0);
//
// log.info("Found "+NumberFormat.getNumberInstance(Locale.US).format(mlsNumbers.size())+" listings to images pull for.");
//
// // check to see if log file exists (used to exclude MLS#s already processed)
// File logFile = new File(path+PHOTO_PROCESS_LOG_FILE_NAME);
// if(logFile.exists() && !logFile.isDirectory()){
// mlsNumbersAlreadyProcessed = processLogFile(logFile);
// log.info("Heads up! Log file found that "+NumberFormat.getNumberInstance(Locale.US).format(mlsNumbersAlreadyProcessed.size())+" listings have already been processed and will be skipped this time around.");
// }
// // Create the connection class
// RETSConnection connection = new RETSConnection(loginurl);
//
// // Authenticate the user and get session
// RETSUserSession session = connection.getSession(username, password);
//
// // remove the MLS#s already processed
// if(mlsNumbersAlreadyProcessed!=null&&mlsNumbersAlreadyProcessed.size()>0){
// // remove from current list
// mlsNumbers.removeAll(mlsNumbersAlreadyProcessed);
// log.info("Pruned list to process for listings and have "+NumberFormat.getNumberInstance(Locale.US).format(mlsNumbers.size())+" left.");
// // add to list of removes
// for(int mlsNumber : mlsNumbersAlreadyProcessed)
// listingPhotos.put(mlsNumber, 0);
// } // End if
//
// for(int mlsNumber : mlsNumbers){
//
// // stop processing if limit has been set
// if( counter == processLimit && processLimit > 0 )
// break;
//
// // get listing info to setup the directory location
// try{
//
// // get the listing
// listing = ListingFactory.get(mlsNumber);
//
// log.info("Processing For MlsNumber : " + listing.getMlsNumber());
// // set total # of photos
// totalPhotoCount = listing.getPhotoCount();
//
// // set the image url path
// listingImagePath = NnerenRetsFeed.getListingImageUrlPath(listing);
// log.info("listingImagePath >> NnerenRetsFeed >> For MlsNumber : " + listing.getMlsNumber() + " listingImagePath >> : " + listingImagePath);
//
// // set the location to store images
// directory = new File(tmpImageStorageLocationOnServer + listingImagePath);
// directory.mkdirs();
//
// // some RETS image server logic
// if(feedTypeToProcess.equalsIgnoreCase("MLSPinRets")){
// // MLSPin is a pain and you must specify photo location for Commercial listings special
// if(listing.getType()==ListingType.Commercial){
// requestPath = String.format(objectPath,"COMM",String.valueOf(mlsNumber));
// log.debug("Yo! We got a commercial listing and pulled MLSPin Commercial photos specially.");
// }else
// requestPath = String.format(objectPath,"RESI",String.valueOf(mlsNumber));
// }else
// requestPath = String.format(objectPath,String.valueOf(mlsNumber));
//
//
// log.debug("Getting photos from: "+requestPath +" and downloading to: "+tmpImageStorageLocationOnServer + listingImagePath);
//
// // call the photos
// objects = session.getObjects(requestPath, directory);
//
// }catch(Exception ex){
// storeProcessedMlsNumbers(listingPhotos,path);
// log.error(ex.fillInStackTrace());
// ex.printStackTrace();
// } // End try/catch
//
// // record this mls number has been processed
// listingPhotos.put(mlsNumber, totalPhotoCount);
//
// counter++;
//
// } // End mls number loop
//
// // write the list to file
// storeProcessedMlsNumbers(listingPhotos,path);
//
// // Logout from the session
// session.logout();
//
// // after you got all the images, then push them all up to Cloud server
//
// try{
//
// log.info("Pushing listing images to Rackspace Cloud Files CDN for MlsNumber..." + listing.getMlsNumber()
// + " tmpImageStorageLocationOnServer : " + tmpImageStorageLocationOnServer + " imageStorageLocationInCloudFiles : " + imageStorageLocationInCloudFiles);
//
// // Use the Rackspace Cloud Files API/Examples to push the entire folder /tmp/cloud-images/
// CloudFiles.uploadDirectoryContentsToCloud(tmpImageStorageLocationOnServer, imageStorageLocationInCloudFiles);
//
// log.info("Successfully moved files to Rackspace Cloud Files CDN and removed local copies for MlsNUmber " + listing.getMlsNumber());
//
// }catch(Exception e){
// log.error("Error occured while trying to PUSH NnerenRets images to Rackspace CloudFiles" + e.getMessage());
// e.printStackTrace();
// }
//
// }catch(Exception ex){
// System.out.println("Problem processing mls numbers to get photos. [check the log: "+path+"processed-listings.log]");
// log.error("Problem processing mls numbers to get photos. [check the log: "+path+"processed-listings.log]" + ex.getMessage());
// ex.printStackTrace();
// } // End try/catch
//
// log.info("FINISHED : Downloading all ");
//
// } // End main
//
// /**
// * Open log file and put in HashMap in memory.
// *
// * @author Mark Kelleher (mark.kelleher@verani.com)
// * @date Sep 25, 2014
// *
// * @param logFile
// * @return
// */
// private static List<Integer> processLogFile(File logFile){
//
//// Map<Integer, Integer> tempList = new HashMap<Integer, Integer>();
// List<Integer> tempList = new ArrayList();
//
// try{
//
// BufferedReader br = new BufferedReader(new FileReader(logFile));
// String line = null;
// HashMap<String,String> map = new HashMap<String, String>();
//
// while((line=br.readLine())!=null){
// String str[] = line.split(",");
//// tempList.put(Integer.parseInt(str[0]), Integer.parseInt(str[1]));
// tempList.add(Integer.parseInt(str[0]));
// }
//
// }catch(Exception ex){
// log.error("processLogFile : " + ex.getMessage());
// ex.printStackTrace();
// } // End try/catch
//
// return tempList;
//
// } // End processLogFile
//
// /**
// * Will take the current hashmap and store to disk in CSV format.
// * Can be used to reference or pick up where we left off if something fails.
// *
// * @author Mark Kelleher (mark.kelleher@verani.com)
// * @date Sep 24, 2014
// *
// * @param processedListings
// */
// private static void storeProcessedMlsNumbers(Map<Integer, Integer> processedListings, String path){
//
// try{
//
// FileWriter writer = new FileWriter(path+PHOTO_PROCESS_LOG_FILE_NAME);
//
// Iterator it = processedListings.entrySet().iterator();
//
// // loop the map and create csv for storage
// while (it.hasNext()) {
// Map.Entry pairs = (Map.Entry)it.next();
// writer.append(pairs.getKey() + "," + pairs.getValue());
// writer.append('\n');
// }
//
// writer.flush();
// writer.close();
//
// }catch(IOException io){
// System.out.println("Problem writing CSV log to disk.");
// log.error("Problem writing CSV log to disk." + io.getMessage());
// io.printStackTrace();
// } // End try/catch
//
// } // End storeProcessedMlsNumbers
//
// /**********************************************/
// /******** OLD CODE TO REFERENCE ***************/
//
//// String urlFormatNneren = "http://invweb03.offutt-innovia.com/nne/%sjpg%s/%s/%s.jpg"; // old high-res url
//// String urlFormatMlsPin = "http://media.mlspin.com/photo.aspx?mls=%s&h=%s&w=%s&n=%s"; // old high-res url
//// String currentUrlFormat = "";
//// String stringMlsNumber = "";
//// String imageNumberString= "";
//// String externalURL = "";
//// URL url = null;
//// BufferedImage image = null;
//// File outputfile = null;
//
//// // loop list
//// for(int mlsNumber : mlsNumbers){
////
//// // has this mlsNumber already been processed?
//// // if yes, then ignore
//// if(listingPhotos.get(mlsNumber)==null){
////
//// counter++;
////
//// try{
////
//// // get the listing
//// listing = ListingFactory.get(mlsNumber);
////
//// // set the image url path
//// listingImagePath = NnerenRetsFeed.getListingImageUrlPath(listing);
////
//// // set the url format
//// if(listing.getFeed().getName().equalsIgnoreCase("NNEREN"))
//// currentUrlFormat = urlFormatNneren;
//// else if(listing.getFeed().getName().equalsIgnoreCase("MLSPinRets"))
//// currentUrlFormat = urlFormatMlsPin;
////
//// // set total # of photos
//// totalPhotoCount = listing.getPhotoCount();
////
//// // go get the photos
//// for (int i = 1; i < totalPhotoCount+1; i++) {
////
//// log.debug("i = "+i+" and photoCount = "+totalPhotoCount);
////
//// stringMlsNumber = Integer.toString(listing.getMlsNumber());
////
//// // create the URL to get image from
//// if(i==1)
//// imageNumberString = "";
//// else
//// imageNumberString = String.valueOf(i);
////
//// externalURL = String.format(currentUrlFormat,"high",imageNumberString,stringMlsNumber.substring(stringMlsNumber.length()-4,stringMlsNumber.length()),stringMlsNumber);
////
//// log.debug("External Image "+imageNumberString+" URL: "+externalURL);
////
//// // for this listing... go get the HIGH RES photos
//// url = new URL(externalURL);
////
//// try{
////
//// // read the url
//// image = ImageIO.read(url);
////
//// // write the image to tmp disk location to be pushed to cloud later
//// // Images will be downloaded to folder structure /tmp/cloud-images/<street number>-<street name>-<town>-<state>-<zip>-<mls number>/<mls number>_<photo number>.jpg
//// outputfile = new File(tmpImageStorageLocationOnServer + listingImagePath + listing.getMlsNumber() + "_" + (i-1) + ".jpg");
//// outputfile.mkdirs();
//// ImageIO.write(image, "jpg", outputfile);
////
//// }catch (IIOException ex){
//// // do nothing if image not found... just ignore and proceed on.
//// log.warn("ERROR : Skipping download of HIGH RES image #"+i+" for listing: "+Integer.toString(listing.getMlsNumber()));
//// }
////
//// // reset some variables
//// image = null;
//// url = null;
////
//// } // end photo loop
////
//// // record this mls number has been processed
//// listingPhotos.put(mlsNumber, totalPhotoCount);
////
//// }catch(Exception ex){
//// storeProcessedMlsNumbers(listingPhotos,path);
//// } // End try/catch
////
//// if(counter%BATCH_PROCESS_STORE_CSV == 0)
//// storeProcessedMlsNumbers(listingPhotos,path);
////
//// }else
//// log.trace("Already processed mls number: "+mlsNumber+", skipping photos for it.");
////
//// /************* TESTING ************************/
//// if(counter==11) break;
////
//// } // loop mls numbers
////
//// log.info("Downloaded photos for "+counter+" listings. Now it's time to PUSH to cloud.");
////
//// // write the list to file
//// storeProcessedMlsNumbers(listingPhotos,path);
////
//
//} // End CloudPhotos

489
covid-19-service/src/main/java/com/ossez/covid19/service/field/ParagonRetsFields.java Normal file
View File

@ -0,0 +1,489 @@
/**
*
*/
package com.ossez.covid19.service.field;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
/**
* @author YuCheng Hu
*
*/
public enum ParagonRetsFields {
MLS_NUMBER("L_ListingID"), // ListingID
LISTING_STATUS("L_Status"), // ListingStatus
STREET_NUMBER("L_AddressNumber"), // StreetNumberIDX
STREET_NAME("L_AddressStreet"), // StreetNameIDX
STREET_TYPE("L_streetdesignationid"), // STREET_TYPE
TOWN("L_City"), // OfficialTown
STATE("L_State"), // StateOrProvince
COUNTY("L_Area"), // County
ZIP("L_Zip"), // PostalCode
LATITUDE("LM_char100_5"), // Latitude
LONGITUDE("LM_char100_6"), // Longitude
// PHOTO
PHOTO_COUNT("L_PictureCount"), // photoCount
// TIMESTAMP
TS_PHOTO_UPDATED("L_Last_Photo_updt"), // Photo Updated TimeStamp
TS_CLOSED("L_ClosingDate"),
TS_LEASED("LM_DateTime_7"),
MODIFICATION_TIMESTAMP("L_UpdateDate"), // ModificationTimestamp
// MORE
FULL_BATH_ROOMS("LM_Int1_1"), // Baths
BED_ROOMS("LM_Int1_6"), // Bedrooms
AGENT_MLSID("LA1_AgentID"), // ListingAgentID
LISTAGENT_AGENTID("L_ListAgent1"), // ListAgentAgentID
LISTAGENT_OFFICEID("L_ListOffice1"), // ListAgentOfficeID
LISTPRICE("L_AskingPrice"), // ListPrice
PRICE_CLOSED("L_SoldPrice"),
PRICE_LEASED("LM_Dec_30"),
MAPCOORDINATES("LM_Char25_7"), // MapCoordinates
LISTING_TYPE("L_Type_"), // PropertyType
DESCRIPTION("LR_remarks22"), // description
ROOMS_TOTAL("LM_Int1_16"), // TotalRooms
SCHOOL_DISTRICT("LM_Char10_32"), // SchoolDistrict
SCHOOL_ELEMENTARY("LM_Char10_68"), // SchoolElementary
SCHOOL_HIGH("LM_Char10_2"), // SchoolHigh
SCHOOL_JUNIORHIG("LM_Char10_69"), // JuniorHighSchool
SEASONAL("LM_Char5_60"), // SeasonalYN
FINISHED_SQFT("LM_Int4_16"), // ApxFinSqFtTotal
LOT_SIZE("L_NumAcres"), // LotSizeArea
TAXES("LM_Dec_29"), // TaxAmount
BATH_ROOMS("LM_Int4_36"), // TotalBaths
VIRTUALTOUR_URL("VT_VTourURL"), // VirtualTourURL
YEAR_BUILT("LM_Int2_4"), // YearBuilt
AMENITIES("LFD_FeaturesInterior_15"), // Amenities
BASEMENT("LFD_BasementDescription_3"), // Basement
CONSTRUCTION("LFD_Construction_4"), // Construction
DRIVEWAY("LFD_Driveway_8"), // Driveway
ELECTRIC("LFD_Electric_9"), // Electric
EQUIPMENT_AND_APPLIANCES("LFD_Appliances_2"), // EquipmentAndAppliances
EXTERIOR("LFD_Exterior_12"), // Exterior
EXTERIOR_FEAT("LFD_FeaturesExterior_14"), // ExteriorFeat
FINANCING("LFD_Financing_18"), // Financing
FOUNDATION("LFD_Foundation_20"), // Foundation
GARAGE_AND_PARKING("LM_Char10_26"), // GarageAndParking
GARAGE("LM_Char1_2"),
GARAGE_SIZE("LM_Dec_8"), // GarageCapacityNumber
HEATING("LFD_Heating_23"), // HeatingAndCooling
COOLING("LFD_Cooling_6"), // HeatingAndCooling
HEAT_FUEL("LFD_HeatFuel_22"), // HeatFuel
INTERIOR_FEAT("LFD_SaleIncludes_100"), // InteriorFeat
LOT_DESCRIPTION("LFD_LotDescription_24"), // LotDescription
ROOF("LFD_Roof_29"), // Roof
ROADS("LFD_Roads_28"), // Roads
SEWER("LFD_Sewer_30"), // Sewer
STYLE("LFD_Style_32"), // Style
WATER_HEATER("LFD_WaterHeater_36"), // WaterHeater
DISABILITY_FEATURES("LFD_FeaturesAccessibility_13"), // DisabilityFeatures
TAX_YEAR("LM_Char10_48"), // TaxYear
LOT("LM_Char25_12"), // Lot
SURVEYED("LM_Char1_26"), // SurveyedYN
FLOODZONE("LM_Char1_18"), // FloodZone
DINING_ROOM_SIZE("LM_Char30_12"), // DiningRoomSize
ASSESSMENT_AMOUNT("LM_Int4_1"), // AssessmentAmount
BATHS_PARTIAL("LM_Int1_2"), // BathsPartial
BATHS34("LM_Int1_4"), // Baths34
HOA_FEES("LM_Dec_10"), // CondoOwnersAssociationDues
BEDROOM2_SIZE("LM_Char30_18"), // Bedroom2Size
BEDROOM3_SIZE("LM_Char30_19"), // Bedroom3Size
BEDROOM4_SIZE("LM_Char30_20"), // Bedroom4Size
LIVINGROOM_SIZE("LM_Char30_13"), // LivingRoomSize
BEDROOM_MASTER_DIMENSIONS("LM_Char30_17"), // BedRoomMasterDimensions
KITCHEN_SIZE("LM_Char30_11"), // KitchenSize
WATER_FRONTAGEP("LM_Int4_40"), // WaterFrontage
DEVELOPMENT_DESC("LM_Char50_4"), // DevelopmentDesc
FAMILYROOM_SIZE("LM_Char30_14"), // FamilyRoomSize
SERIALNUMBER1("LM_Char25_26"), // SerialNumber1
MOBILEHOME_MAKE("LM_Char30_5"), // MobileHomeMake
MODEL_NAME("LM_Char50_2"), // ModelName
// ZIP("L_ListOffice1"), // TransferFee
OCCUPANT_RESTRICTIONS("LFD_Restrictions_27"), // OccupantRestrictions
NEGOTIABLE("LFD_Possession_26"), // Negotiable
WATER_CF("LFD_Water_35"), // WaterCF
// ZIP("L_ListOffice1"), // FeeIncludes
SUITABLE_USE("LFD_SuitableUse_33"), // SuitableUse
CONDO_UNIT_NUMBER("L_Address2"), // CondoUnitNumber
FINISHED_ABOVE_GRADE("LM_Dec_21"), // FinishedAboveGrade
SQFT_FINISHED_BELOW_GRADE("LM_Dec_22"), // SqFtFinishedBelowGrade
// MEDIA
MED_MLSNUMBER("L_DisplayId"), // SqFtFinishedBelowGrade
MED_IMG_URL("MED_media_url"), // SqFtFinishedBelowGrade
MED_IMG_SEQUENCE("MED_sequence"), // SqFtFinishedBelowGrade
// CLOSE
DOM("L_DOM"),
CLOSE_AGENTID("L_SellingAgent1"),
CLOSE_OFFICEID("L_SellingOffice1");
public String val;
ParagonRetsFields(String val) {
this.val = val;
}
public String getVal() {
return val;
}
public void setVal(String val) {
this.val = val;
}
/**
* Residential
*/
public static String[] getResidentialFields() {
List<ParagonRetsFields> fieldList = new ArrayList<ParagonRetsFields>();
fieldList.add(MLS_NUMBER);
fieldList.add(LISTING_STATUS);
fieldList.add(STREET_NUMBER);
fieldList.add(STREET_NAME);
fieldList.add(STREET_TYPE);
fieldList.add(TOWN);
fieldList.add(STATE);
fieldList.add(COUNTY);
fieldList.add(ZIP);
fieldList.add(LATITUDE);
fieldList.add(LONGITUDE);
fieldList.add(PHOTO_COUNT);
fieldList.add(TS_PHOTO_UPDATED);
fieldList.add(FULL_BATH_ROOMS);
fieldList.add(BED_ROOMS);
fieldList.add(MODIFICATION_TIMESTAMP);
fieldList.add(GARAGE);
fieldList.add(GARAGE_SIZE);
fieldList.add(AGENT_MLSID);
fieldList.add(LISTAGENT_AGENTID);
fieldList.add(LISTAGENT_OFFICEID);
fieldList.add(LISTPRICE);
fieldList.add(PRICE_CLOSED);
fieldList.add(CLOSE_AGENTID);
fieldList.add(CLOSE_OFFICEID);
fieldList.add(TS_CLOSED);
fieldList.add(TS_LEASED);
fieldList.add(DOM);
fieldList.add(MAPCOORDINATES);
fieldList.add(LISTING_TYPE);
fieldList.add(DESCRIPTION);
fieldList.add(ROOMS_TOTAL);
fieldList.add(SCHOOL_DISTRICT);
fieldList.add(SCHOOL_ELEMENTARY);
fieldList.add(SCHOOL_HIGH);
fieldList.add(SCHOOL_JUNIORHIG);
fieldList.add(SEASONAL);
fieldList.add(FINISHED_SQFT);
fieldList.add(LOT_SIZE);
fieldList.add(TAXES);
fieldList.add(BATH_ROOMS);
fieldList.add(VIRTUALTOUR_URL);
fieldList.add(YEAR_BUILT);
fieldList.add(AMENITIES);
fieldList.add(BASEMENT);
fieldList.add(CONSTRUCTION);
fieldList.add(DRIVEWAY);
fieldList.add(ELECTRIC);
fieldList.add(EQUIPMENT_AND_APPLIANCES);
fieldList.add(EXTERIOR);
fieldList.add(EXTERIOR_FEAT);
fieldList.add(FINANCING);
fieldList.add(FOUNDATION);
fieldList.add(GARAGE_AND_PARKING);
fieldList.add(HEATING);
fieldList.add(COOLING);
fieldList.add(HEAT_FUEL);
fieldList.add(INTERIOR_FEAT);
fieldList.add(LOT_DESCRIPTION);
fieldList.add(ROOF);
fieldList.add(ROADS);
fieldList.add(SEWER);
fieldList.add(STYLE);
fieldList.add(WATER_HEATER);
fieldList.add(DISABILITY_FEATURES);
fieldList.add(TAX_YEAR);
fieldList.add(LOT);
fieldList.add(SURVEYED);
fieldList.add(FLOODZONE);
fieldList.add(DINING_ROOM_SIZE);
fieldList.add(ASSESSMENT_AMOUNT);
fieldList.add(BATHS_PARTIAL);
fieldList.add(BATHS34);
fieldList.add(BEDROOM2_SIZE);
fieldList.add(BEDROOM3_SIZE);
fieldList.add(BEDROOM4_SIZE);
fieldList.add(LIVINGROOM_SIZE);
fieldList.add(BEDROOM_MASTER_DIMENSIONS);
fieldList.add(KITCHEN_SIZE);
fieldList.add(WATER_FRONTAGEP);
fieldList.add(DEVELOPMENT_DESC);
fieldList.add(FAMILYROOM_SIZE);
fieldList.add(SERIALNUMBER1);
fieldList.add(MOBILEHOME_MAKE);
fieldList.add(MODEL_NAME);
fieldList.add(OCCUPANT_RESTRICTIONS);
fieldList.add(NEGOTIABLE);
fieldList.add(WATER_CF);
fieldList.add(SUITABLE_USE);
fieldList.add(CONDO_UNIT_NUMBER);
fieldList.add(FINISHED_ABOVE_GRADE);
fieldList.add(SQFT_FINISHED_BELOW_GRADE);
fieldList.add(HOA_FEES);
return fieldsToArray(fieldList);
}
/**
* Land
*
* @return
*/
public static String[] getLandFields() {
String[] retsFields = new String[] { MLS_NUMBER.val, LISTING_STATUS.val, STREET_NUMBER.val, STREET_NAME.val, STREET_TYPE.val,
TOWN.val, STATE.val, COUNTY.val, ZIP.val, LATITUDE.val, LONGITUDE.val, PHOTO_COUNT.val, TS_PHOTO_UPDATED.val,
FULL_BATH_ROOMS.val, BED_ROOMS.val, MODIFICATION_TIMESTAMP.val, AGENT_MLSID.val, LISTAGENT_AGENTID.val,
LISTAGENT_OFFICEID.val, LISTPRICE.val, MAPCOORDINATES.val, LISTING_TYPE.val, DESCRIPTION.val, SCHOOL_DISTRICT.val,
SCHOOL_ELEMENTARY.val, SCHOOL_HIGH.val, SCHOOL_JUNIORHIG.val, LOT_SIZE.val, TAXES.val, BATH_ROOMS.val,
VIRTUALTOUR_URL.val };
return retsFields;
}
/**
* Commercial Sale
*
* @return
*/
public static String[] getCommercialSaleFields() {
String[] retsFields = new String[] { MLS_NUMBER.val, LISTING_STATUS.val, STREET_NUMBER.val, STREET_NAME.val, STREET_TYPE.val,
TOWN.val, STATE.val, COUNTY.val, ZIP.val, LATITUDE.val, LONGITUDE.val, PHOTO_COUNT.val, TS_PHOTO_UPDATED.val,
FULL_BATH_ROOMS.val, BED_ROOMS.val, MODIFICATION_TIMESTAMP.val, GARAGE_SIZE.val, AGENT_MLSID.val, LISTAGENT_AGENTID.val,
LISTAGENT_OFFICEID.val, LISTPRICE.val, MAPCOORDINATES.val, LISTING_TYPE.val, DESCRIPTION.val, SEASONAL.val,
FINISHED_SQFT.val, LOT_SIZE.val, TAXES.val, BATH_ROOMS.val, VIRTUALTOUR_URL.val, YEAR_BUILT.val };
return retsFields;
}
/**
* Commercial Lease
*
* @return
*/
public static String[] getCommercialLeaseFields() {
String[] retsFields = new String[] { MLS_NUMBER.val, LISTING_STATUS.val, STREET_NUMBER.val, STREET_NAME.val, STREET_TYPE.val,
TOWN.val, STATE.val, COUNTY.val, ZIP.val, LATITUDE.val, LONGITUDE.val, PHOTO_COUNT.val, TS_PHOTO_UPDATED.val,
FULL_BATH_ROOMS.val, BED_ROOMS.val, MODIFICATION_TIMESTAMP.val, AGENT_MLSID.val, LISTAGENT_AGENTID.val,
LISTAGENT_OFFICEID.val, LISTPRICE.val, MAPCOORDINATES.val, LISTING_TYPE.val, DESCRIPTION.val, SEASONAL.val,
FINISHED_SQFT.val, LOT_SIZE.val, BATH_ROOMS.val, VIRTUALTOUR_URL.val, YEAR_BUILT.val };
return retsFields;
}
/**
* Multi-Family
*
* @return
*/
public static String[] getMultiFamilyFields() {
String[] retsFields = new String[] { MLS_NUMBER.val, LISTING_STATUS.val, STREET_NUMBER.val, STREET_NAME.val, STREET_TYPE.val,
TOWN.val, STATE.val, COUNTY.val, ZIP.val, LATITUDE.val, LONGITUDE.val, PHOTO_COUNT.val, TS_PHOTO_UPDATED.val,
FULL_BATH_ROOMS.val, BED_ROOMS.val, MODIFICATION_TIMESTAMP.val, GARAGE_SIZE.val, AGENT_MLSID.val, LISTAGENT_AGENTID.val,
LISTAGENT_OFFICEID.val, LISTPRICE.val, MAPCOORDINATES.val, LISTING_TYPE.val, DESCRIPTION.val, ROOMS_TOTAL.val,
SCHOOL_DISTRICT.val, SCHOOL_ELEMENTARY.val, SCHOOL_HIGH.val, SCHOOL_JUNIORHIG.val, SEASONAL.val, FINISHED_SQFT.val,
LOT_SIZE.val, TAXES.val, BATH_ROOMS.val, VIRTUALTOUR_URL.val, YEAR_BUILT.val };
return retsFields;
}
/**
* Get PruneListingFields
*
* @return
*/
public static String[] getPruneListingFields() {
List<ParagonRetsFields> fieldList = new ArrayList<ParagonRetsFields>();
fieldList.add(MLS_NUMBER);
fieldList.add(LISTING_STATUS);
return fieldsToArray(fieldList);
}
public static String[] getCloseListingFields() {
List<ParagonRetsFields> fieldList = new ArrayList<ParagonRetsFields>();
fieldList.add(MLS_NUMBER);
fieldList.add(PRICE_CLOSED);
fieldList.add(LISTING_STATUS);
fieldList.add(CLOSE_AGENTID);
fieldList.add(CLOSE_OFFICEID);
fieldList.add(TS_CLOSED);
fieldList.add(TS_LEASED);
fieldList.add(DOM);
return fieldsToArray(fieldList);
}
public static String[] getCloseCommercialLeaseListingFields() {
List<ParagonRetsFields> fieldList = new ArrayList<ParagonRetsFields>();
fieldList.add(MLS_NUMBER);
fieldList.add(PRICE_CLOSED);
fieldList.add(PRICE_LEASED);
fieldList.add(LISTING_STATUS);
fieldList.add(CLOSE_AGENTID);
fieldList.add(CLOSE_OFFICEID);
fieldList.add(TS_CLOSED);
fieldList.add(TS_LEASED);
fieldList.add(DOM);
return fieldsToArray(fieldList);
}
/**
* PHOTO FIELDS
*
* @return
*/
public static String[] getPhotoFields() {
List<ParagonRetsFields> fieldList = new ArrayList<ParagonRetsFields>();
fieldList.add(MLS_NUMBER);
fieldList.add(PHOTO_COUNT);
fieldList.add(TS_PHOTO_UPDATED);
return fieldsToArray(fieldList);
}
/**
* Media Fields
*
* @return
*/
public static String[] getMediaFields() {
List<ParagonRetsFields> fieldList = new ArrayList<ParagonRetsFields>();
fieldList.add(MED_MLSNUMBER);
fieldList.add(MED_IMG_URL);
fieldList.add(MED_IMG_SEQUENCE);
return fieldsToArray(fieldList);
}
/**
* Get Residential FeatureNames
*
* @return
*/
public static HashMap<String, String> getResidentialFeatureNames() {
HashMap<String, String> residentialFeatureMap = new LinkedHashMap<String, String>();
residentialFeatureMap.put(AMENITIES.val, "Amenities");
residentialFeatureMap.put(BASEMENT.val, "Basement");
residentialFeatureMap.put(CONSTRUCTION.val, "Construction");
residentialFeatureMap.put(DRIVEWAY.val, "Driveway");
residentialFeatureMap.put(ELECTRIC.val, "Electric");
residentialFeatureMap.put(EQUIPMENT_AND_APPLIANCES.val, "EquipmentAndAppliances");
residentialFeatureMap.put(EXTERIOR.val, "Exterior");
residentialFeatureMap.put(EXTERIOR_FEAT.val, "ExteriorFeat");
residentialFeatureMap.put(FINANCING.val, "Financing");
residentialFeatureMap.put(FOUNDATION.val, "Foundation");
residentialFeatureMap.put(GARAGE_AND_PARKING.val, "GarageAndParking");
residentialFeatureMap.put(INTERIOR_FEAT.val, "InteriorFeat");
residentialFeatureMap.put(HEATING.val, "HeatingAndCooling");
residentialFeatureMap.put(COOLING.val, "HeatingAndCooling");
residentialFeatureMap.put(HEAT_FUEL.val, "HeatFuel");
residentialFeatureMap.put(LOT_DESCRIPTION.val, "LotDescription");
residentialFeatureMap.put(NEGOTIABLE.val, "Negotiable");
residentialFeatureMap.put(OCCUPANT_RESTRICTIONS.val, "OccupantRestrictions");
residentialFeatureMap.put(ROADS.val, "Roads");
residentialFeatureMap.put(ROOF.val, "Roof");
residentialFeatureMap.put(SEWER.val, "Sewer");
residentialFeatureMap.put(WATER_HEATER.val, "WaterHeater");
// residentialFeatureMap.put(FEE.val, "FeeIncludes");
residentialFeatureMap.put(DISABILITY_FEATURES.val, "DisabilityFeatures");
residentialFeatureMap.put(STYLE.val, "Style");
residentialFeatureMap.put(WATER_CF.val, "WaterCF");
residentialFeatureMap.put(SUITABLE_USE.val, "SuitableUse");
residentialFeatureMap.put(FINISHED_ABOVE_GRADE.val, "FinishedAboveGrade");
residentialFeatureMap.put(SQFT_FINISHED_BELOW_GRADE.val, "SqFtFinishedBelowGrade");
return residentialFeatureMap;
}
public static HashMap<String, String> getLandFeatureNames() {
HashMap<String, String> featureMap = new LinkedHashMap<String, String>();
featureMap.put(AMENITIES.val, "Amenities");
featureMap.put(BASEMENT.val, "Basement");
featureMap.put(CONSTRUCTION.val, "Construction");
featureMap.put(DRIVEWAY.val, "Driveway");
featureMap.put(ELECTRIC.val, "Electric");
featureMap.put(EQUIPMENT_AND_APPLIANCES.val, "EquipmentAndAppliances");
featureMap.put(EXTERIOR.val, "Exterior");
featureMap.put(EXTERIOR_FEAT.val, "ExteriorFeat");
featureMap.put(FINANCING.val, "Financing");
featureMap.put(FOUNDATION.val, "Foundation");
featureMap.put(GARAGE_AND_PARKING.val, "GarageAndParking");
featureMap.put(INTERIOR_FEAT.val, "InteriorFeat");
featureMap.put(HEATING.val, "HeatingAndCooling");
featureMap.put(COOLING.val, "HeatingAndCooling");
featureMap.put(HEAT_FUEL.val, "HeatFuel");
featureMap.put(LOT_DESCRIPTION.val, "LotDescription");
featureMap.put(NEGOTIABLE.val, "Negotiable");
featureMap.put(OCCUPANT_RESTRICTIONS.val, "OccupantRestrictions");
featureMap.put(ROADS.val, "Roads");
featureMap.put(ROOF.val, "Roof");
featureMap.put(SEWER.val, "Sewer");
featureMap.put(WATER_HEATER.val, "WaterHeater");
// residentialFeatureMap.put(FEE.val, "FeeIncludes");
featureMap.put(DISABILITY_FEATURES.val, "DisabilityFeatures");
featureMap.put(STYLE.val, "Style");
featureMap.put(WATER_CF.val, "WaterCF");
featureMap.put(SUITABLE_USE.val, "SuitableUse");
featureMap.put(FINISHED_ABOVE_GRADE.val, "FinishedAboveGrade");
featureMap.put(SQFT_FINISHED_BELOW_GRADE.val, "SqFtFinishedBelowGrade");
return featureMap;
}
/**
* Get Land FeatureNames
*
* @return
*/
public static HashMap<String, String> getCommercialFeatureNames() {
HashMap<String, String> featureMap = new LinkedHashMap<String, String>();
featureMap.put(HEATING.val, "HeatingAndCooling");
featureMap.put(COOLING.val, "HeatingAndCooling");
featureMap.put(HEAT_FUEL.val, "Fuel");
featureMap.put(GARAGE_AND_PARKING.val, "Parking");
featureMap.put(CONSTRUCTION.val, "Construction");
featureMap.put(EXTERIOR.val, "Exterior");
featureMap.put(FINANCING.val, "Financing");
featureMap.put(BASEMENT.val, "Basement");
featureMap.put(FOUNDATION.val, "Foundation");
// featureMap.put(??, "Misc");
featureMap.put(ROOF.val, "Roof");
featureMap.put(INTERIOR_FEAT.val, "SaleIncludes");
// featureMap.put(??, "Transport");
featureMap.put(LISTING_TYPE.val, "Type");
// featureMap.put(LOT_DESCRIPTION.val, "Utilities");
featureMap.put(DISABILITY_FEATURES.val, "DisabilityFeatures");
featureMap.put(STYLE.val, "Style");
// featureMap.put(OCCUPANT_RESTRICTIONS.val, "Floors");
// featureMap.put(ROADS.val, "LeaseType");
// featureMap.put(SEWER.val, "MLSType");
featureMap.put(LISTING_TYPE.val, "PropertyType");
return featureMap;
}
/**
*
* @param fieldList
* @return
*/
private static String[] fieldsToArray(List<ParagonRetsFields> fieldList) {
List<String> valList = new ArrayList<String>();
for (int i = 0; i < fieldList.size(); i++) {
valList.add(fieldList.get(i).val);
}
return valList.toArray(new String[valList.size()]);
}
}

43
covid-19-service/src/main/java/com/ossez/covid19/service/field/RetsField.java Normal file
View File

@ -0,0 +1,43 @@
/**
*
*/
package com.ossez.covid19.service.field;
/**
* @author YuCheng Hu
*
*/
public class RetsField {
// AGENT INFO FIELDs
public static enum Agent {
AGENT_ID, EMAIL, FIRST_NAME, LAST_NAME, DATE_UPDATE
}
// OFFICE INFO FIELDs
public static enum Office {
OFFICE_ID, OFFICE_NAME, OFFICE_ABBREVIATION, DATE_UPDATE
}
// RE INFO FIELDs
public static enum Residential {
MLS_NUMBER,
LISTING_TYPE,
LISTING_STATUS,
LISTING_PRICE,
// ADDRESS
ADDRESS,
ADDRESS_NUMBER,
ADDRESS_NAME,
ADDRESS_TYPE,
CITY,
COUNTY,
STATE,
ZIP,
LATITUDE,
LONGITUDE,
// IMG
TS_MODIFICATION
}
}

54
covid-19-service/src/main/java/com/ossez/covid19/service/rets/MaineLocation.java Normal file
View File

@ -0,0 +1,54 @@
package com.ossez.covid19.service.rets;
/**
* Class to store a town,state,county association. This class is used as
* part of a larger collection to map a town id to the three aforementioned
* values.
*
* @author YUCHENG
*
*/
public class MaineLocation {
private String townName = null;
private String stateName = null;
private String countyName = null;
/**
*
*
* @param townName
* @param stateName
* @param countyName
*/
public MaineLocation(String townName, String stateName, String countyName) {
this.townName = townName;
this.stateName = stateName;
this.countyName = countyName;
}
/**
*
*
* @return
*/
public String getTownName() {
return townName;
}
/**
*
*
* @return
*/
public String getStateName() {
return stateName;
}
/**
*
* @return
*/
public String getCountyName() {
return countyName;
}
}

31
covid-19-service/src/main/java/com/ossez/covid19/service/rets/MlsPinLocation.java Normal file
View File

@ -0,0 +1,31 @@
package com.ossez.covid19.service.rets;
/**
* Class to store a town,state,county association. This class is used as part of a larger collection to map a town id to the three
* aforementioned values.
*
* @author YuCheng Hu
*/
public class MlsPinLocation {
private String townName = null;
private String stateName = null;
private String countyName = null;
public MlsPinLocation(String townName, String stateName, String countyName) {
this.townName = townName;
this.stateName = stateName;
this.countyName = countyName;
}
public String getTownName() {
return townName;
}
public String getStateName() {
return stateName;
}
public String getCountyName() {
return countyName;
}
}

69
covid-19-service/src/main/java/com/ossez/covid19/service/rets/RetsCriteria.java Normal file
View File

@ -0,0 +1,69 @@
package com.ossez.covid19.service.rets;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
/**
*
* @author YuCheng Hu
*/
public class RetsCriteria {
private String query = "";
private String path = "";
private List<String> fields = new ArrayList<String>();
public RetsCriteria(String query, String path, String[] fields) {
this.query = query;
this.path = path;
this.fields = Arrays.asList(fields);
}
public RetsCriteria(String query, String path, List<String> fields) {
this.query = query;
this.path = path;
this.fields = fields;
}
/**
* @return
*/
public String getQuery() {
return query;
}
/**
* @param query
*/
public void setQuery(String query) {
this.query = query;
}
/**
* @return
*/
public String getPath() {
return path;
}
/**
* @param path
*/
public void setPath(String path) {
this.path = path;
}
/**
* @return
*/
public List<String> getFields() {
return fields;
}
/**
* @param fields
*/
public void setFields(List<String> fields) {
this.fields = fields;
}
}

3
covid-19-service/src/main/resources/GuidedTours.properties Normal file
View File

@ -0,0 +1,3 @@
ftp.host = ftp.guidedtour.tv
ftp.user = pruverani
ftp.password = pruverani7788

2
covid-19-service/src/main/resources/META-INF/MANIFEST.MF Normal file
View File

@ -0,0 +1,2 @@
Manifest-Version: 1.0
Class-Path:

9
covid-19-service/src/main/resources/application.properties Normal file
View File

@ -0,0 +1,9 @@
# RABBIT (RabbitProperties)
spring.rabbitmq.dynamic=true
spring.rabbitmq.host=mq.ossez.com
spring.rabbitmq.password=Lucas#1120
spring.rabbitmq.username=admin
# BATCH
spring.batch.job.enabled=false

7
covid-19-service/src/main/resources/feeds.properties Normal file
View File

@ -0,0 +1,7 @@
#feeds=NH;com.verani.feeds.mls.NH,MA;com.ossez.reoc.mls.MA,MlsPinRets;com.ossez.reoc.mls.rets.MlsPinRetsFeed,NnerenRets;com.ossez.reoc.mls.rets.NnerenRetsFeed,NnerenRetsSold;com.ossez.reoc.mls.rets.NnerenRetsSoldFeed,MreisRets;com.ossez.reoc.mls.rets.MreisRetsFeed
NH=com.verani.feeds.mls.NH
MA=com.ossez.reoc.mls.MA
MlsPinRets=com.ossez.reoc.mls.rets.MlsPinRetsFeed
MreisRets=com.ossez.reoc.mls.rets.MreisRetsFeed
NnerenRetsSold=com.ossez.reoc.mls.rets.NnerenRetsSoldFeed
NHParagon=com.ossez.reoc.service.rets.NHParagon

7
covid-19-service/src/main/resources/hibernate.properties Normal file
View File

@ -0,0 +1,7 @@
hibernate.connection.driver_class = com.mysql.jdbc.Driver
hibernate.dialect = org.hibernate.dialect.MySQL5Dialect
hibernate.connection.url=jdbc\:mysql\://db-ossez/
hibernate.connection.username=ossez.com
hibernate.connection.password=pODi7um4Fu6E6eMab2maFIPI6ukaHi
hibernate.jdbc.batch_size = 20
hibernate.cache.use_second_level_cache = false

16
covid-19-service/src/main/resources/logback.xml Normal file
View File

@ -0,0 +1,16 @@
<?xml version="1.0" encoding="UTF-8" ?>
<configuration debug="true">
<appender name="STDOUT"
class="ch.qos.logback.core.ConsoleAppender">
<!-- encoders are assigned by default the type
ch.qos.logback.classic.encoder.PatternLayoutEncoder -->
<encoder>
<pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n</pattern>
</encoder>
</appender>
<root level="info">
<appender-ref ref="STDOUT"/>
</root>
</configuration>

4
covid-19-service/src/main/resources/rets.properties Normal file
View File

@ -0,0 +1,4 @@
# RETS SESSION INFO
rets_loginUrl=http://neren.rets.paragonrels.com/rets/fnisrets.aspx/NEREN/login
rets_username=username
rets_password=password

24
covid-19-service/src/test/java/com/ossez/reoc/mls/test/ChunksIntegrationTest.java Normal file
View File

@ -0,0 +1,24 @@
package com.ossez.reoc.mls.test;
import com.ossez.covid19.service.batch.jobs.BatchConfiguration;
import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.batch.core.ExitStatus;
import org.springframework.batch.core.JobExecution;
import org.springframework.batch.test.JobLauncherTestUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
@RunWith(SpringJUnit4ClassRunner.class)
@ContextConfiguration(classes = BatchConfiguration.class)
public class ChunksIntegrationTest {
@Autowired private JobLauncherTestUtils jobLauncherTestUtils;
@Test
public void givenChunksJob_WhenJobEnds_ThenStatusCompleted() throws Exception {
JobExecution jobExecution = jobLauncherTestUtils.launchJob();
Assert.assertEquals(ExitStatus.COMPLETED, jobExecution.getExitStatus());
}
}

111
covid-19-service/src/test/java/com/ossez/reoc/mls/test/RetsTestCase.java Normal file
View File

@ -0,0 +1,111 @@
package com.ossez.reoc.mls.test;
import junit.framework.TestCase;
import org.junit.BeforeClass;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.URL;
import java.util.Properties;
/**
* RetsTestCase for RETS tests
*
* @author YuCheng Hu
*/
public abstract class RetsTestCase extends TestCase {
public Properties props = new Properties();
public String retsLoginUrl;
public String retsUsername;
public String retsPassword;
public String retsUserAgent;
@BeforeClass
public void setUp() throws IOException {
ClassLoader loader = Thread.currentThread().getContextClassLoader();
props.load(loader.getResourceAsStream("rets.properties"));
retsLoginUrl = props.getProperty("rets_loginurl");
retsUsername = props.getProperty("rets_username");
retsPassword = props.getProperty("rets_password");
retsUserAgent = props.getProperty("rets_useragent");
}
/**
* Get Resource from file
*
* @param name
* @return
*/
protected static InputStream getResource(String name) {
ClassLoader cl = Thread.currentThread().getContextClassLoader();
return cl.getResourceAsStream(name);
}
/**
* @param urlStr
* @return
*/
protected static InputStream getResourceFromURL(String urlStr) {
try {
// in = new URL( "" ).openStream();
URL oracle = new URL("https://cdn.ossez.com/reso/rets-1x/login/login_response_valid_1.0.xml");
BufferedReader in = new BufferedReader(
new InputStreamReader(oracle.openStream()));
String inputLine;
while ((inputLine = in.readLine()) != null)
System.out.println(inputLine);
in.close();
} catch (IOException e) {
e.printStackTrace();
}
return null;
}
/**
*
* @param message
* @param expected
* @param actual
*/
public void assertEquals(String message, Object[] expected, Object[] actual) {
boolean success;
if (expected.length == actual.length) {
success = true;
for (int i = 0; i < expected.length; i++) {
success = true;
if (!expected[i].equals(actual[i])) {
success = false;
break;
}
}
} else {
success = false;
}
if (!success) {
fail(message + " expected: " + arrayToString(expected) + " but got: " + arrayToString((actual)));
}
}
private String arrayToString(Object[] array) {
StringBuffer sb = new StringBuffer();
sb.append("{");
for (int i = 0; i < array.length; i++) {
Object o = array[i];
if (i > 0) {
sb.append(", ");
}
sb.append("\"");
sb.append(o.toString());
sb.append("\"");
}
sb.append("}");
return sb.toString();
}
}

16
covid-19-service/src/test/resources/logback-test.xml Normal file
View File

@ -0,0 +1,16 @@
<?xml version="1.0" encoding="UTF-8" ?>
<configuration debug="true">
<appender name="STDOUT"
class="ch.qos.logback.core.ConsoleAppender">
<!-- encoders are assigned by default the type
ch.qos.logback.classic.encoder.PatternLayoutEncoder -->
<encoder>
<pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n</pattern>
</encoder>
</appender>
<root level="debug">
<appender-ref ref="STDOUT"/>
</root>
</configuration>