From 0bc8a140b82150d2abee6f3116f9fc2c81ae11c9 Mon Sep 17 00:00:00 2001 From: YuCheng Hu Date: Mon, 13 Apr 2020 11:08:04 -0400 Subject: [PATCH] Adjust the main function for Spring Boot Application --- covid-19-service/build.gradle | 7 +- .../ossez/covid19/service/Application.java | 52 + .../java/com/ossez/covid19/service/Main.java | 262 --- .../java/com/ossez/covid19/service/NH.java | 1442 ----------------- ...Application.java => BatchApplication.java} | 4 +- 5 files changed, 58 insertions(+), 1709 deletions(-) create mode 100644 covid-19-service/src/main/java/com/ossez/covid19/service/Application.java delete mode 100644 covid-19-service/src/main/java/com/ossez/covid19/service/Main.java delete mode 100644 covid-19-service/src/main/java/com/ossez/covid19/service/NH.java rename covid-19-service/src/main/java/com/ossez/covid19/service/batch/{Application.java => BatchApplication.java} (92%) diff --git a/covid-19-service/build.gradle b/covid-19-service/build.gradle index 83461f6..7604d38 100644 --- a/covid-19-service/build.gradle +++ b/covid-19-service/build.gradle @@ -44,9 +44,10 @@ dependencies { implementation group: 'org.apache.solr', name: 'solr-solrj', version: '8.1.0' implementation group: 'org.jasypt', name: 'jasypt-hibernate4', version: '1.9.3' - // SPRING BATCH - implementation 'org.springframework.boot:spring-boot-starter-batch:2.2.6.RELEASE' - implementation 'org.springframework.boot:spring-boot-starter-amqp:2.2.6.RELEASE' + // SPRING BOOT AND BATCH + implementation 'org.springframework.boot:spring-boot-starter-web' + implementation 'org.springframework.boot:spring-boot-starter-batch' + implementation 'org.springframework.boot:spring-boot-starter-amqp' // CLOUD implementation 'com.amazonaws:aws-java-sdk-s3:1.11.699' diff --git a/covid-19-service/src/main/java/com/ossez/covid19/service/Application.java b/covid-19-service/src/main/java/com/ossez/covid19/service/Application.java new file mode 100644 index 0000000..0cb032f --- /dev/null +++ b/covid-19-service/src/main/java/com/ossez/covid19/service/Application.java @@ -0,0 +1,52 @@ +package com.ossez.covid19.service; + +import org.springframework.batch.core.Job; +import org.springframework.batch.core.launch.JobLauncher; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.builder.SpringApplicationBuilder; +import org.springframework.boot.web.servlet.support.SpringBootServletInitializer; +import org.springframework.context.annotation.ComponentScan; +import org.springframework.scheduling.annotation.EnableScheduling; +import org.springframework.web.context.request.RequestContextListener; + +import javax.servlet.ServletContext; +import javax.servlet.ServletException; + +//import org.apache.log4j.Level; + +@EnableScheduling +@ComponentScan("com.ossez.covid19") +@SpringBootApplication +public class Application extends SpringBootServletInitializer{ + + @Autowired + JobLauncher jobLauncher; + + @Autowired + Job cloudClean; + + @Override + protected SpringApplicationBuilder configure(SpringApplicationBuilder application) { + return application.sources(Application.class); + } + + @Override + public void onStartup(ServletContext servletContext) throws ServletException { + servletContext.addListener(new RequestContextListener()); + super.onStartup(servletContext); + } + + /** + * Main function for service application + * + * @param args + * @throws Exception + */ + public static void main(String... args) throws Exception { + SpringApplication.run(Application.class, args); + + } + +} diff --git a/covid-19-service/src/main/java/com/ossez/covid19/service/Main.java b/covid-19-service/src/main/java/com/ossez/covid19/service/Main.java deleted file mode 100644 index 5f275ce..0000000 --- a/covid-19-service/src/main/java/com/ossez/covid19/service/Main.java +++ /dev/null @@ -1,262 +0,0 @@ -package com.ossez.covid19.service; - -import com.ossez.covid19.common.Factory; -import com.ossez.covid19.common.mls.feeds.FeedStep; -import com.ossez.covid19.common.mls.feeds.ListingFeed; -import org.apache.commons.cli.CommandLine; -import org.apache.commons.cli.CommandLineParser; -import org.apache.commons.cli.GnuParser; -import org.apache.commons.cli.Options; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.File; -import java.io.FileNotFoundException; -import java.util.*; - -//import org.apache.log4j.Level; - - -public class Main { - private static final Logger logger = LoggerFactory.getLogger(Main.class); - - private static Options options = new Options(); - private static Properties properties = new Properties(); - - private static CommandLine cl = null; - private static Map feeds = new HashMap(); - private static boolean dryRun = false; - private static List feedsToRun = new ArrayList(); - private static List feedSteps = new ArrayList(); - private static int limit = 0; - private static boolean force = false; - private static List mlsNumbers = new ArrayList(); - - public static void main(String[] args) { - - // get the idx feed properties file - Main.parseProperties(); - - // load console options - Main.parseCommandLine(args); - - logger.debug("Starting feeds..."); - System.out.println("starting feeds..."); - - // execute the feeds - Main.executeFeeds(); - - Factory.close(); - } - - /** - * Executes the feeds specified in the feeds.properties file. - */ - private static void executeFeeds() { - // run through all feeds specified - for (ListingFeed feed : Main.feedsToRun) { - feed.setDryRun(Main.dryRun); - feed.setLimit(Main.limit); - feed.setMlsNumbers(Main.mlsNumbers); - - // load the properties file for the feed - Properties properties = new Properties(); - - logger.debug("Loading: " + feed.getName() + ".properties"); - -// try { -// properties.load(Main.class.getClassLoader().getResourceAsStream(feed.getName() + ".properties")); -// } catch (IOException ex) { -// logger.error("Could not load " + feed.getName() + ".properties"); -// continue; -// } - -// feed.setProperties(properties); - feed.setForce(Main.force); - // run the feed - try { - feed.run(Main.feedSteps, null); - } catch (Exception ex) { - logger.error("Error proccessing " + feed.getName() + " feed", ex); - } - } - } - - /** - * Parses the properties file to get a list of all feeds. - */ - private static void parseProperties() { - try { - // load the properties file - logger.debug("Parsing properties"); - Main.properties.load(Main.class.getClassLoader().getResourceAsStream("mls.properties")); - - // load the feeds - - logger.debug("Loading feeds"); - Set keys = Main.properties.keySet(); - - for (Object k : keys) { - - String name = (String) k; - String feedClass = Main.properties.getProperty(name); - - logger.trace("Loading feed class: " + k); - - // load the class - Class clazz = Class.forName(feedClass); - - // make sure the class implements ListingFeed - if (ListingFeed.class.isAssignableFrom(clazz)) { - logger.trace("Adding " + feedClass + " to list of feeds."); - // instantiate the class - Main.feeds.put(name, (ListingFeed) clazz.newInstance()); - } else { - logger.error("Class " + feedClass + " does not implement the com.verani.common.mls.idx.ListingFeed interface."); - } - - } - - // String[] feedClasses = Main.properties.getProperty("feeds").split(","); - // - // for (String feed : feedClasses) { - // logger.trace("Loading feed class: " + feed); - // - // String name = feed.substring(0, feed.indexOf(";")); - // String feedClass = feed.substring(feed.indexOf(";") + 1); - // - // // load the class - // Class clazz = Class.forName(feedClass); - // - // // make sure the class implements ListingFeed - // if (ListingFeed.class.isAssignableFrom(clazz)) { - // logger.trace("Adding " + feedClass + " to list of feeds."); - // // instantiate the class - // Main.feeds.put(name, (ListingFeed) clazz.newInstance()); - // } else { - // logger.error("Class " + feedClass + " does not implement the com.verani.common.mls.idx.ListingFeed interface."); - // } - // } - - } catch (Exception ex) { - ex.printStackTrace(); - logger.error("Could not parse feed properties", ex); - } - } - - /** - * Handles creation of console options. - */ - private static void parseCommandLine(String[] args) { - // configure command line options - Main.options.addOption("f", true, "List of which feeds to run."); - Main.options.addOption("s", true, "List of which steps in the feed should be run."); - Main.options.addOption("d", false, "Instructs the process to preform a \"dry\" run, meaning no data will actually be updated."); - Main.options.addOption("l", true, "Max limit of listings to process."); - Main.options.addOption("u", false, "Forces an update of any listings processed."); - Main.options.addOption("i", true, "One or more MLS IDs used when updating a single listings (UpdateListing step). Comma separated"); - Main.options.addOption("v", false, "Enables verbose logging."); - Main.options.addOption("m", true, "Name of the FLAT TXT file that will be used to load spacific MLSNumbers"); - - String flatFileName = ""; - - // parse command line options - CommandLineParser parser = new GnuParser(); - - try { - Main.cl = parser.parse(Main.options, args); - - // make sure feeds were specified - if (Main.cl.hasOption("f")) { - // get the feeds to run - for (String feed : Main.cl.getOptionValue("f").split(",")) { - logger.debug("Adding feed " + feed + " to the list of feeds to run."); - Main.feedsToRun.add(Main.feeds.get(feed)); - } - } else { - logger.error("No feeds have been specified."); - } - - // make sure feed steps were specified - if (Main.cl.hasOption("s")) { - // get the feed steps - for (String feedStep : Main.cl.getOptionValue("s").split(",")) { - logger.trace("Adding feed step: " + feedStep); - Main.feedSteps.add(FeedStep.valueOf(feedStep)); - } - } else { - logger.info( - "No feed steps have been specified. [Defualt run sequence: UpdateListings -> UpdateImages -> PruneListings -> UpdateSoldListings]"); - Main.feedSteps.add(FeedStep.valueOf("UpdateListings")); - Main.feedSteps.add(FeedStep.valueOf("UpdateImages")); - Main.feedSteps.add(FeedStep.valueOf("PruneListings")); - Main.feedSteps.add(FeedStep.valueOf("UpdateSoldListings")); - } - - // if we want to load from FILE, the look for the -f option and get name of file - if (Main.cl.hasOption("m")) - flatFileName = Main.cl.getOptionValue("m"); - - // add mls nos via input or flat file - if (Main.cl.hasOption("i") || Main.cl.hasOption("m")) { - - String[] mlsNos = null; - - // do logic to see if flat file is going to be used. - if (flatFileName != null && !flatFileName.isEmpty()) { - File file = new File(flatFileName); - if (file == null) - throw new FileNotFoundException(); - String mlsstring = null;//MlsUtility.getMlsNumbersStringFromFile(file); - mlsNos = mlsstring.split(","); - } else if (Main.cl.hasOption("i")) { - mlsNos = Main.cl.getOptionValue("i", "").split(","); - } // End if/else - - int maxLen = 60001; // Hardcoded for safety. Limit the maximum number of listings that can be updated from the CLI to - // prevent performance-expensive subqueries on the MLS provider's server. - - if (mlsNos.length > maxLen) - throw new Exception("Maximum length of " + maxLen - + " numbers in the single update list exceeded. Do not pass more than this number of listing numbers. This is a safety measure."); - - Main.mlsNumbers = new ArrayList(); - for (String mlsNo : mlsNos) { - try { - Main.mlsNumbers.add(Integer.parseInt(mlsNo.trim())); - } catch (Exception e) { - // Throw exception, but continue. - logger.error("An error ocurred parsing list of single MLS numbers to update", e); - } - } - - logger.trace("Value of mls numbers: " + Main.mlsNumbers); - } - - // if the verbose option is specified, change logging - if (Main.cl.hasOption("v")) { - // org.apache.log4j.Logger.getLogger(NH.class).setLevel(Level.TRACE); - // org.apache.log4j.Logger.getLogger(MA.class).setLevel(Level.TRACE); - // org.apache.log4j.Logger.getLogger(MreisRetsFeed.class).setLevel(Level.TRACE); - // org.apache.log4j.Logger.getLogger(Main.class).setLevel(Level.TRACE); - // org.apache.log4j.Logger.getLogger(NnerenRetsFeed.class).setLevel(Level.TRACE); - // org.apache.log4j.Logger.getLogger(MlsPinRetsFeed.class).setLevel(Level.TRACE); - // org.apache.log4j.Logger.getLogger(NnerenRetsSoldFeed.class).setLevel(Level.TRACE); - } - - // get the dry run option - Main.dryRun = Main.cl.hasOption("d"); - logger.trace("Value of dryRun: " + dryRun); - - // get the limit option -// Main.limit = Utility.parseInt(Main.cl.getOptionValue("l", "0")); - logger.trace("Value of limit: " + Main.limit); - - // get the force option - Main.force = Main.cl.hasOption("u"); - logger.trace("Value of force: " + Main.force); - } catch (Exception ex) { - logger.error("An error ocurred parsing command line arguments", ex); - } - } -} diff --git a/covid-19-service/src/main/java/com/ossez/covid19/service/NH.java b/covid-19-service/src/main/java/com/ossez/covid19/service/NH.java deleted file mode 100644 index a99bf20..0000000 --- a/covid-19-service/src/main/java/com/ossez/covid19/service/NH.java +++ /dev/null @@ -1,1442 +0,0 @@ -package com.ossez.covid19.service; - -import com.ossez.covid19.common.mls.feeds.FeedException; -import com.ossez.covid19.common.mls.feeds.FeedStep; -import com.ossez.covid19.common.mls.feeds.ListingFeed; - -import java.util.List; -import java.util.Properties; - -public class NH implements ListingFeed { - @Override - public void run(List list, String s) throws FeedException { - - } - - @Override - public void setProperties(Properties properties) { - - } - - @Override - public void setLimit(int i) { - - } - - @Override - public void setDryRun(boolean b) { - - } - - @Override - public void setForce(boolean b) { - - } - - @Override - public void setMlsNumbers(List list) { - - } - - @Override - public String getName() { - return null; - } - - @Override - public String getUpdateStatus(String s) { - return null; - } - - @Override - public void addProgressMessage(String s, String s1) { - - } - - -// private Logger log = LoggerFactory.getLogger(NH.class); -// private boolean dryRun = false; -// private int limit = 0; -// private Properties properties = null; -// private Connection nnerenConnection = null; -// private boolean force = false; -// private static Pattern properCasePattern = Pattern.compile("(\\w)(\\w+)([\\s\\/\\+]+)?"); -// private Map> featureCache = new HashMap>(); -// private Map townCache = new HashMap(); -// private Map styleCache = new HashMap(); -// private Map agentCache = new HashMap(); -// private Map countyCache = new HashMap(); -// private int newListings = 0; -// private int updatedListings = 0; -// private int newPhotos = 0; -// private int updatedPhotos = 0; -// private Feed feed = null; -// private Map statusMessages = new HashMap(); -// private List mlsNumbers = new ArrayList(); -// -// /* (non-Javadoc) -// * @see com.verani.common.mls.idx.ListingFeed#setDryRun(boolean) -// */ -// @Override -// public void setDryRun(boolean dryRun) { -// this.dryRun = dryRun; -// } -// -// /* (non-Javadoc) -// * @see com.verani.common.mls.idx.ListingFeed#setLimit(int) -// */ -// @Override -// public void setLimit(int limit) { -// this.limit = limit; -// } -// -// /* (non-Javadoc) -// * @see com.verani.common.mls.idx.ListingFeed#setProperties(java.util.Properties) -// */ -// @Override -// public void setProperties(Properties properties) { -// this.properties = properties; -// } -// -// /* (non-Javadoc) -// * @see com.verani.common.mls.idx.ListingFeed#setForce(boolean) -// */ -// @Override -// public void setForce(boolean force) { -// this.force = force; -// } -// -// /* (non-Javadoc) -// * @see com.verani.common.mls.idx.ListingFeed#getName() -// */ -// @Override -// public String getName() { -// return "NH"; -// } -// -// /* (non-Javadoc) -// * @see com.verani.common.mls.idx.ListingFeed#setMlsNumber(int) -// */ -// @Override -// public void setMlsNumbers(List mlsNumbers) { -// this.mlsNumbers = mlsNumbers; -// } -// -// /** -// * Gets the set list of mls numbers that this feed will update. -// * @return -// */ -// public List getMlsNumbers() { -// return mlsNumbers; -// } -// -// /* (non-Javadoc) -// * @see com.verani.common.mls.idx.ListingFeed#run(java.util.List) -// */ -// @Override -// public void run(List steps, String progressToken) throws FeedException { -// this.log.info("Starting NH feed"); -// this.log.debug("Initializing hibernate"); -// -// // check to see if a feed exists in the db -// Factory.beginTransaction(); -// -// this.feed = FeedFactory.get("NNEREN"); -// -// if (this.feed == null) { -// this.feed = new Feed("NNEREN", "New hampshire nneren feed"); -// FeedFactory.save(this.feed); -// } -// -// if (this.dryRun) -// Factory.rollbackTransaction(); -// else -// Factory.commitTransaction(); -// -// try { -// this.log.debug("Connecting to nneren"); -// // connect to the database -// Class.forName("com.microsoft.sqlserver.jdbc.SQLServerDriver"); -// this.nnerenConnection = DriverManager.getConnection(this.properties.getProperty("nnerenConnectionUrl")); -// } catch (Exception ex) { -// this.log.error("Error connecting to nneren database", ex); -// } -// -// // execute the specified steps -// for (FeedStep step : steps) { -// switch (step) { -// case UpdateListings: -// this.updateListings(); -// break; -// case UpdateImages: -// this.updateImages(progressToken); -// break; -// case PruneListings: -// this.pruneListings(); -// break; -// case UpdateFeatures: -// this.updateFeatures(); -// break; -// case UpdateTowns: -// this.updateTowns(); -// break; -// case UpdateListing: -// this.updateListing(this.mlsNumbers); -// break; -// case UpdateOfficeMlsNumbers: -// this.updateOfficeMlsNumbers(); -// break; -// } -// } -// -// try { -// this.log.debug("Disconnecting from nneren"); -// // disconnect -// this.nnerenConnection.close(); -// this.log.debug("Disconnected from nneren"); -// } catch (SQLException ex) { -// this.log.error("Error disconnecting from nneren database", ex); -// } -// } -// -// private void updateOfficeMlsNumbers() { -// this.updateOfficeMlsNumbers("RES"); -// this.updateOfficeMlsNumbers("LAND"); -// this.updateOfficeMlsNumbers("COM"); -// this.updateOfficeMlsNumbers("MLT"); -// } -// -// private void updateOfficeMlsNumbers(String type) { -// this.log.info("Updating office mls numbers for " + type); -// try { -// Statement statement = this.nnerenConnection.createStatement(); -// ResultSet result = statement.executeQuery("select [MLS Number], [List Office] from IDX3_" + type + " (nolock);"); -// while (result.next()) { -// Factory.beginTransaction(); -// -// Listing listing = ListingFactory.get(result.getInt("MLS Number")); -// if (listing != null) { -// if (listing.getAgent() != null && listing.getStatus() != ListingStatus.Closed && Utility.isEmpty(listing.getOfficeMlsId())) { -// this.log.debug("Processing listing " + result.getInt("MLS Number")); -// listing.setOfficeMlsId(result.getString("List Office")); -// ListingFactory.save(listing); -// } -// } -// -// Factory.commitTransaction(); -// } -// -// result.close(); -// statement.close(); -// } catch (SQLException ex) { -// this.log.error("Error updating office mls numbers.", ex); -// } -// } -// -// private void updateListing(List mlsNumbers) { -// try { -// Factory.beginTransaction(); -// -// // load the feature cache -// this.initFeatureCache(); -// this.initTownCache(); -// this.initStyleCache(); -// this.initAgentCache(); -// -// if (this.dryRun) -// Factory.rollbackTransaction(); -// else -// Factory.commitTransaction(); -// -//// File file = new File("D:\\mlsNumbers.txt"); -//// -//// mlsNumbers = MlsUtility.getMlsNumbersFromFile(file); -// String mlsNos = StringUtils.join(mlsNumbers,","); -// -// -// this.log.info("Updating Listing(s) #: " + mlsNos); -// -// Statement statement = this.nnerenConnection.createStatement(); -// ResultSet result = statement.executeQuery("select * from IDX3_RES (nolock) where [MLS" + -// " Number] IN (" + mlsNos + ")"); -// while (result.next()) { -// this.updateListing(result, "RES"); -// } -// -// result.close(); -// statement.close(); -// -// statement = this.nnerenConnection.createStatement(); -// result = statement.executeQuery("select * from IDX3_MLT (nolock) where [Mls Number] IN (" + mlsNos + ")"); -// while (result.next()) { -// this.updateListing(result, "MLT"); -// } -// -// result.close(); -// statement.close(); -// -// statement = this.nnerenConnection.createStatement(); -// result = statement.executeQuery("select * from IDX3_COM (nolock) where [Mls Number] IN (" + mlsNos + ")"); -// while (result.next()) { -// this.updateListing(result, "COM"); -// } -// -// result.close(); -// statement.close(); -// -// statement = this.nnerenConnection.createStatement(); -// result = statement.executeQuery("select * from IDX3_LAND (nolock) where [Mls Number] IN (" + mlsNos + ")"); -// while (result.next()) { -// this.updateListing(result, "LAND"); -// } -// -// result.close(); -// statement.close(); -// } catch (Exception ex) { -// this.log.error("Error processing Listing(s) #: " + StringUtils.join(mlsNumbers,",") + " listings", ex); -// Factory.rollbackTransaction(); -// } -// } -// -// private void updateTowns() { -// this.log.info("Updating towns"); -// Factory.beginTransaction(); -// -// try { -// Statement statement = this.nnerenConnection.createStatement(); -// ResultSet result = statement.executeQuery( -// "select distinct town, county, state from IDX3_RES (nolock) union " + -// "select distinct town, county, state from IDX3_MLT (nolock) union " + -// "select distinct town, county, state from IDX3_LAND (nolock) union " + -// "select distinct town, county, state from IDX3_COM (nolock);" -// ); -// -// // loop through each town -// while (result.next()) { -// String name = result.getString("town").trim(); -// String state = result.getString("state").trim(); -// String countyName = result.getString("county").trim(); -// countyName = countyName.substring(0, countyName.length() - 2); -// -// // get the county -// County county = CountyFactory.get(countyName, state); -// -// if (county == null) { -// county = new County(countyName, state); -// CountyFactory.save(county); -// this.log.trace("Adding county: " + countyName + ", " + state); -// } -// -// // get the town from the database -// Town town = TownFactory.get(name, state); -// -// if (town == null) -// town = new Town(name, state, county); -// else -// town.setCounty(county); -// -// this.log.trace("Updating town: " + name + ", " + state + " with county " + countyName); -// -// TownFactory.save(town); -// } -// -// result.close(); -// statement.close(); -// -// if (this.dryRun) -// Factory.rollbackTransaction(); -// else -// Factory.commitTransaction(); -// } catch (Throwable ex) { -// this.log.error("Error processing towns", ex); -// -// if (Factory.getSession().getTransaction().isActive()) -// Factory.rollbackTransaction(); -// } -// } -// -// private void pruneListings() { -// List mlsNumbers = new ArrayList(); -// -// try { -// // Sanity check must pass to continue. -// if(checkKeeplistSanityLevel() == false) -// return; -// -// // normal run. -// this.log.info("Getting listings from NNEREN."); -// Statement statement = this.nnerenConnection.createStatement(); -// ResultSet result = statement.executeQuery("select LM_MST_MLS_NO from IDX3_KEEPLIST (nolock);"); -// -// while (result.next()) { -// mlsNumbers.add(result.getInt("LM_MST_MLS_NO")); -// } -// -// result.close(); -// statement.close(); -// -// this.log.info("Getting listings from our database."); -// -// // now update any withdrawn listings -// Factory.beginTransaction(); -// List currentMlsNumbers = ListingFactory.getMlsNumbersByFeed(this.feed, 0); -// Factory.commitTransaction(); -// -// int count = 0; -// -// this.log.info("Updating withdrawn listings."); -// -// for (int mlsNumber : currentMlsNumbers) { -// if (!mlsNumbers.contains(mlsNumber)) { -// try { -// Factory.beginTransaction(); -// -// Listing listing = ListingFactory.get(mlsNumber); -// listing.setStatus(ListingStatus.Withdrawn); -// listing.setDelisted(new Date()); -// listing.setUpdated(new Date()); -// -// ListingFactory.save(listing); -// -// Factory.commitTransaction(); -// } catch (Exception ex) { -// Factory.rollbackTransaction(); -// this.log.error("Could not update listing " + mlsNumber, ex); -// } -// -// count++; -// } -// } -// -// this.log.info("Marked " + count + " listings as withdrawn."); -// this.log.info("Updating active listings."); -// -// count = 0; -// -// // now make sure everything in the keep list is also active on our site -// for (int mlsNumber : mlsNumbers) { -// if (!currentMlsNumbers.contains(mlsNumber)) { -// try { -// Factory.beginTransaction(); -// -// Listing listing = ListingFactory.get(mlsNumber); -// -// if (listing != null) { -// listing.setStatus(ListingStatus.Active); -// listing.setUpdated(new Date()); -// -// ListingFactory.save(listing); -// -// count++; -// } -// -// if(this.dryRun) -// Factory.rollbackTransaction(); -// else -// Factory.commitTransaction(); -// } catch (Exception ex) { -// Factory.rollbackTransaction(); -// this.log.error("Could not update listing " + mlsNumber, ex); -// } -// } -// } -// -// this.log.info("Marked " + count + " listings as active."); -// } catch (SQLException ex) { -// this.log.error("Error pruning listings.", ex); -// -// if (Factory.getSession().getTransaction().isActive()) -// Factory.rollbackTransaction(); -// } -// } -// -// private void updateImages(String progressToken) { -// this.log.info("Updating listing images."); -// Date startTime = new Date(); -// -// if (this.mlsNumbers == null || this.mlsNumbers.size() == 0) { -// Factory.beginTransaction(); -// // get all active listings for this feed -// List listings = ListingFactory.getMlsNumbersByFeed(this.feed, this.limit); -// Factory.commitTransaction(); -// -// this.updateImages(listings, progressToken); -// } else { -// this.updateImages(this.mlsNumbers, progressToken); -// } -// -// String diffTime = Integer.toString(DateUtility.diffMinutes(new Date(), startTime)); -// -// this.log.info("Added " + this.newPhotos); -// this.log.info("Updated " + this.updatedPhotos); -// this.log.info("Took " + diffTime + " minutes"); -// this.addProgressMessage(progressToken, "Added " + this.newPhotos + "\n"); -// this.addProgressMessage(progressToken, "Updated " + this.updatedPhotos + "\n"); -// this.addProgressMessage(progressToken, "Took " + diffTime + " minutes\n"); -// } -// -// /** -// * This goes through the given list and updates images for each one. -// * -// * In the interest of not making a query that is too long, this queries the NNEREN server on a listing-by-listing basis instead of getting the info all at once. -// * @param mlsNumbers -// * @param progressToken -// */ -// private void updateImages(List mlsNumbers, String progressToken) { -// for(Integer l: mlsNumbers) -// { -// Factory.beginTransaction(); -// try { -// Date updated = null; -// int photoCount = 0; -// Listing listing = ListingFactory.get(l.intValue()); -// -// if(listing == null){ -// this.log.error("MLS number not found in our database: " + l.toString()); -// continue; -// } -// -// // if the listing has custom photos, do not update the photos -// if (listing.isCustomPhotos()){ -// this.log.info("The listing #" + l.toString() + " is flagged as having custom photos in our database, skipping update."); -// this.addProgressMessage(progressToken, "The listing #" + l.toString() + " is flagged as having custom photos in our database, skipping update."); -// continue; -// } -// -// this.log.trace("Checking photos for listing " + l.toString()); -// // get the updated date from nneren -// Statement statement = this.nnerenConnection.createStatement(); -// ResultSet result = statement.executeQuery("select p_mod_date_time, Photos from IDX3_" + this.translateType(listing.getType()) + " (nolock) where [MLS Number] = " + l.toString() + ";"); -// if (result.next()) { -// updated = result.getTimestamp("p_mod_date_time"); -// photoCount = result.getInt("Photos"); -// } -// result.close(); -// statement.close(); -// -// // check if the images have been updated -// if (listing.getPhotoUpdated() == null -// || (updated != null && listing.getPhotoUpdated().before(updated)) -// || (photoCount > 0 && listing.getPhotoCount() < photoCount) || this.force) { -// -// this.log.trace("Updating photos for listing " + listing.getMlsNumber()); -// this.addProgressMessage(progressToken, "Updating photos for listing " + listing.getMlsNumber() + ". " + photoCount + " photos in the MLS.\n"); -// -// // get the photos now -// statement = this.nnerenConnection.createStatement(); -// result = statement.executeQuery("select * from IDX_IMAGES_FULL (nolock) where mls_no = " + l.toString() + ";"); -// -// // we have to maintain a separate index because innovia sometimes throws in a null in the middle of the list of photos -// int photoIndex = 0; -// -// if (result.next()) { -// // loop through each image -// for (int i = 1; i <= 24; i++) { -// byte [] imageData = result.getBytes("image" + i); -// -// if (imageData != null) { -// // get the current image -// ListingPhoto photo = listing.getPhoto(photoIndex); -// -// // if the photo was not found create it -// if (photo == null) { -// this.log.trace("Adding photo " + photoIndex); -// this.addProgressMessage(progressToken, "Adding photo " + photoIndex + "\n"); -// -// photo = new ListingPhoto(); -// photo.setImageNumber(photoIndex); -// listing.addPhoto(photo); -// this.newPhotos++; -// } else { -// this.log.trace("Updating photo " + photoIndex); -// this.addProgressMessage(progressToken, "Updating photo " + photoIndex + "\n"); -// this.updatedPhotos++; -// } -// -// // update the photo -// photo.setImage(imageData); -// photo.setUpdated(new Date()); -// photo.setCaption(""); -// -// photoIndex++; -// } -// } -// } -// -// result.close(); -// statement.close(); -// // update the photo updated column -// listing.setPhotoUpdated(new Date()); -// // mark the listing as updated so the indexer will pick up the potential change to the photo count -// listing.setUpdated(new Date()); -// // update the listing which will save all of the photos -// ListingFactory.save(listing); -// } -// -// if (this.dryRun) -// Factory.rollbackTransaction(); -// else -// Factory.commitTransaction(); -// } catch (Exception ex) { -// Factory.rollbackTransaction(); -// this.log.error("Error processing images for listing: " + l.toString(), ex); -// this.addProgressMessage(progressToken, "Error processing images for listing: " + l.toString() + "\n"); -// } -// } -// } -// -// /** -// * Updates all listings. -// * @throws FeedException -// */ -// private void updateListings() throws FeedException { -// this.log.info("Updating listing data"); -// -// Factory.beginTransaction(); -// -// // load the feature cache -// this.initFeatureCache(); -// this.initTownCache(); -// this.initStyleCache(); -// this.initAgentCache(); -// -// if (this.dryRun) -// Factory.rollbackTransaction(); -// else -// Factory.commitTransaction(); -// -// // process all listings -// this.updateListings("RES"); -// this.updateListings("MLT"); -// this.updateListings("LAND"); -// this.updateListings("COM"); -// -// this.log.info("Updated " + this.updatedListings + " listings\nAdded " + this.newListings + " listings"); -// } -// -// private void initAgentCache() { -// this.log.debug("Initializing agent cache"); -// List mlsNumbers = AgentFactory.getAllMlsNumbers(); -// for (AgentMlsNumber mlsNumber : mlsNumbers) { -// this.agentCache.put(mlsNumber.getMlsNumber() + "_" + mlsNumber.getState(), mlsNumber.getAgent()); -// } -// } -// -// private void initTownCache() { -// this.log.debug("Initializing town cache"); -// List towns = TownFactory.getAll(); -// -// for (Town town : towns) { -// this.townCache.put(town.getName() + "_" + town.getState(), town); -// } -// -// // get counties -// List counties = CountyFactory.getAll(); -// -// for (County county : counties) { -// this.countyCache.put(county.getName() + "_" + county.getState(), county); -// } -// } -// -// private void initStyleCache() { -// this.log.debug("Initializing style cache"); -// List