From 4e84a7cfe809192a2f4adb82768f8dda0ff1bf51 Mon Sep 17 00:00:00 2001 From: Robert van der Spek Date: Mon, 16 Jul 2018 12:45:37 +0200 Subject: [PATCH 1/4] Closes streams in finally clause whenever they are opened. --- .../sitemapgenerator/SitemapGenerator.java | 74 +++++++++++-------- .../SitemapIndexGenerator.java | 35 +++++---- .../sitemapgenerator/SitemapValidator.java | 58 +++++++++------ 3 files changed, 98 insertions(+), 69 deletions(-) diff --git a/src/main/java/com/redfin/sitemapgenerator/SitemapGenerator.java b/src/main/java/com/redfin/sitemapgenerator/SitemapGenerator.java index 6544c5f..5ee5a31 100644 --- a/src/main/java/com/redfin/sitemapgenerator/SitemapGenerator.java +++ b/src/main/java/com/redfin/sitemapgenerator/SitemapGenerator.java @@ -1,18 +1,17 @@ package com.redfin.sitemapgenerator; +import org.xml.sax.SAXException; + import java.io.File; import java.io.FileOutputStream; import java.io.IOException; import java.io.OutputStreamWriter; -import java.net.MalformedURLException; import java.net.URL; import java.nio.charset.Charset; import java.util.ArrayList; import java.util.List; import java.util.zip.GZIPOutputStream; -import org.xml.sax.SAXException; - abstract class SitemapGenerator> { /** 50000 URLs per sitemap maximum */ public static final int MAX_URLS_PER_SITEMAP = 50000; @@ -61,8 +60,9 @@ abstract class SitemapGenerator urls) { + public THIS addUrls(Iterable urls) throws IOException { + for (U url : urls) addUrl(url); + return getThis(); + } + + /** Add multiple URLs of the appropriate type to this sitemap, one at a time. + * If we have reached the maximum number of URLs, we'll throw an exception if {@link #allowMultipleSitemaps} is false, + * or write out one sitemap immediately. + * @param urls the URLs to add to this sitemap + * @return this + * @throws IOException when closing of streams has failed. + */ + public THIS addUrls(U... urls) throws IOException { for (U url : urls) addUrl(url); return getThis(); } @@ -95,19 +108,7 @@ abstract class SitemapGenerator write() { if (finished) throw new RuntimeException("Sitemap already printed; you must create a new generator to make more sitemaps"); if (!allowEmptySitemap && urls.isEmpty() && mapCount == 0) throw new RuntimeException("No URLs added, sitemap would be empty; you must add some URLs with addUrls"); - writeSiteMap(); + try { + writeSiteMap(); + } catch (IOException ex) { + throw new RuntimeException("Closing of streams has failed at some point.", ex); + } finished = true; return outFiles; } @@ -211,8 +215,9 @@ abstract class SitemapGenerator\n"); } out.write(""); - out.close(); } } diff --git a/src/main/java/com/redfin/sitemapgenerator/SitemapValidator.java b/src/main/java/com/redfin/sitemapgenerator/SitemapValidator.java index d6e9e73..e0cc5fc 100644 --- a/src/main/java/com/redfin/sitemapgenerator/SitemapValidator.java +++ b/src/main/java/com/redfin/sitemapgenerator/SitemapValidator.java @@ -1,9 +1,7 @@ package com.redfin.sitemapgenerator; -import java.io.File; -import java.io.FileReader; -import java.io.IOException; -import java.io.InputStream; +import org.xml.sax.InputSource; +import org.xml.sax.SAXException; import javax.xml.XMLConstants; import javax.xml.transform.sax.SAXSource; @@ -11,9 +9,10 @@ import javax.xml.transform.stream.StreamSource; import javax.xml.validation.Schema; import javax.xml.validation.SchemaFactory; import javax.xml.validation.Validator; - -import org.xml.sax.InputSource; -import org.xml.sax.SAXException; +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.io.InputStream; /** Validates sitemaps and sitemap indexes * @@ -41,44 +40,55 @@ public class SitemapValidator { SchemaFactory factory = SchemaFactory.newInstance(XMLConstants.W3C_XML_SCHEMA_NS_URI); try { - InputStream stream = SitemapValidator.class.getResourceAsStream("sitemap.xsd"); - if (stream == null) throw new RuntimeException("BUG Couldn't load sitemap.xsd"); - StreamSource source = new StreamSource(stream); - sitemapSchema = factory.newSchema(source); - stream.close(); - - stream = SitemapValidator.class.getResourceAsStream("siteindex.xsd"); - if (stream == null) throw new RuntimeException("BUG Couldn't load siteindex.xsd"); - source = new StreamSource(stream); - sitemapIndexSchema = factory.newSchema(source); - stream.close(); + sitemapSchema = lazyLoad(factory, "sitemap.xsd"); + sitemapIndexSchema = lazyLoad(factory, "siteindex.xsd"); } catch (Exception e) { throw new RuntimeException("BUG", e); } } + + private synchronized static Schema lazyLoad(SchemaFactory factory, String resource) throws IOException, SAXException { + InputStream stream = null; + + try { + stream = SitemapValidator.class.getResourceAsStream(resource); + if (stream == null) throw new RuntimeException("BUG Couldn't load " + resource); + StreamSource source = new StreamSource(stream); + return factory.newSchema(source); + } finally { + if(stream != null) { + stream.close(); + } + } + + } /** Validates an ordinary web sitemap file (NOT a Google-specific sitemap) */ - public static void validateWebSitemap(File sitemap) throws SAXException { + public static void validateWebSitemap(File sitemap) throws SAXException, IOException { lazyLoad(); validateXml(sitemap, sitemapSchema); } /** Validates a sitemap index file */ - public static void validateSitemapIndex(File sitemap) throws SAXException { + public static void validateSitemapIndex(File sitemap) throws SAXException, IOException { lazyLoad(); validateXml(sitemap, sitemapIndexSchema); } - private static void validateXml(File sitemap, Schema schema) throws SAXException { + private static void validateXml(File sitemap, Schema schema) throws SAXException, IOException { Validator validator = schema.newValidator(); + FileReader reader = null; try { - FileReader reader = new FileReader(sitemap); + reader = new FileReader(sitemap); SAXSource source = new SAXSource(new InputSource(reader)); validator.validate(source); - reader.close(); } catch (IOException e) { throw new RuntimeException(e); - } + } finally { + if(reader != null) { + reader.close(); + } + } } } From ab872270143fc73c2ec87fcbbc324407d15e29f5 Mon Sep 17 00:00:00 2001 From: Robert van der Spek Date: Mon, 16 Jul 2018 13:25:07 +0200 Subject: [PATCH 2/4] Adds flush() to ensure the file contents are available in the following procedure. --- .../java/com/redfin/sitemapgenerator/SitemapGenerator.java | 4 +++- .../com/redfin/sitemapgenerator/SitemapIndexGenerator.java | 2 ++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/redfin/sitemapgenerator/SitemapGenerator.java b/src/main/java/com/redfin/sitemapgenerator/SitemapGenerator.java index 5ee5a31..4043008 100644 --- a/src/main/java/com/redfin/sitemapgenerator/SitemapGenerator.java +++ b/src/main/java/com/redfin/sitemapgenerator/SitemapGenerator.java @@ -260,8 +260,10 @@ abstract class SitemapGenerator Date: Tue, 17 Jul 2018 10:20:39 +0200 Subject: [PATCH 3/4] Replaces spaced indents with tab indents. --- .../sitemapgenerator/SitemapGenerator.java | 38 +++++++++--------- .../SitemapIndexGenerator.java | 40 +++++++++---------- .../sitemapgenerator/SitemapValidator.java | 36 ++++++++--------- 3 files changed, 57 insertions(+), 57 deletions(-) diff --git a/src/main/java/com/redfin/sitemapgenerator/SitemapGenerator.java b/src/main/java/com/redfin/sitemapgenerator/SitemapGenerator.java index 4043008..1737873 100644 --- a/src/main/java/com/redfin/sitemapgenerator/SitemapGenerator.java +++ b/src/main/java/com/redfin/sitemapgenerator/SitemapGenerator.java @@ -60,7 +60,7 @@ abstract class SitemapGenerator urls) throws IOException { for (U url : urls) addUrl(url); @@ -95,7 +95,7 @@ abstract class SitemapGenerator Date: Tue, 17 Jul 2018 10:33:39 +0200 Subject: [PATCH 4/4] Wraps codeblocks that could throw IOException in try-catch. Catches IOException and throws RuntimeException instead. This is in line with code-practice. This way the upgrade won't break any code. --- .../sitemapgenerator/SitemapGenerator.java | 21 ++++++------- .../sitemapgenerator/SitemapValidator.java | 31 +++++++++++-------- 2 files changed, 28 insertions(+), 24 deletions(-) diff --git a/src/main/java/com/redfin/sitemapgenerator/SitemapGenerator.java b/src/main/java/com/redfin/sitemapgenerator/SitemapGenerator.java index 1737873..99b5242 100644 --- a/src/main/java/com/redfin/sitemapgenerator/SitemapGenerator.java +++ b/src/main/java/com/redfin/sitemapgenerator/SitemapGenerator.java @@ -60,16 +60,19 @@ abstract class SitemapGenerator urls) throws IOException { + public THIS addUrls(Iterable urls) { for (U url : urls) addUrl(url); return getThis(); } @@ -95,9 +97,8 @@ abstract class SitemapGenerator