Closes streams in finally clause whenever they are opened.

This commit is contained in:
Robert van der Spek 2018-07-16 12:45:37 +02:00
parent 168ac4f200
commit 4e84a7cfe8
3 changed files with 98 additions and 69 deletions

View File

@ -1,18 +1,17 @@
package com.redfin.sitemapgenerator; package com.redfin.sitemapgenerator;
import org.xml.sax.SAXException;
import java.io.File; import java.io.File;
import java.io.FileOutputStream; import java.io.FileOutputStream;
import java.io.IOException; import java.io.IOException;
import java.io.OutputStreamWriter; import java.io.OutputStreamWriter;
import java.net.MalformedURLException;
import java.net.URL; import java.net.URL;
import java.nio.charset.Charset; import java.nio.charset.Charset;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
import java.util.zip.GZIPOutputStream; import java.util.zip.GZIPOutputStream;
import org.xml.sax.SAXException;
abstract class SitemapGenerator<U extends ISitemapUrl, THIS extends SitemapGenerator<U,THIS>> { abstract class SitemapGenerator<U extends ISitemapUrl, THIS extends SitemapGenerator<U,THIS>> {
/** 50000 URLs per sitemap maximum */ /** 50000 URLs per sitemap maximum */
public static final int MAX_URLS_PER_SITEMAP = 50000; public static final int MAX_URLS_PER_SITEMAP = 50000;
@ -61,8 +60,9 @@ abstract class SitemapGenerator<U extends ISitemapUrl, THIS extends SitemapGener
* or else write out one sitemap immediately. * or else write out one sitemap immediately.
* @param url the URL to add to this sitemap * @param url the URL to add to this sitemap
* @return this * @return this
* @throws IOException when closing of streams has failed
*/ */
public THIS addUrl(U url) { public THIS addUrl(U url) throws IOException {
if (finished) throw new RuntimeException("Sitemap already printed; you must create a new generator to make more sitemaps"); if (finished) throw new RuntimeException("Sitemap already printed; you must create a new generator to make more sitemaps");
UrlUtils.checkUrl(url.getUrl(), baseUrl); UrlUtils.checkUrl(url.getUrl(), baseUrl);
if (urls.size() == maxUrls) { if (urls.size() == maxUrls) {
@ -83,8 +83,21 @@ abstract class SitemapGenerator<U extends ISitemapUrl, THIS extends SitemapGener
* or write out one sitemap immediately. * or write out one sitemap immediately.
* @param urls the URLs to add to this sitemap * @param urls the URLs to add to this sitemap
* @return this * @return this
* @throws IOException when closing of streams has failed.
*/ */
public THIS addUrls(Iterable<? extends U> urls) { public THIS addUrls(Iterable<? extends U> urls) throws IOException {
for (U url : urls) addUrl(url);
return getThis();
}
/** Add multiple URLs of the appropriate type to this sitemap, one at a time.
* If we have reached the maximum number of URLs, we'll throw an exception if {@link #allowMultipleSitemaps} is false,
* or write out one sitemap immediately.
* @param urls the URLs to add to this sitemap
* @return this
* @throws IOException when closing of streams has failed.
*/
public THIS addUrls(U... urls) throws IOException {
for (U url : urls) addUrl(url); for (U url : urls) addUrl(url);
return getThis(); return getThis();
} }
@ -95,19 +108,7 @@ abstract class SitemapGenerator<U extends ISitemapUrl, THIS extends SitemapGener
* @param urls the URLs to add to this sitemap * @param urls the URLs to add to this sitemap
* @return this * @return this
*/ */
public THIS addUrls(U... urls) { public THIS addUrls(String... urls) {
for (U url : urls) addUrl(url);
return getThis();
}
/** Add multiple URLs of the appropriate type to this sitemap, one at a time.
* If we have reached the maximum number of URLs, we'll throw an exception if {@link #allowMultipleSitemaps} is false,
* or write out one sitemap immediately.
* @param urls the URLs to add to this sitemap
* @return this
* @throws MalformedURLException
*/
public THIS addUrls(String... urls) throws MalformedURLException {
for (String url : urls) addUrl(url); for (String url : urls) addUrl(url);
return getThis(); return getThis();
} }
@ -117,16 +118,15 @@ abstract class SitemapGenerator<U extends ISitemapUrl, THIS extends SitemapGener
* or else write out one sitemap immediately. * or else write out one sitemap immediately.
* @param url the URL to add to this sitemap * @param url the URL to add to this sitemap
* @return this * @return this
* @throws MalformedURLException
*/ */
public THIS addUrl(String url) throws MalformedURLException { public THIS addUrl(String url) {
U sitemapUrl; U sitemapUrl;
try { try {
sitemapUrl = renderer.getUrlClass().getConstructor(String.class).newInstance(url); sitemapUrl = renderer.getUrlClass().getConstructor(String.class).newInstance(url);
} catch (Exception e) { return addUrl(sitemapUrl);
} catch (Exception e) {
throw new RuntimeException(e); throw new RuntimeException(e);
} }
return addUrl(sitemapUrl);
} }
/** Add multiple URLs of the appropriate type to this sitemap, one at a time. /** Add multiple URLs of the appropriate type to this sitemap, one at a time.
@ -150,10 +150,10 @@ abstract class SitemapGenerator<U extends ISitemapUrl, THIS extends SitemapGener
U sitemapUrl; U sitemapUrl;
try { try {
sitemapUrl = renderer.getUrlClass().getConstructor(URL.class).newInstance(url); sitemapUrl = renderer.getUrlClass().getConstructor(URL.class).newInstance(url);
} catch (Exception e) { return addUrl(sitemapUrl);
} catch (Exception e) {
throw new RuntimeException(e); throw new RuntimeException(e);
} }
return addUrl(sitemapUrl);
} }
@SuppressWarnings("unchecked") @SuppressWarnings("unchecked")
@ -168,7 +168,11 @@ abstract class SitemapGenerator<U extends ISitemapUrl, THIS extends SitemapGener
public List<File> write() { public List<File> write() {
if (finished) throw new RuntimeException("Sitemap already printed; you must create a new generator to make more sitemaps"); if (finished) throw new RuntimeException("Sitemap already printed; you must create a new generator to make more sitemaps");
if (!allowEmptySitemap && urls.isEmpty() && mapCount == 0) throw new RuntimeException("No URLs added, sitemap would be empty; you must add some URLs with addUrls"); if (!allowEmptySitemap && urls.isEmpty() && mapCount == 0) throw new RuntimeException("No URLs added, sitemap would be empty; you must add some URLs with addUrls");
writeSiteMap(); try {
writeSiteMap();
} catch (IOException ex) {
throw new RuntimeException("Closing of streams has failed at some point.", ex);
}
finished = true; finished = true;
return outFiles; return outFiles;
} }
@ -211,8 +215,9 @@ abstract class SitemapGenerator<U extends ISitemapUrl, THIS extends SitemapGener
/** /**
* After you've called {@link #write()}, call this to generate a sitemap index of all sitemaps you generated. * After you've called {@link #write()}, call this to generate a sitemap index of all sitemaps you generated.
* The sitemap index is written to {baseDir}/sitemap_index.xml * The sitemap index is written to {baseDir}/sitemap_index.xml
* @throws IOException when closing of streams has failed
*/ */
public File writeSitemapsWithIndex() { public File writeSitemapsWithIndex() throws IOException {
if (!finished) throw new RuntimeException("Sitemaps not generated yet; call write() first"); if (!finished) throw new RuntimeException("Sitemaps not generated yet; call write() first");
File outFile = new File(baseDir, "sitemap_index.xml"); File outFile = new File(baseDir, "sitemap_index.xml");
return writeSitemapsWithIndex(outFile); return writeSitemapsWithIndex(outFile);
@ -222,8 +227,9 @@ abstract class SitemapGenerator<U extends ISitemapUrl, THIS extends SitemapGener
* After you've called {@link #write()}, call this to generate a sitemap index of all sitemaps you generated. * After you've called {@link #write()}, call this to generate a sitemap index of all sitemaps you generated.
* *
* @param outFile the destination file of the sitemap index. * @param outFile the destination file of the sitemap index.
* @throws IOException when closing of streams has failed
*/ */
public File writeSitemapsWithIndex(File outFile) { public File writeSitemapsWithIndex(File outFile) throws IOException {
if (!finished) throw new RuntimeException("Sitemaps not generated yet; call write() first"); if (!finished) throw new RuntimeException("Sitemaps not generated yet; call write() first");
SitemapIndexGenerator sig; SitemapIndexGenerator sig;
sig = new SitemapIndexGenerator.Options(baseUrl, outFile).dateFormat(dateFormat).autoValidate(autoValidate).build(); sig = new SitemapIndexGenerator.Options(baseUrl, outFile).dateFormat(dateFormat).autoValidate(autoValidate).build();
@ -231,7 +237,7 @@ abstract class SitemapGenerator<U extends ISitemapUrl, THIS extends SitemapGener
return outFile; return outFile;
} }
private void writeSiteMap() { private void writeSiteMap() throws IOException {
if (baseDir == null) { if (baseDir == null) {
throw new NullPointerException("To write to files, baseDir must not be null"); throw new NullPointerException("To write to files, baseDir must not be null");
} }
@ -244,8 +250,9 @@ abstract class SitemapGenerator<U extends ISitemapUrl, THIS extends SitemapGener
} }
File outFile = new File(baseDir, fileNamePrefix+fileNameSuffix); File outFile = new File(baseDir, fileNamePrefix+fileNameSuffix);
outFiles.add(outFile); outFiles.add(outFile);
try {
OutputStreamWriter out; OutputStreamWriter out = null;
try {
if (gzip) { if (gzip) {
FileOutputStream fileStream = new FileOutputStream(outFile); FileOutputStream fileStream = new FileOutputStream(outFile);
GZIPOutputStream gzipStream = new GZIPOutputStream(fileStream); GZIPOutputStream gzipStream = new GZIPOutputStream(fileStream);
@ -260,14 +267,17 @@ abstract class SitemapGenerator<U extends ISitemapUrl, THIS extends SitemapGener
throw new RuntimeException("Problem writing sitemap file " + outFile, e); throw new RuntimeException("Problem writing sitemap file " + outFile, e);
} catch (SAXException e) { } catch (SAXException e) {
throw new RuntimeException("Sitemap file failed to validate (bug?)", e); throw new RuntimeException("Sitemap file failed to validate (bug?)", e);
} } finally {
if(out != null) {
out.close();
}
}
} }
private void writeSiteMap(OutputStreamWriter out) throws IOException { private void writeSiteMap(OutputStreamWriter out) throws IOException {
StringBuilder sb = new StringBuilder(); StringBuilder sb = new StringBuilder();
writeSiteMapAsString(sb, urls); writeSiteMapAsString(sb, urls);
out.write(sb.toString()); out.write(sb.toString());
out.close();
} }
} }

View File

@ -1,5 +1,7 @@
package com.redfin.sitemapgenerator; package com.redfin.sitemapgenerator;
import org.xml.sax.SAXException;
import java.io.File; import java.io.File;
import java.io.FileWriter; import java.io.FileWriter;
import java.io.IOException; import java.io.IOException;
@ -9,8 +11,6 @@ import java.net.URL;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Date; import java.util.Date;
import org.xml.sax.SAXException;
/** /**
* Builds a sitemap index, which points only to other sitemaps. * Builds a sitemap index, which points only to other sitemaps.
* @author Dan Fabulich * @author Dan Fabulich
@ -222,16 +222,26 @@ public class SitemapIndexGenerator {
/** Writes out the sitemap index */ /** Writes out the sitemap index */
public void write() { public void write() {
if (!allowEmptyIndex && urls.isEmpty()) throw new RuntimeException("No URLs added, sitemap index would be empty; you must add some URLs with addUrls"); if (!allowEmptyIndex && urls.isEmpty()) throw new RuntimeException("No URLs added, sitemap index would be empty; you must add some URLs with addUrls");
try { try {
// TODO gzip? is that legal for a sitemap index? FileWriter out = null;
FileWriter out = new FileWriter(outFile); try {
writeSiteMap(out); // TODO gzip? is that legal for a sitemap index?
if (autoValidate) SitemapValidator.validateSitemapIndex(outFile); out = new FileWriter(outFile);
} catch (IOException e) { writeSiteMap(out);
throw new RuntimeException("Problem writing sitemap index file " + outFile, e); if (autoValidate) SitemapValidator.validateSitemapIndex(outFile);
} catch (SAXException e) { } catch (IOException e) {
throw new RuntimeException("Problem validating sitemap index file (bug?)", e); throw new RuntimeException("Problem writing sitemap index file " + outFile, e);
} } catch (SAXException e) {
throw new RuntimeException("Problem validating sitemap index file (bug?)", e);
} finally {
if(out != null) {
out.close();
}
}
} catch (IOException ex) {
throw new RuntimeException("Closing of stream has failed.", ex);
}
} }
private void writeSiteMap(OutputStreamWriter out) throws IOException { private void writeSiteMap(OutputStreamWriter out) throws IOException {
@ -254,7 +264,6 @@ public class SitemapIndexGenerator {
out.write(" </sitemap>\n"); out.write(" </sitemap>\n");
} }
out.write("</sitemapindex>"); out.write("</sitemapindex>");
out.close();
} }
} }

View File

@ -1,9 +1,7 @@
package com.redfin.sitemapgenerator; package com.redfin.sitemapgenerator;
import java.io.File; import org.xml.sax.InputSource;
import java.io.FileReader; import org.xml.sax.SAXException;
import java.io.IOException;
import java.io.InputStream;
import javax.xml.XMLConstants; import javax.xml.XMLConstants;
import javax.xml.transform.sax.SAXSource; import javax.xml.transform.sax.SAXSource;
@ -11,9 +9,10 @@ import javax.xml.transform.stream.StreamSource;
import javax.xml.validation.Schema; import javax.xml.validation.Schema;
import javax.xml.validation.SchemaFactory; import javax.xml.validation.SchemaFactory;
import javax.xml.validation.Validator; import javax.xml.validation.Validator;
import java.io.File;
import org.xml.sax.InputSource; import java.io.FileReader;
import org.xml.sax.SAXException; import java.io.IOException;
import java.io.InputStream;
/** Validates sitemaps and sitemap indexes /** Validates sitemaps and sitemap indexes
* *
@ -41,44 +40,55 @@ public class SitemapValidator {
SchemaFactory factory = SchemaFactory factory =
SchemaFactory.newInstance(XMLConstants.W3C_XML_SCHEMA_NS_URI); SchemaFactory.newInstance(XMLConstants.W3C_XML_SCHEMA_NS_URI);
try { try {
InputStream stream = SitemapValidator.class.getResourceAsStream("sitemap.xsd"); sitemapSchema = lazyLoad(factory, "sitemap.xsd");
if (stream == null) throw new RuntimeException("BUG Couldn't load sitemap.xsd"); sitemapIndexSchema = lazyLoad(factory, "siteindex.xsd");
StreamSource source = new StreamSource(stream);
sitemapSchema = factory.newSchema(source);
stream.close();
stream = SitemapValidator.class.getResourceAsStream("siteindex.xsd");
if (stream == null) throw new RuntimeException("BUG Couldn't load siteindex.xsd");
source = new StreamSource(stream);
sitemapIndexSchema = factory.newSchema(source);
stream.close();
} catch (Exception e) { } catch (Exception e) {
throw new RuntimeException("BUG", e); throw new RuntimeException("BUG", e);
} }
} }
private synchronized static Schema lazyLoad(SchemaFactory factory, String resource) throws IOException, SAXException {
InputStream stream = null;
try {
stream = SitemapValidator.class.getResourceAsStream(resource);
if (stream == null) throw new RuntimeException("BUG Couldn't load " + resource);
StreamSource source = new StreamSource(stream);
return factory.newSchema(source);
} finally {
if(stream != null) {
stream.close();
}
}
}
/** Validates an ordinary web sitemap file (NOT a Google-specific sitemap) */ /** Validates an ordinary web sitemap file (NOT a Google-specific sitemap) */
public static void validateWebSitemap(File sitemap) throws SAXException { public static void validateWebSitemap(File sitemap) throws SAXException, IOException {
lazyLoad(); lazyLoad();
validateXml(sitemap, sitemapSchema); validateXml(sitemap, sitemapSchema);
} }
/** Validates a sitemap index file */ /** Validates a sitemap index file */
public static void validateSitemapIndex(File sitemap) throws SAXException { public static void validateSitemapIndex(File sitemap) throws SAXException, IOException {
lazyLoad(); lazyLoad();
validateXml(sitemap, sitemapIndexSchema); validateXml(sitemap, sitemapIndexSchema);
} }
private static void validateXml(File sitemap, Schema schema) throws SAXException { private static void validateXml(File sitemap, Schema schema) throws SAXException, IOException {
Validator validator = schema.newValidator(); Validator validator = schema.newValidator();
FileReader reader = null;
try { try {
FileReader reader = new FileReader(sitemap); reader = new FileReader(sitemap);
SAXSource source = new SAXSource(new InputSource(reader)); SAXSource source = new SAXSource(new InputSource(reader));
validator.validate(source); validator.validate(source);
reader.close();
} catch (IOException e) { } catch (IOException e) {
throw new RuntimeException(e); throw new RuntimeException(e);
} } finally {
if(reader != null) {
reader.close();
}
}
} }
} }