Closes streams in finally clause whenever they are opened.
This commit is contained in:
parent
168ac4f200
commit
4e84a7cfe8
|
@ -1,18 +1,17 @@
|
||||||
package com.redfin.sitemapgenerator;
|
package com.redfin.sitemapgenerator;
|
||||||
|
|
||||||
|
import org.xml.sax.SAXException;
|
||||||
|
|
||||||
import java.io.File;
|
import java.io.File;
|
||||||
import java.io.FileOutputStream;
|
import java.io.FileOutputStream;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.io.OutputStreamWriter;
|
import java.io.OutputStreamWriter;
|
||||||
import java.net.MalformedURLException;
|
|
||||||
import java.net.URL;
|
import java.net.URL;
|
||||||
import java.nio.charset.Charset;
|
import java.nio.charset.Charset;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.zip.GZIPOutputStream;
|
import java.util.zip.GZIPOutputStream;
|
||||||
|
|
||||||
import org.xml.sax.SAXException;
|
|
||||||
|
|
||||||
abstract class SitemapGenerator<U extends ISitemapUrl, THIS extends SitemapGenerator<U,THIS>> {
|
abstract class SitemapGenerator<U extends ISitemapUrl, THIS extends SitemapGenerator<U,THIS>> {
|
||||||
/** 50000 URLs per sitemap maximum */
|
/** 50000 URLs per sitemap maximum */
|
||||||
public static final int MAX_URLS_PER_SITEMAP = 50000;
|
public static final int MAX_URLS_PER_SITEMAP = 50000;
|
||||||
|
@ -61,8 +60,9 @@ abstract class SitemapGenerator<U extends ISitemapUrl, THIS extends SitemapGener
|
||||||
* or else write out one sitemap immediately.
|
* or else write out one sitemap immediately.
|
||||||
* @param url the URL to add to this sitemap
|
* @param url the URL to add to this sitemap
|
||||||
* @return this
|
* @return this
|
||||||
|
* @throws IOException when closing of streams has failed
|
||||||
*/
|
*/
|
||||||
public THIS addUrl(U url) {
|
public THIS addUrl(U url) throws IOException {
|
||||||
if (finished) throw new RuntimeException("Sitemap already printed; you must create a new generator to make more sitemaps");
|
if (finished) throw new RuntimeException("Sitemap already printed; you must create a new generator to make more sitemaps");
|
||||||
UrlUtils.checkUrl(url.getUrl(), baseUrl);
|
UrlUtils.checkUrl(url.getUrl(), baseUrl);
|
||||||
if (urls.size() == maxUrls) {
|
if (urls.size() == maxUrls) {
|
||||||
|
@ -83,8 +83,21 @@ abstract class SitemapGenerator<U extends ISitemapUrl, THIS extends SitemapGener
|
||||||
* or write out one sitemap immediately.
|
* or write out one sitemap immediately.
|
||||||
* @param urls the URLs to add to this sitemap
|
* @param urls the URLs to add to this sitemap
|
||||||
* @return this
|
* @return this
|
||||||
|
* @throws IOException when closing of streams has failed.
|
||||||
*/
|
*/
|
||||||
public THIS addUrls(Iterable<? extends U> urls) {
|
public THIS addUrls(Iterable<? extends U> urls) throws IOException {
|
||||||
|
for (U url : urls) addUrl(url);
|
||||||
|
return getThis();
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Add multiple URLs of the appropriate type to this sitemap, one at a time.
|
||||||
|
* If we have reached the maximum number of URLs, we'll throw an exception if {@link #allowMultipleSitemaps} is false,
|
||||||
|
* or write out one sitemap immediately.
|
||||||
|
* @param urls the URLs to add to this sitemap
|
||||||
|
* @return this
|
||||||
|
* @throws IOException when closing of streams has failed.
|
||||||
|
*/
|
||||||
|
public THIS addUrls(U... urls) throws IOException {
|
||||||
for (U url : urls) addUrl(url);
|
for (U url : urls) addUrl(url);
|
||||||
return getThis();
|
return getThis();
|
||||||
}
|
}
|
||||||
|
@ -95,19 +108,7 @@ abstract class SitemapGenerator<U extends ISitemapUrl, THIS extends SitemapGener
|
||||||
* @param urls the URLs to add to this sitemap
|
* @param urls the URLs to add to this sitemap
|
||||||
* @return this
|
* @return this
|
||||||
*/
|
*/
|
||||||
public THIS addUrls(U... urls) {
|
public THIS addUrls(String... urls) {
|
||||||
for (U url : urls) addUrl(url);
|
|
||||||
return getThis();
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Add multiple URLs of the appropriate type to this sitemap, one at a time.
|
|
||||||
* If we have reached the maximum number of URLs, we'll throw an exception if {@link #allowMultipleSitemaps} is false,
|
|
||||||
* or write out one sitemap immediately.
|
|
||||||
* @param urls the URLs to add to this sitemap
|
|
||||||
* @return this
|
|
||||||
* @throws MalformedURLException
|
|
||||||
*/
|
|
||||||
public THIS addUrls(String... urls) throws MalformedURLException {
|
|
||||||
for (String url : urls) addUrl(url);
|
for (String url : urls) addUrl(url);
|
||||||
return getThis();
|
return getThis();
|
||||||
}
|
}
|
||||||
|
@ -117,16 +118,15 @@ abstract class SitemapGenerator<U extends ISitemapUrl, THIS extends SitemapGener
|
||||||
* or else write out one sitemap immediately.
|
* or else write out one sitemap immediately.
|
||||||
* @param url the URL to add to this sitemap
|
* @param url the URL to add to this sitemap
|
||||||
* @return this
|
* @return this
|
||||||
* @throws MalformedURLException
|
|
||||||
*/
|
*/
|
||||||
public THIS addUrl(String url) throws MalformedURLException {
|
public THIS addUrl(String url) {
|
||||||
U sitemapUrl;
|
U sitemapUrl;
|
||||||
try {
|
try {
|
||||||
sitemapUrl = renderer.getUrlClass().getConstructor(String.class).newInstance(url);
|
sitemapUrl = renderer.getUrlClass().getConstructor(String.class).newInstance(url);
|
||||||
} catch (Exception e) {
|
return addUrl(sitemapUrl);
|
||||||
|
} catch (Exception e) {
|
||||||
throw new RuntimeException(e);
|
throw new RuntimeException(e);
|
||||||
}
|
}
|
||||||
return addUrl(sitemapUrl);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Add multiple URLs of the appropriate type to this sitemap, one at a time.
|
/** Add multiple URLs of the appropriate type to this sitemap, one at a time.
|
||||||
|
@ -150,10 +150,10 @@ abstract class SitemapGenerator<U extends ISitemapUrl, THIS extends SitemapGener
|
||||||
U sitemapUrl;
|
U sitemapUrl;
|
||||||
try {
|
try {
|
||||||
sitemapUrl = renderer.getUrlClass().getConstructor(URL.class).newInstance(url);
|
sitemapUrl = renderer.getUrlClass().getConstructor(URL.class).newInstance(url);
|
||||||
} catch (Exception e) {
|
return addUrl(sitemapUrl);
|
||||||
|
} catch (Exception e) {
|
||||||
throw new RuntimeException(e);
|
throw new RuntimeException(e);
|
||||||
}
|
}
|
||||||
return addUrl(sitemapUrl);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@SuppressWarnings("unchecked")
|
@SuppressWarnings("unchecked")
|
||||||
|
@ -168,7 +168,11 @@ abstract class SitemapGenerator<U extends ISitemapUrl, THIS extends SitemapGener
|
||||||
public List<File> write() {
|
public List<File> write() {
|
||||||
if (finished) throw new RuntimeException("Sitemap already printed; you must create a new generator to make more sitemaps");
|
if (finished) throw new RuntimeException("Sitemap already printed; you must create a new generator to make more sitemaps");
|
||||||
if (!allowEmptySitemap && urls.isEmpty() && mapCount == 0) throw new RuntimeException("No URLs added, sitemap would be empty; you must add some URLs with addUrls");
|
if (!allowEmptySitemap && urls.isEmpty() && mapCount == 0) throw new RuntimeException("No URLs added, sitemap would be empty; you must add some URLs with addUrls");
|
||||||
writeSiteMap();
|
try {
|
||||||
|
writeSiteMap();
|
||||||
|
} catch (IOException ex) {
|
||||||
|
throw new RuntimeException("Closing of streams has failed at some point.", ex);
|
||||||
|
}
|
||||||
finished = true;
|
finished = true;
|
||||||
return outFiles;
|
return outFiles;
|
||||||
}
|
}
|
||||||
|
@ -211,8 +215,9 @@ abstract class SitemapGenerator<U extends ISitemapUrl, THIS extends SitemapGener
|
||||||
/**
|
/**
|
||||||
* After you've called {@link #write()}, call this to generate a sitemap index of all sitemaps you generated.
|
* After you've called {@link #write()}, call this to generate a sitemap index of all sitemaps you generated.
|
||||||
* The sitemap index is written to {baseDir}/sitemap_index.xml
|
* The sitemap index is written to {baseDir}/sitemap_index.xml
|
||||||
|
* @throws IOException when closing of streams has failed
|
||||||
*/
|
*/
|
||||||
public File writeSitemapsWithIndex() {
|
public File writeSitemapsWithIndex() throws IOException {
|
||||||
if (!finished) throw new RuntimeException("Sitemaps not generated yet; call write() first");
|
if (!finished) throw new RuntimeException("Sitemaps not generated yet; call write() first");
|
||||||
File outFile = new File(baseDir, "sitemap_index.xml");
|
File outFile = new File(baseDir, "sitemap_index.xml");
|
||||||
return writeSitemapsWithIndex(outFile);
|
return writeSitemapsWithIndex(outFile);
|
||||||
|
@ -222,8 +227,9 @@ abstract class SitemapGenerator<U extends ISitemapUrl, THIS extends SitemapGener
|
||||||
* After you've called {@link #write()}, call this to generate a sitemap index of all sitemaps you generated.
|
* After you've called {@link #write()}, call this to generate a sitemap index of all sitemaps you generated.
|
||||||
*
|
*
|
||||||
* @param outFile the destination file of the sitemap index.
|
* @param outFile the destination file of the sitemap index.
|
||||||
|
* @throws IOException when closing of streams has failed
|
||||||
*/
|
*/
|
||||||
public File writeSitemapsWithIndex(File outFile) {
|
public File writeSitemapsWithIndex(File outFile) throws IOException {
|
||||||
if (!finished) throw new RuntimeException("Sitemaps not generated yet; call write() first");
|
if (!finished) throw new RuntimeException("Sitemaps not generated yet; call write() first");
|
||||||
SitemapIndexGenerator sig;
|
SitemapIndexGenerator sig;
|
||||||
sig = new SitemapIndexGenerator.Options(baseUrl, outFile).dateFormat(dateFormat).autoValidate(autoValidate).build();
|
sig = new SitemapIndexGenerator.Options(baseUrl, outFile).dateFormat(dateFormat).autoValidate(autoValidate).build();
|
||||||
|
@ -231,7 +237,7 @@ abstract class SitemapGenerator<U extends ISitemapUrl, THIS extends SitemapGener
|
||||||
return outFile;
|
return outFile;
|
||||||
}
|
}
|
||||||
|
|
||||||
private void writeSiteMap() {
|
private void writeSiteMap() throws IOException {
|
||||||
if (baseDir == null) {
|
if (baseDir == null) {
|
||||||
throw new NullPointerException("To write to files, baseDir must not be null");
|
throw new NullPointerException("To write to files, baseDir must not be null");
|
||||||
}
|
}
|
||||||
|
@ -244,8 +250,9 @@ abstract class SitemapGenerator<U extends ISitemapUrl, THIS extends SitemapGener
|
||||||
}
|
}
|
||||||
File outFile = new File(baseDir, fileNamePrefix+fileNameSuffix);
|
File outFile = new File(baseDir, fileNamePrefix+fileNameSuffix);
|
||||||
outFiles.add(outFile);
|
outFiles.add(outFile);
|
||||||
try {
|
|
||||||
OutputStreamWriter out;
|
OutputStreamWriter out = null;
|
||||||
|
try {
|
||||||
if (gzip) {
|
if (gzip) {
|
||||||
FileOutputStream fileStream = new FileOutputStream(outFile);
|
FileOutputStream fileStream = new FileOutputStream(outFile);
|
||||||
GZIPOutputStream gzipStream = new GZIPOutputStream(fileStream);
|
GZIPOutputStream gzipStream = new GZIPOutputStream(fileStream);
|
||||||
|
@ -260,14 +267,17 @@ abstract class SitemapGenerator<U extends ISitemapUrl, THIS extends SitemapGener
|
||||||
throw new RuntimeException("Problem writing sitemap file " + outFile, e);
|
throw new RuntimeException("Problem writing sitemap file " + outFile, e);
|
||||||
} catch (SAXException e) {
|
} catch (SAXException e) {
|
||||||
throw new RuntimeException("Sitemap file failed to validate (bug?)", e);
|
throw new RuntimeException("Sitemap file failed to validate (bug?)", e);
|
||||||
}
|
} finally {
|
||||||
|
if(out != null) {
|
||||||
|
out.close();
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private void writeSiteMap(OutputStreamWriter out) throws IOException {
|
private void writeSiteMap(OutputStreamWriter out) throws IOException {
|
||||||
StringBuilder sb = new StringBuilder();
|
StringBuilder sb = new StringBuilder();
|
||||||
writeSiteMapAsString(sb, urls);
|
writeSiteMapAsString(sb, urls);
|
||||||
out.write(sb.toString());
|
out.write(sb.toString());
|
||||||
out.close();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,5 +1,7 @@
|
||||||
package com.redfin.sitemapgenerator;
|
package com.redfin.sitemapgenerator;
|
||||||
|
|
||||||
|
import org.xml.sax.SAXException;
|
||||||
|
|
||||||
import java.io.File;
|
import java.io.File;
|
||||||
import java.io.FileWriter;
|
import java.io.FileWriter;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
|
@ -9,8 +11,6 @@ import java.net.URL;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.Date;
|
import java.util.Date;
|
||||||
|
|
||||||
import org.xml.sax.SAXException;
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Builds a sitemap index, which points only to other sitemaps.
|
* Builds a sitemap index, which points only to other sitemaps.
|
||||||
* @author Dan Fabulich
|
* @author Dan Fabulich
|
||||||
|
@ -222,16 +222,26 @@ public class SitemapIndexGenerator {
|
||||||
/** Writes out the sitemap index */
|
/** Writes out the sitemap index */
|
||||||
public void write() {
|
public void write() {
|
||||||
if (!allowEmptyIndex && urls.isEmpty()) throw new RuntimeException("No URLs added, sitemap index would be empty; you must add some URLs with addUrls");
|
if (!allowEmptyIndex && urls.isEmpty()) throw new RuntimeException("No URLs added, sitemap index would be empty; you must add some URLs with addUrls");
|
||||||
try {
|
try {
|
||||||
// TODO gzip? is that legal for a sitemap index?
|
FileWriter out = null;
|
||||||
FileWriter out = new FileWriter(outFile);
|
try {
|
||||||
writeSiteMap(out);
|
// TODO gzip? is that legal for a sitemap index?
|
||||||
if (autoValidate) SitemapValidator.validateSitemapIndex(outFile);
|
out = new FileWriter(outFile);
|
||||||
} catch (IOException e) {
|
writeSiteMap(out);
|
||||||
throw new RuntimeException("Problem writing sitemap index file " + outFile, e);
|
if (autoValidate) SitemapValidator.validateSitemapIndex(outFile);
|
||||||
} catch (SAXException e) {
|
} catch (IOException e) {
|
||||||
throw new RuntimeException("Problem validating sitemap index file (bug?)", e);
|
throw new RuntimeException("Problem writing sitemap index file " + outFile, e);
|
||||||
}
|
} catch (SAXException e) {
|
||||||
|
throw new RuntimeException("Problem validating sitemap index file (bug?)", e);
|
||||||
|
} finally {
|
||||||
|
if(out != null) {
|
||||||
|
out.close();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch (IOException ex) {
|
||||||
|
throw new RuntimeException("Closing of stream has failed.", ex);
|
||||||
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private void writeSiteMap(OutputStreamWriter out) throws IOException {
|
private void writeSiteMap(OutputStreamWriter out) throws IOException {
|
||||||
|
@ -254,7 +264,6 @@ public class SitemapIndexGenerator {
|
||||||
out.write(" </sitemap>\n");
|
out.write(" </sitemap>\n");
|
||||||
}
|
}
|
||||||
out.write("</sitemapindex>");
|
out.write("</sitemapindex>");
|
||||||
out.close();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,9 +1,7 @@
|
||||||
package com.redfin.sitemapgenerator;
|
package com.redfin.sitemapgenerator;
|
||||||
|
|
||||||
import java.io.File;
|
import org.xml.sax.InputSource;
|
||||||
import java.io.FileReader;
|
import org.xml.sax.SAXException;
|
||||||
import java.io.IOException;
|
|
||||||
import java.io.InputStream;
|
|
||||||
|
|
||||||
import javax.xml.XMLConstants;
|
import javax.xml.XMLConstants;
|
||||||
import javax.xml.transform.sax.SAXSource;
|
import javax.xml.transform.sax.SAXSource;
|
||||||
|
@ -11,9 +9,10 @@ import javax.xml.transform.stream.StreamSource;
|
||||||
import javax.xml.validation.Schema;
|
import javax.xml.validation.Schema;
|
||||||
import javax.xml.validation.SchemaFactory;
|
import javax.xml.validation.SchemaFactory;
|
||||||
import javax.xml.validation.Validator;
|
import javax.xml.validation.Validator;
|
||||||
|
import java.io.File;
|
||||||
import org.xml.sax.InputSource;
|
import java.io.FileReader;
|
||||||
import org.xml.sax.SAXException;
|
import java.io.IOException;
|
||||||
|
import java.io.InputStream;
|
||||||
|
|
||||||
/** Validates sitemaps and sitemap indexes
|
/** Validates sitemaps and sitemap indexes
|
||||||
*
|
*
|
||||||
|
@ -41,44 +40,55 @@ public class SitemapValidator {
|
||||||
SchemaFactory factory =
|
SchemaFactory factory =
|
||||||
SchemaFactory.newInstance(XMLConstants.W3C_XML_SCHEMA_NS_URI);
|
SchemaFactory.newInstance(XMLConstants.W3C_XML_SCHEMA_NS_URI);
|
||||||
try {
|
try {
|
||||||
InputStream stream = SitemapValidator.class.getResourceAsStream("sitemap.xsd");
|
sitemapSchema = lazyLoad(factory, "sitemap.xsd");
|
||||||
if (stream == null) throw new RuntimeException("BUG Couldn't load sitemap.xsd");
|
sitemapIndexSchema = lazyLoad(factory, "siteindex.xsd");
|
||||||
StreamSource source = new StreamSource(stream);
|
|
||||||
sitemapSchema = factory.newSchema(source);
|
|
||||||
stream.close();
|
|
||||||
|
|
||||||
stream = SitemapValidator.class.getResourceAsStream("siteindex.xsd");
|
|
||||||
if (stream == null) throw new RuntimeException("BUG Couldn't load siteindex.xsd");
|
|
||||||
source = new StreamSource(stream);
|
|
||||||
sitemapIndexSchema = factory.newSchema(source);
|
|
||||||
stream.close();
|
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
throw new RuntimeException("BUG", e);
|
throw new RuntimeException("BUG", e);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private synchronized static Schema lazyLoad(SchemaFactory factory, String resource) throws IOException, SAXException {
|
||||||
|
InputStream stream = null;
|
||||||
|
|
||||||
|
try {
|
||||||
|
stream = SitemapValidator.class.getResourceAsStream(resource);
|
||||||
|
if (stream == null) throw new RuntimeException("BUG Couldn't load " + resource);
|
||||||
|
StreamSource source = new StreamSource(stream);
|
||||||
|
return factory.newSchema(source);
|
||||||
|
} finally {
|
||||||
|
if(stream != null) {
|
||||||
|
stream.close();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
/** Validates an ordinary web sitemap file (NOT a Google-specific sitemap) */
|
/** Validates an ordinary web sitemap file (NOT a Google-specific sitemap) */
|
||||||
public static void validateWebSitemap(File sitemap) throws SAXException {
|
public static void validateWebSitemap(File sitemap) throws SAXException, IOException {
|
||||||
lazyLoad();
|
lazyLoad();
|
||||||
validateXml(sitemap, sitemapSchema);
|
validateXml(sitemap, sitemapSchema);
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Validates a sitemap index file */
|
/** Validates a sitemap index file */
|
||||||
public static void validateSitemapIndex(File sitemap) throws SAXException {
|
public static void validateSitemapIndex(File sitemap) throws SAXException, IOException {
|
||||||
lazyLoad();
|
lazyLoad();
|
||||||
validateXml(sitemap, sitemapIndexSchema);
|
validateXml(sitemap, sitemapIndexSchema);
|
||||||
}
|
}
|
||||||
|
|
||||||
private static void validateXml(File sitemap, Schema schema) throws SAXException {
|
private static void validateXml(File sitemap, Schema schema) throws SAXException, IOException {
|
||||||
Validator validator = schema.newValidator();
|
Validator validator = schema.newValidator();
|
||||||
|
FileReader reader = null;
|
||||||
try {
|
try {
|
||||||
FileReader reader = new FileReader(sitemap);
|
reader = new FileReader(sitemap);
|
||||||
SAXSource source = new SAXSource(new InputSource(reader));
|
SAXSource source = new SAXSource(new InputSource(reader));
|
||||||
validator.validate(source);
|
validator.validate(source);
|
||||||
reader.close();
|
|
||||||
} catch (IOException e) {
|
} catch (IOException e) {
|
||||||
throw new RuntimeException(e);
|
throw new RuntimeException(e);
|
||||||
}
|
} finally {
|
||||||
|
if(reader != null) {
|
||||||
|
reader.close();
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue