Added the ability to specify the amount of text to extract and index from an attachment.
This commit is contained in:
parent
bf12b2be21
commit
6a08ca673a
17
pom.xml
17
pom.xml
|
@ -35,6 +35,22 @@
|
||||||
</properties>
|
</properties>
|
||||||
|
|
||||||
<repositories>
|
<repositories>
|
||||||
|
<repository>
|
||||||
|
<id>codehausSnapshots</id>
|
||||||
|
<name>Codehaus Snapshots</name>
|
||||||
|
<releases>
|
||||||
|
<enabled>true</enabled>
|
||||||
|
<updatePolicy>always</updatePolicy>
|
||||||
|
<checksumPolicy>warn</checksumPolicy>
|
||||||
|
</releases>
|
||||||
|
<snapshots>
|
||||||
|
<enabled>true</enabled>
|
||||||
|
<updatePolicy>always</updatePolicy>
|
||||||
|
<checksumPolicy>fail</checksumPolicy>
|
||||||
|
</snapshots>
|
||||||
|
<url>http://oss.sonatype.org/content/repositories/releases/</url>
|
||||||
|
<layout>default</layout>
|
||||||
|
</repository>
|
||||||
</repositories>
|
</repositories>
|
||||||
|
|
||||||
<dependencies>
|
<dependencies>
|
||||||
|
@ -90,6 +106,7 @@
|
||||||
<include>**/*.json</include>
|
<include>**/*.json</include>
|
||||||
<include>**/*.yml</include>
|
<include>**/*.yml</include>
|
||||||
<include>**/*.html</include>
|
<include>**/*.html</include>
|
||||||
|
<include>**/*.txt</include>
|
||||||
</includes>
|
</includes>
|
||||||
</testResource>
|
</testResource>
|
||||||
</testResources>
|
</testResources>
|
||||||
|
|
|
@ -45,12 +45,15 @@ import static org.elasticsearch.plugin.mapper.attachments.tika.TikaInstance.tika
|
||||||
* {
|
* {
|
||||||
* file1 : {
|
* file1 : {
|
||||||
* _content_type : "application/pdf",
|
* _content_type : "application/pdf",
|
||||||
|
* _content_length : "500000000",
|
||||||
* _name : "..../something.pdf",
|
* _name : "..../something.pdf",
|
||||||
* content : ""
|
* content : ""
|
||||||
* }
|
* }
|
||||||
* }
|
* }
|
||||||
* </pre>
|
* </pre>
|
||||||
*
|
*
|
||||||
|
* _content_length = Specify the maximum amount of characters to extract from the attachment. If not specified, then the default for
|
||||||
|
* tika is 100,000 characters. Caution is required when setting large values as this can cause memory issues.
|
||||||
*
|
*
|
||||||
*/
|
*/
|
||||||
public class AttachmentMapper implements Mapper {
|
public class AttachmentMapper implements Mapper {
|
||||||
|
@ -237,6 +240,7 @@ public class AttachmentMapper implements Mapper {
|
||||||
public void parse(ParseContext context) throws IOException {
|
public void parse(ParseContext context) throws IOException {
|
||||||
byte[] content = null;
|
byte[] content = null;
|
||||||
String contentType = null;
|
String contentType = null;
|
||||||
|
int contentLength = 100000;
|
||||||
String name = null;
|
String name = null;
|
||||||
|
|
||||||
XContentParser parser = context.parser();
|
XContentParser parser = context.parser();
|
||||||
|
@ -256,6 +260,10 @@ public class AttachmentMapper implements Mapper {
|
||||||
} else if ("_name".equals(currentFieldName)) {
|
} else if ("_name".equals(currentFieldName)) {
|
||||||
name = parser.text();
|
name = parser.text();
|
||||||
}
|
}
|
||||||
|
} else if (token == XContentParser.Token.VALUE_NUMBER) {
|
||||||
|
if ("_content_length".equals(currentFieldName)) {
|
||||||
|
contentLength = parser.intValue();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -270,9 +278,12 @@ public class AttachmentMapper implements Mapper {
|
||||||
|
|
||||||
String parsedContent;
|
String parsedContent;
|
||||||
try {
|
try {
|
||||||
|
// Set the maximum length of strings returned by the parseToString method, -1 sets no limit
|
||||||
|
tika().setMaxStringLength(contentLength);
|
||||||
|
|
||||||
parsedContent = tika().parseToString(new FastByteArrayInputStream(content), metadata);
|
parsedContent = tika().parseToString(new FastByteArrayInputStream(content), metadata);
|
||||||
} catch (TikaException e) {
|
} catch (TikaException e) {
|
||||||
throw new MapperParsingException("Failed to extract text for [" + name + "]", e);
|
throw new MapperParsingException("Failed to extract [" + contentLength + "] characters of text for [" + name + "]", e);
|
||||||
}
|
}
|
||||||
|
|
||||||
context.externalValue(parsedContent);
|
context.externalValue(parsedContent);
|
||||||
|
|
|
@ -0,0 +1,9 @@
|
||||||
|
Begin
|
||||||
|
|
||||||
|
BeforeLimit AfterLimit
|
||||||
|
|
||||||
|
Broadway
|
||||||
|
|
||||||
|
Nearing the end
|
||||||
|
|
||||||
|
End
|
|
@ -95,4 +95,42 @@ public class SimpleAttachmentIntegrationTests {
|
||||||
countResponse = node.client().count(countRequest("test").query(fieldQuery("file", "tests the ability"))).actionGet();
|
countResponse = node.client().count(countRequest("test").query(fieldQuery("file", "tests the ability"))).actionGet();
|
||||||
assertThat(countResponse.count(), equalTo(1l));
|
assertThat(countResponse.count(), equalTo(1l));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testSimpleAttachmentContentLengthLimit() throws Exception {
|
||||||
|
String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/xcontent/test-mapping.json");
|
||||||
|
byte[] txt = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/xcontent/testContentLength.txt");
|
||||||
|
final int CONTENT_LENGTH_LIMIT = 18;
|
||||||
|
|
||||||
|
node.client().admin().indices().putMapping(putMappingRequest("test").type("person").source(mapping)).actionGet();
|
||||||
|
|
||||||
|
node.client().index(indexRequest("test").type("person")
|
||||||
|
.source(jsonBuilder().startObject().field("file").startObject().field("content", txt).field("_content_length", CONTENT_LENGTH_LIMIT).endObject())).actionGet();
|
||||||
|
node.client().admin().indices().refresh(refreshRequest()).actionGet();
|
||||||
|
|
||||||
|
CountResponse countResponse = node.client().count(countRequest("test").query(fieldQuery("file", "BeforeLimit"))).actionGet();
|
||||||
|
assertThat(countResponse.count(), equalTo(1l));
|
||||||
|
|
||||||
|
countResponse = node.client().count(countRequest("test").query(fieldQuery("file", "AfterLimit"))).actionGet();
|
||||||
|
assertThat(countResponse.count(), equalTo(0l));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testSimpleAttachmentNoContentLengthLimit() throws Exception {
|
||||||
|
String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/xcontent/test-mapping.json");
|
||||||
|
byte[] txt = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/xcontent/testContentLength.txt");
|
||||||
|
final int CONTENT_LENGTH_LIMIT = -1;
|
||||||
|
|
||||||
|
node.client().admin().indices().putMapping(putMappingRequest("test").type("person").source(mapping)).actionGet();
|
||||||
|
|
||||||
|
node.client().index(indexRequest("test").type("person")
|
||||||
|
.source(jsonBuilder().startObject().field("file").startObject().field("content", txt).field("_content_length", CONTENT_LENGTH_LIMIT).endObject())).actionGet();
|
||||||
|
node.client().admin().indices().refresh(refreshRequest()).actionGet();
|
||||||
|
|
||||||
|
CountResponse countResponse = node.client().count(countRequest("test").query(fieldQuery("file", "Begin"))).actionGet();
|
||||||
|
assertThat(countResponse.count(), equalTo(1l));
|
||||||
|
|
||||||
|
countResponse = node.client().count(countRequest("test").query(fieldQuery("file", "End"))).actionGet();
|
||||||
|
assertThat(countResponse.count(), equalTo(1l));
|
||||||
|
}
|
||||||
}
|
}
|
Loading…
Reference in New Issue