honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Added the ability to specify the amount of text to extract and index from an attachment.

This commit is contained in:
Henac 2012-03-04 16:09:21 +11:00
parent bf12b2be21
commit 6a08ca673a
4 changed files with 77 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
pom.xml
View File

@ -35,6 +35,22 @@
</properties>
<repositories>
<repository>
<id>codehausSnapshots</id>
<name>Codehaus Snapshots</name>
<releases>
<enabled>true</enabled>
<updatePolicy>always</updatePolicy>
<checksumPolicy>warn</checksumPolicy>
</releases>
<snapshots>
<enabled>true</enabled>
<updatePolicy>always</updatePolicy>
<checksumPolicy>fail</checksumPolicy>
</snapshots>
<url>http://oss.sonatype.org/content/repositories/releases/</url>
<layout>default</layout>
</repository>
</repositories>
<dependencies>
@ -90,6 +106,7 @@
<include>**/*.json</include>
<include>**/*.yml</include>
<include>**/*.html</include>
<include>**/*.txt</include>
</includes>
</testResource>
</testResources>

15
src/main/java/org/elasticsearch/index/mapper/attachment/AttachmentMapper.java
View File

@ -45,12 +45,15 @@ import static org.elasticsearch.plugin.mapper.attachments.tika.TikaInstance.tika
* {
* file1 : {
* _content_type : "application/pdf",
* _content_length : "500000000",
* _name : "..../something.pdf",
* content : ""
* }
* }
* </pre>
*
* _content_length = Specify the maximum amount of characters to extract from the attachment. If not specified, then the default for
* tika is 100,000 characters. Caution is required when setting large values as this can cause memory issues.
*
*/
public class AttachmentMapper implements Mapper {
@ -237,6 +240,7 @@ public class AttachmentMapper implements Mapper {
public void parse(ParseContext context) throws IOException {
byte[] content = null;
String contentType = null;
int contentLength = 100000;
String name = null;
XContentParser parser = context.parser();
@ -256,10 +260,14 @@ public class AttachmentMapper implements Mapper {
} else if ("_name".equals(currentFieldName)) {
name = parser.text();
}
} else if (token == XContentParser.Token.VALUE_NUMBER) {
if ("_content_length".equals(currentFieldName)) {
contentLength = parser.intValue();
}
}
}
}
Metadata metadata = new Metadata();
if (contentType != null) {
metadata.add(Metadata.CONTENT_TYPE, contentType);
@ -270,9 +278,12 @@ public class AttachmentMapper implements Mapper {
String parsedContent;
try {
// Set the maximum length of strings returned by the parseToString method, -1 sets no limit
tika().setMaxStringLength(contentLength);
parsedContent = tika().parseToString(new FastByteArrayInputStream(content), metadata);
} catch (TikaException e) {
throw new MapperParsingException("Failed to extract text for [" + name + "]", e);
throw new MapperParsingException("Failed to extract [" + contentLength + "] characters of text for [" + name + "]", e);
}
context.externalValue(parsedContent);

9
src/test/java/org/elasticsearch/index/mapper/xcontent/testContentLength.txt Normal file
View File

@ -0,0 +1,9 @@
Begin
BeforeLimit AfterLimit
Broadway
Nearing the end
End

38
src/test/java/org/elasticsearch/plugin/mapper/attachments/test/SimpleAttachmentIntegrationTests.java
View File

@ -95,4 +95,42 @@ public class SimpleAttachmentIntegrationTests {
countResponse = node.client().count(countRequest("test").query(fieldQuery("file", "tests the ability"))).actionGet();
assertThat(countResponse.count(), equalTo(1l));
}
@Test
public void testSimpleAttachmentContentLengthLimit() throws Exception {
String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/xcontent/test-mapping.json");
byte[] txt = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/xcontent/testContentLength.txt");
final int CONTENT_LENGTH_LIMIT = 18;
node.client().admin().indices().putMapping(putMappingRequest("test").type("person").source(mapping)).actionGet();
node.client().index(indexRequest("test").type("person")
.source(jsonBuilder().startObject().field("file").startObject().field("content", txt).field("_content_length", CONTENT_LENGTH_LIMIT).endObject())).actionGet();
node.client().admin().indices().refresh(refreshRequest()).actionGet();
CountResponse countResponse = node.client().count(countRequest("test").query(fieldQuery("file", "BeforeLimit"))).actionGet();
assertThat(countResponse.count(), equalTo(1l));
countResponse = node.client().count(countRequest("test").query(fieldQuery("file", "AfterLimit"))).actionGet();
assertThat(countResponse.count(), equalTo(0l));
}
@Test
public void testSimpleAttachmentNoContentLengthLimit() throws Exception {
String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/xcontent/test-mapping.json");
byte[] txt = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/xcontent/testContentLength.txt");
final int CONTENT_LENGTH_LIMIT = -1;
node.client().admin().indices().putMapping(putMappingRequest("test").type("person").source(mapping)).actionGet();
node.client().index(indexRequest("test").type("person")
.source(jsonBuilder().startObject().field("file").startObject().field("content", txt).field("_content_length", CONTENT_LENGTH_LIMIT).endObject())).actionGet();
node.client().admin().indices().refresh(refreshRequest()).actionGet();
CountResponse countResponse = node.client().count(countRequest("test").query(fieldQuery("file", "Begin"))).actionGet();
assertThat(countResponse.count(), equalTo(1l));
countResponse = node.client().count(countRequest("test").query(fieldQuery("file", "End"))).actionGet();
assertThat(countResponse.count(), equalTo(1l));
}
}