From 933edf7bcc410b261485ee0a8861f990f8936fe3 Mon Sep 17 00:00:00 2001 From: Jun Ohtani Date: Fri, 24 Apr 2015 18:00:32 +0900 Subject: [PATCH] Analysis: Fix wrong position number by analyze API Add breaking chages comment to migrate docs Fix the stopword included text using stopword filter --- docs/reference/migration/migrate_2_0.asciidoc | 4 ++++ .../indices/analyze/TransportAnalyzeAction.java | 2 +- .../indices/analyze/AnalyzeActionTests.java | 12 ++++++++++++ 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/docs/reference/migration/migrate_2_0.asciidoc b/docs/reference/migration/migrate_2_0.asciidoc index 4988d05006c..af77ed20ad6 100644 --- a/docs/reference/migration/migrate_2_0.asciidoc +++ b/docs/reference/migration/migrate_2_0.asciidoc @@ -410,3 +410,7 @@ a single `expand_wildcards` parameter. See <> The `_shutdown` API has been removed without a replacement. Nodes should be managed via operating systems and the provided start/stop scripts. +=== Analyze API + +The Analyze API return 0 as first Token's position instead of 1. + diff --git a/src/main/java/org/elasticsearch/action/admin/indices/analyze/TransportAnalyzeAction.java b/src/main/java/org/elasticsearch/action/admin/indices/analyze/TransportAnalyzeAction.java index 143670809a7..7cdb026e9fb 100644 --- a/src/main/java/org/elasticsearch/action/admin/indices/analyze/TransportAnalyzeAction.java +++ b/src/main/java/org/elasticsearch/action/admin/indices/analyze/TransportAnalyzeAction.java @@ -220,7 +220,7 @@ public class TransportAnalyzeAction extends TransportSingleCustomOperationAction OffsetAttribute offset = stream.addAttribute(OffsetAttribute.class); TypeAttribute type = stream.addAttribute(TypeAttribute.class); - int position = 0; + int position = -1; while (stream.incrementToken()) { int increment = posIncr.getPositionIncrement(); if (increment > 0) { diff --git a/src/test/java/org/elasticsearch/indices/analyze/AnalyzeActionTests.java b/src/test/java/org/elasticsearch/indices/analyze/AnalyzeActionTests.java index 67548a83e3b..7541de9132f 100644 --- a/src/test/java/org/elasticsearch/indices/analyze/AnalyzeActionTests.java +++ b/src/test/java/org/elasticsearch/indices/analyze/AnalyzeActionTests.java @@ -53,18 +53,22 @@ public class AnalyzeActionTests extends ElasticsearchIntegrationTest { assertThat(token.getTerm(), equalTo("this")); assertThat(token.getStartOffset(), equalTo(0)); assertThat(token.getEndOffset(), equalTo(4)); + assertThat(token.getPosition(), equalTo(0)); token = analyzeResponse.getTokens().get(1); assertThat(token.getTerm(), equalTo("is")); assertThat(token.getStartOffset(), equalTo(5)); assertThat(token.getEndOffset(), equalTo(7)); + assertThat(token.getPosition(), equalTo(1)); token = analyzeResponse.getTokens().get(2); assertThat(token.getTerm(), equalTo("a")); assertThat(token.getStartOffset(), equalTo(8)); assertThat(token.getEndOffset(), equalTo(9)); + assertThat(token.getPosition(), equalTo(2)); token = analyzeResponse.getTokens().get(3); assertThat(token.getTerm(), equalTo("test")); assertThat(token.getStartOffset(), equalTo(10)); assertThat(token.getEndOffset(), equalTo(14)); + assertThat(token.getPosition(), equalTo(3)); } } @@ -107,6 +111,14 @@ public class AnalyzeActionTests extends ElasticsearchIntegrationTest { assertThat(token.getTerm(), equalTo("a")); token = analyzeResponse.getTokens().get(3); assertThat(token.getTerm(), equalTo("tset")); + + analyzeResponse = client().admin().indices().prepareAnalyze("of course").setTokenizer("standard").setTokenFilters("stop").get(); + assertThat(analyzeResponse.getTokens().size(), equalTo(1)); + assertThat(analyzeResponse.getTokens().get(0).getTerm(), equalTo("course")); + assertThat(analyzeResponse.getTokens().get(0).getPosition(), equalTo(1)); + assertThat(analyzeResponse.getTokens().get(0).getStartOffset(), equalTo(3)); + assertThat(analyzeResponse.getTokens().get(0).getEndOffset(), equalTo(9)); + } @Test