From 32a7d50eb387ea27f3cd989bd87b2ba77b61ba5b Mon Sep 17 00:00:00 2001
From: Robert Muir <rmuir@apache.org>
Date: Fri, 29 Jun 2012 11:06:59 +0000
Subject: [PATCH] LUCENE-4178: default FieldType.tokenized to true

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1355311 13f79535-47bb-0310-9956-ffa450edef68
---
 lucene/CHANGES.txt                                            | 4 ++++
 .../core/src/java/org/apache/lucene/document/FieldType.java   | 2 +-
 .../core/src/java/org/apache/lucene/document/StringField.java | 2 ++
 .../src/test/org/apache/lucene/document/TestDocument.java     | 1 -
 4 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 61fca0ffa6a..19eaa758a99 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -22,6 +22,10 @@ API Changes
   has a different API (carries a list of tags instead of a compound tag). Upgrade
   of embedded morfologik dictionaries to version 1.9. (Dawid Weiss)
 
+* LUCENE-4178: set 'tokenized' to true on FieldType by default, so that if you
+  make a custom FieldType and set indexed = true, its analyzed by the analyzer.
+  (Robert Muir)
+
 Bug Fixes
 
 * LUCENE-4176: Fix AnalyzingQueryParser to analyze range endpoints as bytes,
diff --git a/lucene/core/src/java/org/apache/lucene/document/FieldType.java b/lucene/core/src/java/org/apache/lucene/document/FieldType.java
index dd9b6acb02c..07cf30d11e5 100644
--- a/lucene/core/src/java/org/apache/lucene/document/FieldType.java
+++ b/lucene/core/src/java/org/apache/lucene/document/FieldType.java
@@ -35,7 +35,7 @@ public class FieldType implements IndexableFieldType {
 
   private boolean indexed;
   private boolean stored;
-  private boolean tokenized;
+  private boolean tokenized = true;
   private boolean storeTermVectors;
   private boolean storeTermVectorOffsets;
   private boolean storeTermVectorPositions;
diff --git a/lucene/core/src/java/org/apache/lucene/document/StringField.java b/lucene/core/src/java/org/apache/lucene/document/StringField.java
index 947596f918a..0629c7dd03b 100644
--- a/lucene/core/src/java/org/apache/lucene/document/StringField.java
+++ b/lucene/core/src/java/org/apache/lucene/document/StringField.java
@@ -39,12 +39,14 @@ public final class StringField extends Field {
     TYPE_NOT_STORED.setIndexed(true);
     TYPE_NOT_STORED.setOmitNorms(true);
     TYPE_NOT_STORED.setIndexOptions(IndexOptions.DOCS_ONLY);
+    TYPE_NOT_STORED.setTokenized(false);
     TYPE_NOT_STORED.freeze();
 
     TYPE_STORED.setIndexed(true);
     TYPE_STORED.setOmitNorms(true);
     TYPE_STORED.setIndexOptions(IndexOptions.DOCS_ONLY);
     TYPE_STORED.setStored(true);
+    TYPE_STORED.setTokenized(false);
     TYPE_STORED.freeze();
   }
 
diff --git a/lucene/core/src/test/org/apache/lucene/document/TestDocument.java b/lucene/core/src/test/org/apache/lucene/document/TestDocument.java
index c27481a047b..a75f396e20c 100644
--- a/lucene/core/src/test/org/apache/lucene/document/TestDocument.java
+++ b/lucene/core/src/test/org/apache/lucene/document/TestDocument.java
@@ -62,7 +62,6 @@ public class TestDocument extends LuceneTestCase {
     assertTrue(binaryFld.binaryValue() != null);
     assertTrue(binaryFld.fieldType().stored());
     assertFalse(binaryFld.fieldType().indexed());
-    assertFalse(binaryFld.fieldType().tokenized());
     
     String binaryTest = doc.getBinaryValue("binary").utf8ToString();
     assertTrue(binaryTest.equals(binaryVal));