LUCENE-4055: impl shared docstores as codec attributes

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/branches/lucene4055@1342121 13f79535-47bb-0310-9956-ffa450edef68

lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xSegmentInfoFormat.java

@@ -17,12 +17,9 @@ package org.apache.lucene.codecs.lucene3x;
  * limitations under the License.
  */
 
-import java.util.Set;
-
 import org.apache.lucene.codecs.SegmentInfoFormat;
 import org.apache.lucene.codecs.SegmentInfoReader;
 import org.apache.lucene.codecs.SegmentInfoWriter;
-import org.apache.lucene.index.IndexFileNames;
 import org.apache.lucene.index.SegmentInfo;
 
 /**
@@ -62,4 +59,30 @@ public class Lucene3xSegmentInfoFormat extends SegmentInfoFormat {
   public SegmentInfoWriter getSegmentInfosWriter() {
     throw new UnsupportedOperationException("this codec can only be used for reading");
   }
+  
+  // only for backwards compat
+  public static final String DS_OFFSET_KEY = Lucene3xSegmentInfoFormat.class.getSimpleName() + ".dsoffset";
+  public static final String DS_NAME_KEY = Lucene3xSegmentInfoFormat.class.getSimpleName() + ".dsname";
+  public static final String DS_COMPOUND_KEY = Lucene3xSegmentInfoFormat.class.getSimpleName() + ".dscompound";
+  
+  /** 
+   * @return if this segment shares stored fields & vectors, this
+   *         offset is where in that file this segment's docs begin 
+   */
+  public static int getDocStoreOffset(SegmentInfo si) {
+    String v = si.getAttribute(DS_OFFSET_KEY);
+    return v == null ? -1 : Integer.parseInt(v);
+  }
+  
+  /** @return name used to derive fields/vectors file we share with other segments */
+  public static String getDocStoreSegment(SegmentInfo si) {
+    String v = si.getAttribute(DS_NAME_KEY);
+    return v == null ? si.name : v;
+  }
+  
+  /** @return whether doc store files are stored in compound file (*.cfx) */
+  public static boolean getDocStoreIsCompoundFile(SegmentInfo si) {
+    String v = si.getAttribute(DS_COMPOUND_KEY);
+    return v == null ? false : Boolean.parseBoolean(v);
+  }
 }

lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xSegmentInfoReader.java

@@ -23,7 +23,6 @@ import java.util.HashSet;
 import java.util.Map;
 import java.util.Set;
 
-import org.apache.lucene.codecs.Codec;
 import org.apache.lucene.codecs.SegmentInfoReader;
 import org.apache.lucene.index.IndexFileNames;
 import org.apache.lucene.index.IndexFormatTooNewException;
@@ -31,7 +30,6 @@ import org.apache.lucene.index.IndexFormatTooOldException;
 import org.apache.lucene.index.SegmentInfo;
 import org.apache.lucene.index.SegmentInfoPerCommit;
 import org.apache.lucene.index.SegmentInfos;
-import org.apache.lucene.store.ChecksumIndexInput;
 import org.apache.lucene.store.CompoundFileDirectory;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.IOContext;
@@ -59,10 +57,10 @@ public class Lucene3xSegmentInfoReader extends SegmentInfoReader {
         // 2.x segment, and an IndexFormatTooOldException will be thrown,
         // which is what we want.
         Directory dir = directory;
-        if (si.getDocStoreOffset() != -1) {
+        if (Lucene3xSegmentInfoFormat.getDocStoreOffset(si) != -1) {
-          if (si.getDocStoreIsCompoundFile()) {
+          if (Lucene3xSegmentInfoFormat.getDocStoreIsCompoundFile(si)) {
             dir = new CompoundFileDirectory(dir, IndexFileNames.segmentFileName(
-                si.getDocStoreSegment(), "",
+                Lucene3xSegmentInfoFormat.getDocStoreSegment(si), "",
                 Lucene3xCodec.COMPOUND_FILE_STORE_EXTENSION), IOContext.READONCE, false);
           }
         } else if (si.getUseCompoundFile()) {
@@ -71,7 +69,7 @@ public class Lucene3xSegmentInfoReader extends SegmentInfoReader {
         }
 
         try {
-          Lucene3xStoredFieldsReader.checkCodeVersion(dir, si.getDocStoreSegment());
+          Lucene3xStoredFieldsReader.checkCodeVersion(dir, Lucene3xSegmentInfoFormat.getDocStoreSegment(si));
         } finally {
           // If we opened the directory, close it
           if (dir != directory) dir.close();
@@ -152,15 +150,37 @@ public class Lucene3xSegmentInfoReader extends SegmentInfoReader {
 
     final int docCount = input.readInt();
     final long delGen = input.readLong();
-    final int docStoreOffset = input.readInt();
+    
+    final int docStoreOffset;
     final String docStoreSegment;
     final boolean docStoreIsCompoundFile;
-    if (docStoreOffset != -1) {
+    final Map<String,String> attributes;
-      docStoreSegment = input.readString();
+    
-      docStoreIsCompoundFile = input.readByte() == SegmentInfo.YES;
+    if (format == Lucene3xSegmentInfoFormat.FORMAT_4X_UPGRADE) {
+      // we already upgraded to 4.x si format: so shared docstore stuff is in the attributes map.
+      attributes = input.readStringStringMap();
+      String v = attributes.get(Lucene3xSegmentInfoFormat.DS_OFFSET_KEY);
+      docStoreOffset = v == null ? -1 : Integer.parseInt(v);
+      
+      v = attributes.get(Lucene3xSegmentInfoFormat.DS_NAME_KEY);
+      docStoreSegment = v == null ? segmentName : v;
+      
+      v = attributes.get(Lucene3xSegmentInfoFormat.DS_COMPOUND_KEY);
+      docStoreIsCompoundFile = v == null ? false : Boolean.parseBoolean(v);
     } else {
-      docStoreSegment = name;
+      // for older formats, parse the docstore stuff and shove it into attributes
-      docStoreIsCompoundFile = false;
+      attributes = new HashMap<String,String>();
+      docStoreOffset = input.readInt();
+      if (docStoreOffset != -1) {
+        docStoreSegment = input.readString();
+        docStoreIsCompoundFile = input.readByte() == SegmentInfo.YES;
+        attributes.put(Lucene3xSegmentInfoFormat.DS_OFFSET_KEY, Integer.toString(docStoreOffset));
+        attributes.put(Lucene3xSegmentInfoFormat.DS_NAME_KEY, docStoreSegment);
+        attributes.put(Lucene3xSegmentInfoFormat.DS_COMPOUND_KEY, Boolean.toString(docStoreIsCompoundFile));
+      } else {
+        docStoreSegment = name;
+        docStoreIsCompoundFile = false;
+      }
     }
 
     // pre-4.0 indexes write a byte if there is a single norms file
@@ -237,17 +257,16 @@ public class Lucene3xSegmentInfoReader extends SegmentInfoReader {
           } else if (gen == SegmentInfo.NO) {
             // No separate norm
           } else {
-            // nocommit -- i thought _X_N.sY files were pre-3.0...????
+            // We should have already hit indexformat too old exception
             assert false;
           }
         }
       }
     }
 
-    // nocommit: convert 3.x specific stuff (shared docstores, normgen, etc) into attributes
+    // nocommit: convert normgen into attributes?
-    SegmentInfo info = new SegmentInfo(dir, version, segmentName, docCount, docStoreOffset,
+    SegmentInfo info = new SegmentInfo(dir, version, segmentName, docCount, normGen, isCompoundFile,
-                                       docStoreSegment, docStoreIsCompoundFile, normGen, isCompoundFile,
+                                       null, diagnostics, attributes);
-                                       null, diagnostics, null);
     info.setFiles(files);
 
     SegmentInfoPerCommit infoPerCommit = new SegmentInfoPerCommit(info, delCount, delGen);

lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xStoredFieldsReader.java

@@ -36,7 +36,6 @@ import org.apache.lucene.store.IndexInput;
 import org.apache.lucene.util.IOUtils;
 
 import java.io.Closeable;
-import java.util.Set;
 
 /**
  * Class responsible for access to stored document fields.
@@ -139,13 +138,13 @@ final class Lucene3xStoredFieldsReader extends StoredFieldsReader implements Clo
   }
 
   public Lucene3xStoredFieldsReader(Directory d, SegmentInfo si, FieldInfos fn, IOContext context) throws IOException {
-    final String segment = si.getDocStoreSegment();
+    final String segment = Lucene3xSegmentInfoFormat.getDocStoreSegment(si);
-    final int docStoreOffset = si.getDocStoreOffset();
+    final int docStoreOffset = Lucene3xSegmentInfoFormat.getDocStoreOffset(si);
     final int size = si.getDocCount();
     boolean success = false;
     fieldInfos = fn;
     try {
-      if (docStoreOffset != -1 && si.getDocStoreIsCompoundFile()) {
+      if (docStoreOffset != -1 && Lucene3xSegmentInfoFormat.getDocStoreIsCompoundFile(si)) {
         d = storeCFSReader = new CompoundFileDirectory(si.dir, 
             IndexFileNames.segmentFileName(segment, "", Lucene3xCodec.COMPOUND_FILE_STORE_EXTENSION), context, false);
       } else {

lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xTermVectorsFormat.java

@@ -18,7 +18,6 @@ package org.apache.lucene.codecs.lucene3x;
  */
 
 import java.io.IOException;
-import java.util.Set;
 
 import org.apache.lucene.codecs.TermVectorsFormat;
 import org.apache.lucene.codecs.TermVectorsReader;
@@ -41,7 +40,7 @@ class Lucene3xTermVectorsFormat extends TermVectorsFormat {
 
   @Override
   public TermVectorsReader vectorsReader(Directory directory, SegmentInfo segmentInfo, FieldInfos fieldInfos, IOContext context) throws IOException {
-    final String fileName = IndexFileNames.segmentFileName(segmentInfo.getDocStoreSegment(), "", Lucene3xTermVectorsReader.VECTORS_FIELDS_EXTENSION);
+    final String fileName = IndexFileNames.segmentFileName(Lucene3xSegmentInfoFormat.getDocStoreSegment(segmentInfo), "", Lucene3xTermVectorsReader.VECTORS_FIELDS_EXTENSION);
 
     // Unfortunately, for 3.x indices, each segment's
     // FieldInfos can lie about hasVectors (claim it's true
@@ -49,8 +48,8 @@ class Lucene3xTermVectorsFormat extends TermVectorsFormat {
     // check if the files really exist before trying to open
     // them (4.x has fixed this):
     final boolean exists;
-    if (segmentInfo.getDocStoreOffset() != -1 && segmentInfo.getDocStoreIsCompoundFile()) {
+    if (Lucene3xSegmentInfoFormat.getDocStoreOffset(segmentInfo) != -1 && Lucene3xSegmentInfoFormat.getDocStoreIsCompoundFile(segmentInfo)) {
-      String cfxFileName = IndexFileNames.segmentFileName(segmentInfo.getDocStoreSegment(), "", Lucene3xCodec.COMPOUND_FILE_STORE_EXTENSION);
+      String cfxFileName = IndexFileNames.segmentFileName(Lucene3xSegmentInfoFormat.getDocStoreSegment(segmentInfo), "", Lucene3xCodec.COMPOUND_FILE_STORE_EXTENSION);
       if (segmentInfo.dir.fileExists(cfxFileName)) {
         Directory cfsDir = new CompoundFileDirectory(segmentInfo.dir, cfxFileName, context, false);
         try {

lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xTermVectorsReader.java

@@ -22,7 +22,6 @@ import java.util.Arrays;
 import java.util.Comparator;
 import java.util.HashMap;
 import java.util.Map;
-import java.util.Set;
 
 import org.apache.lucene.codecs.TermVectorsReader;
 import org.apache.lucene.index.CorruptIndexException;
@@ -114,14 +113,14 @@ class Lucene3xTermVectorsReader extends TermVectorsReader {
     
   public Lucene3xTermVectorsReader(Directory d, SegmentInfo si, FieldInfos fieldInfos, IOContext context)
     throws CorruptIndexException, IOException {
-    final String segment = si.getDocStoreSegment();
+    final String segment = Lucene3xSegmentInfoFormat.getDocStoreSegment(si);
-    final int docStoreOffset = si.getDocStoreOffset();
+    final int docStoreOffset = Lucene3xSegmentInfoFormat.getDocStoreOffset(si);
     final int size = si.getDocCount();
     
     boolean success = false;
 
     try {
-      if (docStoreOffset != -1 && si.getDocStoreIsCompoundFile()) {
+      if (docStoreOffset != -1 && Lucene3xSegmentInfoFormat.getDocStoreIsCompoundFile(si)) {
         d = storeCFSReader = new CompoundFileDirectory(si.dir, 
             IndexFileNames.segmentFileName(segment, "", Lucene3xCodec.COMPOUND_FILE_STORE_EXTENSION), context, false);
       } else {

lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40SegmentInfoReader.java

@@ -45,17 +45,13 @@ public class Lucene40SegmentInfoReader extends SegmentInfoReader {
     try {
       final String version = input.readString();
       final int docCount = input.readInt();
-      final int docStoreOffset = -1;
-      final String docStoreSegment = segment;
-      final boolean docStoreIsCompoundFile = false;
       final Map<Integer,Long> normGen = null;
       final boolean isCompoundFile = input.readByte() == SegmentInfo.YES;
       final Map<String,String> diagnostics = input.readStringStringMap();
       final Map<String,String> attributes = input.readStringStringMap();
       final Set<String> files = input.readStringSet();
 
-      final SegmentInfo si = new SegmentInfo(dir, version, segment, docCount, docStoreOffset,
+      final SegmentInfo si = new SegmentInfo(dir, version, segment, docCount, normGen, isCompoundFile,
-                                             docStoreSegment, docStoreIsCompoundFile, normGen, isCompoundFile,
                                              null, diagnostics, attributes);
       si.setFiles(files);
 

lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40SegmentInfoWriter.java

@@ -50,7 +50,6 @@ public class Lucene40SegmentInfoWriter extends SegmentInfoWriter {
       output.writeString(si.getVersion());
       output.writeInt(si.getDocCount());
 
-      assert si.getDocStoreOffset() == -1;
       assert si.getNormGen() == null;
 
       output.writeByte((byte) (si.getUseCompoundFile() ? SegmentInfo.YES : SegmentInfo.NO));

lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSegmentInfoReader.java

@@ -23,14 +23,9 @@ import java.util.HashSet;
 import java.util.Map;
 import java.util.Set;
 
-import org.apache.lucene.codecs.Codec;
 import org.apache.lucene.codecs.SegmentInfoReader;
-import org.apache.lucene.index.CorruptIndexException;
 import org.apache.lucene.index.IndexFileNames;
 import org.apache.lucene.index.SegmentInfo;
-import org.apache.lucene.index.SegmentInfos;
-import org.apache.lucene.store.ChecksumIndexInput;
-import org.apache.lucene.store.DataInput;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.IOContext;
 import org.apache.lucene.store.IndexInput;
@@ -111,9 +106,8 @@ public class SimpleTextSegmentInfoReader extends SegmentInfoReader {
         files.add(fileName);
       }
 
-      SegmentInfo info = new SegmentInfo(directory, version, segmentName, docCount, -1,
+      SegmentInfo info = new SegmentInfo(directory, version, segmentName, docCount, 
-                                         segmentName, false, null, isCompoundFile,
+                                         null, isCompoundFile, null, diagnostics, attributes);
-                                         null, diagnostics, attributes);
       info.setFiles(files);
       success = true;
       return info;

lucene/core/src/java/org/apache/lucene/index/CheckIndex.java

@@ -489,15 +489,7 @@ public class CheckIndex {
           msg("    diagnostics = " + diagnostics);
         }
 
-        final int docStoreOffset = info.info.getDocStoreOffset();
+        // TODO: we could append the info attributes() to the msg?
-        if (docStoreOffset != -1) {
-          msg("    docStoreOffset=" + docStoreOffset);
-          segInfoStat.docStoreOffset = docStoreOffset;
-          msg("    docStoreSegment=" + info.info.getDocStoreSegment());
-          segInfoStat.docStoreSegment = info.info.getDocStoreSegment();
-          msg("    docStoreIsCompoundFile=" + info.info.getDocStoreIsCompoundFile());
-          segInfoStat.docStoreCompoundFile = info.info.getDocStoreIsCompoundFile();
-        }
 
         if (info.hasDeletions()) {
           msg("    no deletions");

lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java

@@ -271,9 +271,7 @@ class DocumentsWriterPerThread {
   private void initSegmentInfo() {
     String segment = writer.newSegmentName();
     segmentInfo = new SegmentInfo(directoryOrig, Constants.LUCENE_MAIN_VERSION, segment, -1,
-                                  -1, segment, false, null, false,
+                                  null, false, codec, null, null);
-                                  codec,
-                                  null, null);
     assert numDocsInRAM == 0;
     if (INFO_VERBOSE && infoStream.isEnabled("DWPT")) {
       infoStream.message("DWPT", Thread.currentThread().getName() + " init seg=" + segment + " delQueue=" + deleteQueue);  

lucene/core/src/java/org/apache/lucene/index/IndexWriter.java

@@ -2218,7 +2218,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit {
           assert !infos.contains(info): "dup info dir=" + info.info.dir + " name=" + info.info.name;
 
           String newSegName = newSegmentName();
-          String dsName = info.info.getDocStoreSegment();
+          String dsName = Lucene3xSegmentInfoFormat.getDocStoreSegment(info.info);
 
           if (infoStream.isEnabled("IW")) {
             infoStream.message("IW", "addIndexes: process segment origName=" + info.info.name + " newName=" + newSegName + " dsName=" + dsName + " info=" + info);
@@ -2285,8 +2285,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit {
       TrackingDirectoryWrapper trackingDir = new TrackingDirectoryWrapper(directory);
 
       SegmentInfo info = new SegmentInfo(directory, Constants.LUCENE_MAIN_VERSION, mergedName, -1,
-                                         -1, mergedName, false, null, false,
+                                         null, false, codec, null, null);
-                                         codec, null, null);
 
       SegmentMerger merger = new SegmentMerger(info, infoStream, trackingDir, config.getTermIndexInterval(),
                                                MergeState.CheckAbort.NONE, payloadProcessorProvider,
@@ -2360,7 +2359,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit {
     // only relevant for segments that share doc store with others,
     // because the DS might have been copied already, in which case we
     // just want to update the DS name of this SegmentInfo.
-    final String dsName = info.info.getDocStoreSegment();
+    final String dsName = Lucene3xSegmentInfoFormat.getDocStoreSegment(info.info);
     assert dsName != null;
     final String newDsName;
     if (dsNames.containsKey(dsName)) {
@@ -2371,7 +2370,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit {
     }
     
     Set<String> codecDocStoreFiles = new HashSet<String>();
-    final boolean hasSharedDocStore = info.info.getDocStoreOffset() != -1;
+    final boolean hasSharedDocStore = Lucene3xSegmentInfoFormat.getDocStoreOffset(info.info) != -1;
     final String segmentInfoFileName3X = IndexFileNames.segmentFileName(info.info.name,
                                                                         "",
                                                                         Lucene3xSegmentInfoFormat.SI_EXTENSION);
@@ -2379,9 +2378,8 @@ public class IndexWriter implements Closeable, TwoPhaseCommit {
     if (hasSharedDocStore) {
       // only violate the codec this way if it's preflex &
       // shares doc stores
-      assert info.info.getDocStoreSegment() != null;
       // nocommit what to do....
-      if (info.info.getDocStoreIsCompoundFile()) {
+      if (Lucene3xSegmentInfoFormat.getDocStoreIsCompoundFile(info.info)) {
         codecDocStoreFiles.add(IndexFileNames.segmentFileName(dsName, "", "cfx"));
       } else {
         codecDocStoreFiles.add(IndexFileNames.segmentFileName(dsName, "", "fdt"));
@@ -2390,13 +2388,15 @@ public class IndexWriter implements Closeable, TwoPhaseCommit {
         codecDocStoreFiles.add(IndexFileNames.segmentFileName(dsName, "", "tvf"));
         codecDocStoreFiles.add(IndexFileNames.segmentFileName(dsName, "", "tvd"));
       }
+      // change docStoreSegment to newDsName
+      info.info.putAttribute(Lucene3xSegmentInfoFormat.DS_NAME_KEY, newDsName);
     }
 
     //System.out.println("copy seg=" + info.info.name + " version=" + info.info.getVersion());
 
     // Same SI as before but we change directory, name and docStoreSegment:
-    SegmentInfo newInfo = new SegmentInfo(directory, info.info.getVersion(), segName, info.info.getDocCount(), info.info.getDocStoreOffset(),
+    SegmentInfo newInfo = new SegmentInfo(directory, info.info.getVersion(), segName, info.info.getDocCount(),
-                                          newDsName, info.info.getDocStoreIsCompoundFile(), info.info.getNormGen(), info.info.getUseCompoundFile(),
+                                          info.info.getNormGen(), info.info.getUseCompoundFile(),
                                           info.info.getCodec(), info.info.getDiagnostics(), info.info.attributes());
     SegmentInfoPerCommit newInfoPerCommit = new SegmentInfoPerCommit(newInfo, info.getDelCount(), info.getDelGen());
 
@@ -3317,7 +3317,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit {
     // ConcurrentMergePolicy we keep deterministic segment
     // names.
     final String mergeSegmentName = newSegmentName();
-    SegmentInfo si = new SegmentInfo(directory, Constants.LUCENE_MAIN_VERSION, mergeSegmentName, -1, -1, mergeSegmentName, false, null, false, codec, details, null);
+    SegmentInfo si = new SegmentInfo(directory, Constants.LUCENE_MAIN_VERSION, mergeSegmentName, -1, null, false, codec, details, null);
     merge.info = new SegmentInfoPerCommit(si, 0, -1L);
 
     // Lock order: IW -> BD
@@ -4015,7 +4015,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit {
     if (infoStream.isEnabled("IW")) {
       infoStream.message("IW", "create compound file " + fileName);
     }
-    assert info.getDocStoreOffset() == -1;
+    assert Lucene3xSegmentInfoFormat.getDocStoreOffset(info) == -1;
     // Now merge all added files
     Collection<String> files = info.files();
     CompoundFileDirectory cfsDir = new CompoundFileDirectory(directory, fileName, context, true);

lucene/core/src/java/org/apache/lucene/index/SegmentInfo.java

@@ -57,15 +57,6 @@ public final class SegmentInfo {
 
   private volatile long sizeInBytes = -1;         // total byte size of all files (computed on demand)
 
-  //TODO: LUCENE-2555: remove once we don't need to support shared doc stores (pre 4.0)
-  private final int docStoreOffset;                     // if this segment shares stored fields & vectors, this
-                                                  // offset is where in that file this segment's docs begin
-  //TODO: LUCENE-2555: remove once we don't need to support shared doc stores (pre 4.0)
-  private final String docStoreSegment;                 // name used to derive fields/vectors file we share with
-                                                  // other segments
-  //TODO: LUCENE-2555: remove once we don't need to support shared doc stores (pre 4.0)
-  private final boolean docStoreIsCompoundFile;         // whether doc store files are stored in compound file (*.cfx)
-
   private Codec codec;
 
   private Map<String,String> diagnostics;
@@ -92,17 +83,14 @@ public final class SegmentInfo {
    * <p>Note: this is public only to allow access from
    * the codecs package.</p>
    */
-  public SegmentInfo(Directory dir, String version, String name, int docCount, int docStoreOffset,
+  public SegmentInfo(Directory dir, String version, String name, int docCount, 
-                     String docStoreSegment, boolean docStoreIsCompoundFile, Map<Integer,Long> normGen, boolean isCompoundFile,
+                     Map<Integer,Long> normGen, boolean isCompoundFile,
                      Codec codec, Map<String,String> diagnostics, Map<String,String> attributes) {
     assert !(dir instanceof TrackingDirectoryWrapper);
     this.dir = dir;
     this.version = version;
     this.name = name;
     this.docCount = docCount;
-    this.docStoreOffset = docStoreOffset;
-    this.docStoreSegment = docStoreSegment;
-    this.docStoreIsCompoundFile = docStoreIsCompoundFile;
     this.normGen = normGen;
     this.isCompoundFile = isCompoundFile;
     this.codec = codec;
@@ -163,33 +151,6 @@ public final class SegmentInfo {
     return isCompoundFile;
   }
 
-  /**
-   * @deprecated shared doc stores are not supported in >= 4.0
-   */
-  @Deprecated
-  public int getDocStoreOffset() {
-    // TODO: LUCENE-2555: remove once we don't need to support shared doc stores (pre 4.0)
-    return docStoreOffset;
-  }
-
-  /**
-   * @deprecated shared doc stores are not supported in >= 4.0
-   */
-  @Deprecated
-  public boolean getDocStoreIsCompoundFile() {
-    // TODO: LUCENE-2555: remove once we don't need to support shared doc stores (pre 4.0)
-    return docStoreIsCompoundFile;
-  }
-
-  /**
-   * @deprecated shared doc stores are not supported in >= 4.0
-   */
-  @Deprecated
-  public String getDocStoreSegment() {
-    // TODO: LUCENE-2555: remove once we don't need to support shared doc stores (pre 4.0)
-    return docStoreSegment;
-  }
-
   /** Can only be called once. */
   public void setCodec(Codec codec) {
     assert this.codec == null;
@@ -240,14 +201,12 @@ public final class SegmentInfo {
   /** Used for debugging.  Format may suddenly change.
    *
    *  <p>Current format looks like
-   *  <code>_a(3.1):c45/4->_1</code>, which means the segment's
+   *  <code>_a(3.1):c45/4</code>, which means the segment's
    *  name is <code>_a</code>; it was created with Lucene 3.1 (or
    *  '?' if it's unknown); it's using compound file
    *  format (would be <code>C</code> if not compound); it
    *  has 45 documents; it has 4 deletions (this part is
-   *  left off when there are no deletions); it's using the
+   *  left off when there are no deletions).</p>
-   *  shared doc stores named <code>_1</code> (this part is
-   *  left off if doc stores are private).</p>
    */
   public String toString(Directory dir, int delCount) {
 
@@ -265,15 +224,7 @@ public final class SegmentInfo {
       s.append('/').append(delCount);
     }
 
-    if (docStoreOffset != -1) {
+    // TODO: we could append toString of attributes() here?
-      s.append("->").append(docStoreSegment);
-      if (docStoreIsCompoundFile) {
-        s.append('c');
-      } else {
-        s.append('C');
-      }
-      s.append('+').append(docStoreOffset);
-    }
 
     return s.toString();
   }

lucene/core/src/java/org/apache/lucene/index/SegmentInfos.java

@@ -428,11 +428,7 @@ public final class SegmentInfos implements Cloneable, Iterable<SegmentInfoPerCom
       // NOTE: a lie
       output.writeLong(0L);
 
-      output.writeInt(si.getDocStoreOffset());
+      output.writeStringStringMap(si.attributes());
-      if (si.getDocStoreOffset() != -1) {
-        output.writeString(si.getDocStoreSegment());
-        output.writeByte((byte) (si.getDocStoreIsCompoundFile() ? 1:0));
-      }
       // pre-4.0 indexes write a byte if there is a single norms file
       output.writeByte((byte) 1);
 

lucene/core/src/test/org/apache/lucene/index/TestCodecs.java

@@ -256,7 +256,7 @@ public class TestCodecs extends LuceneTestCase {
     final Directory dir = newDirectory();
     this.write(fieldInfos, dir, fields, true);
     Codec codec = Codec.getDefault();
-    final SegmentInfo si = new SegmentInfo(dir, Constants.LUCENE_MAIN_VERSION, SEGMENT, 10000, -1, SEGMENT, false, null, false,
+    final SegmentInfo si = new SegmentInfo(dir, Constants.LUCENE_MAIN_VERSION, SEGMENT, 10000, null, false,
                                            codec, null, null);
 
     final FieldsProducer reader = codec.postingsFormat().fieldsProducer(new SegmentReadState(dir, si, fieldInfos, newIOContext(random()), DirectoryReader.DEFAULT_TERMS_INDEX_DIVISOR));
@@ -313,9 +313,8 @@ public class TestCodecs extends LuceneTestCase {
 
     this.write(fieldInfos, dir, fields, false);
     Codec codec = Codec.getDefault();
-    final SegmentInfo si = new SegmentInfo(dir, Constants.LUCENE_MAIN_VERSION, SEGMENT, 10000, -1,
+    final SegmentInfo si = new SegmentInfo(dir, Constants.LUCENE_MAIN_VERSION, SEGMENT, 10000,
-                                           SEGMENT, false, null, false,
+                                           null, false, codec, null, null);
-                                           codec, null, null);
 
     if (VERBOSE) {
       System.out.println("TEST: now read postings");
@@ -619,7 +618,7 @@ public class TestCodecs extends LuceneTestCase {
 
     final int termIndexInterval = _TestUtil.nextInt(random(), 13, 27);
     final Codec codec = Codec.getDefault();
-    final SegmentInfo si = new SegmentInfo(dir, Constants.LUCENE_MAIN_VERSION, SEGMENT, 10000, -1, SEGMENT, false, null, false, codec, null, null);
+    final SegmentInfo si = new SegmentInfo(dir, Constants.LUCENE_MAIN_VERSION, SEGMENT, 10000, null, false, codec, null, null);
     final SegmentWriteState state = new SegmentWriteState(InfoStream.getDefault(), dir, si, fieldInfos, termIndexInterval, null, newIOContext(random()));
 
     final FieldsConsumer consumer = codec.postingsFormat().fieldsConsumer(state);

lucene/core/src/test/org/apache/lucene/index/TestDoc.java

@@ -197,7 +197,7 @@ public class TestDoc extends LuceneTestCase {
 
       final Codec codec = Codec.getDefault();
       TrackingDirectoryWrapper trackingDir = new TrackingDirectoryWrapper(si1.info.dir);
-      final SegmentInfo si = new SegmentInfo(si1.info.dir, Constants.LUCENE_MAIN_VERSION, merged, -1, -1, merged, false, null, false, codec, null, null);
+      final SegmentInfo si = new SegmentInfo(si1.info.dir, Constants.LUCENE_MAIN_VERSION, merged, -1, null, false, codec, null, null);
 
       SegmentMerger merger = new SegmentMerger(si, InfoStream.getDefault(), trackingDir, IndexWriterConfig.DEFAULT_TERM_INDEX_INTERVAL,
                                                MergeState.CheckAbort.NONE, null, new FieldInfos.FieldNumbers(), context);
@@ -208,8 +208,8 @@ public class TestDoc extends LuceneTestCase {
       r1.close();
       r2.close();
       final SegmentInfo info = new SegmentInfo(si1.info.dir, Constants.LUCENE_MAIN_VERSION, merged,
-                                               si1.info.getDocCount() + si2.info.getDocCount(), -1, merged,
+                                               si1.info.getDocCount() + si2.info.getDocCount(),
-                                               false, null, false, codec, null, null);
+                                               null, false, codec, null, null);
       info.setFiles(new HashSet<String>(trackingDir.getCreatedFiles()));
       
       if (useCompoundFile) {

lucene/core/src/test/org/apache/lucene/index/TestSegmentMerger.java

@@ -77,7 +77,7 @@ public class TestSegmentMerger extends LuceneTestCase {
 
   public void testMerge() throws IOException {
     final Codec codec = Codec.getDefault();
-    final SegmentInfo si = new SegmentInfo(mergedDir, Constants.LUCENE_MAIN_VERSION, mergedSegment, -1, -1, mergedSegment, false, null, false, codec, null, null);
+    final SegmentInfo si = new SegmentInfo(mergedDir, Constants.LUCENE_MAIN_VERSION, mergedSegment, -1, null, false, codec, null, null);
 
     SegmentMerger merger = new SegmentMerger(si, InfoStream.getDefault(), mergedDir, IndexWriterConfig.DEFAULT_TERM_INDEX_INTERVAL,
                                              MergeState.CheckAbort.NONE, null, new FieldInfos.FieldNumbers(), newIOContext(random()));
@@ -88,8 +88,8 @@ public class TestSegmentMerger extends LuceneTestCase {
     assertTrue(docsMerged == 2);
     //Should be able to open a new SegmentReader against the new directory
     SegmentReader mergedReader = new SegmentReader(new SegmentInfoPerCommit(
-                                                         new SegmentInfo(mergedDir, Constants.LUCENE_MAIN_VERSION, mergedSegment, docsMerged, -1, mergedSegment,
+                                                         new SegmentInfo(mergedDir, Constants.LUCENE_MAIN_VERSION, mergedSegment, docsMerged,
-                                                                         false, null, false, codec, null, null),
+                                                                         null, false, codec, null, null),
                                                          0, -1L),
                                                    DirectoryReader.DEFAULT_TERMS_INDEX_DIVISOR, newIOContext(random()));
     assertTrue(mergedReader != null);

lucene/misc/src/java/org/apache/lucene/index/IndexSplitter.java

@@ -145,8 +145,8 @@ public class IndexSplitter {
       SegmentInfoPerCommit infoPerCommit = getInfo(n);
       SegmentInfo info = infoPerCommit.info;
       // Same info just changing the dir:
-      SegmentInfo newInfo = new SegmentInfo(destFSDir, info.getVersion(), info.name, info.getDocCount(), info.getDocStoreOffset(),
+      SegmentInfo newInfo = new SegmentInfo(destFSDir, info.getVersion(), info.name, info.getDocCount(), 
-                                            info.getDocStoreSegment(), info.getDocStoreIsCompoundFile(), info.getNormGen(), info.getUseCompoundFile(),
+                                            info.getNormGen(), info.getUseCompoundFile(),
                                             info.getCodec(), info.getDiagnostics(), info.attributes());
       destInfos.add(new SegmentInfoPerCommit(newInfo, infoPerCommit.getDelCount(), infoPerCommit.getDelGen()));
       // now copy files over