LUCENE-4131: add codecheader to .cfs/.cfe

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1349052 13f79535-47bb-0310-9956-ffa450edef68

lucene/core/src/java/org/apache/lucene/store/CompoundFileDirectory.java

@@ -22,6 +22,7 @@ import org.apache.lucene.codecs.LiveDocsFormat; // javadocs
 import org.apache.lucene.index.IndexFileNames;
 import org.apache.lucene.index.IndexFormatTooOldException;
 import org.apache.lucene.store.DataOutput; // javadocs
+import org.apache.lucene.util.CodecUtil; // javadocs
 import org.apache.lucene.util.IOUtils;
 
 import java.util.Collection;
@@ -51,10 +52,10 @@ import java.io.IOException;
  * </ul>
  * <p>Description:</p>
  * <ul>
- *   <li>Compound (.cfs) --&gt; FileData <sup>FileCount</sup></li>
- *   <li>Compound Entry Table (.cfe) --&gt; Version, FileCount, &lt;FileName,
+ *   <li>Compound (.cfs) --&gt; Header, FileData <sup>FileCount</sup></li>
+ *   <li>Compound Entry Table (.cfe) --&gt; Header, FileCount, &lt;FileName,
  *       DataOffset, DataLength&gt; <sup>FileCount</sup></li>
- *   <li>Version --&gt; {@link DataOutput#writeInt Int32}</li>
+ *   <li>Header --&gt; {@link CodecUtil#writeHeader CodecHeader}</li>
  *   <li>FileCount --&gt; {@link DataOutput#writeVInt VInt}</li>
  *   <li>DataOffset,DataLength --&gt; {@link DataOutput#writeLong UInt64}</li>
  *   <li>FileName --&gt; {@link DataOutput#writeString String}</li>
@@ -114,22 +115,23 @@ public final class CompoundFileDirectory extends Directory {
   /** Helper method that reads CFS entries from an input stream */
   private static final Map<String, FileEntry> readEntries(
       IndexInputSlicer handle, Directory dir, String name) throws IOException {
-    // read the first VInt. If it is negative, it's the version number
-    // otherwise it's the count (pre-3.1 indexes)
     final IndexInput stream = handle.openFullSlice();
     final Map<String, FileEntry> mapping;
     boolean success = false;
     try {
-      final int firstInt = stream.readVInt();
-      if (firstInt == CompoundFileWriter.FORMAT_CURRENT) {
+      final int firstInt = stream.readInt();
+      // NOTE: as long as we want to throw indexformattooold (vs corruptindexexception), we need
+      // to read the magic ourselves. See SegmentInfos which also has this.
+      if (firstInt == CodecUtil.CODEC_MAGIC) {
+        CodecUtil.checkHeaderNoMagic(stream, CompoundFileWriter.DATA_CODEC, 
+            CompoundFileWriter.VERSION_START, CompoundFileWriter.VERSION_START);
         IndexInput input = null;
         try {
           final String entriesFileName = IndexFileNames.segmentFileName(
                                                 IndexFileNames.stripExtension(name), "",
                                                 IndexFileNames.COMPOUND_FILE_ENTRIES_EXTENSION);
           input = dir.openInput(entriesFileName, IOContext.READONCE);
-          final int readInt = input.readInt(); // unused right now
-          assert readInt == CompoundFileWriter.ENTRY_FORMAT_CURRENT;
+          CodecUtil.checkHeader(input, CompoundFileWriter.ENTRY_CODEC, CompoundFileWriter.VERSION_START, CompoundFileWriter.VERSION_START);
           final int numEntries = input.readVInt();
           mapping = new HashMap<String, CompoundFileDirectory.FileEntry>(
               numEntries);
@@ -147,8 +149,8 @@ public final class CompoundFileDirectory extends Directory {
           IOUtils.close(input);
         }
       } else {
-        throw new IndexFormatTooOldException(stream, firstInt, 
-            CompoundFileWriter.FORMAT_CURRENT, CompoundFileWriter.FORMAT_CURRENT);
+        throw new IndexFormatTooOldException(stream, firstInt,
+            CodecUtil.CODEC_MAGIC, CodecUtil.CODEC_MAGIC);
       }
     } finally {
       if (success) {

lucene/core/src/java/org/apache/lucene/store/CompoundFileWriter.java

@@ -31,6 +31,7 @@ import java.util.concurrent.atomic.AtomicBoolean;
 
 import org.apache.lucene.index.IndexFileNames;
 import org.apache.lucene.index.MergePolicy.MergeAbortedException;
+import org.apache.lucene.util.CodecUtil;
 import org.apache.lucene.util.IOUtils;
 
 /**
@@ -70,20 +71,13 @@ final class CompoundFileWriter implements Closeable{
     Directory dir;
   }
 
-  // Before versioning started.
-  static final int FORMAT_PRE_VERSION = 0;
+  // versioning for the .cfs file
+  static final String DATA_CODEC = "CompoundFileWriterData";
+  static final int VERSION_START = 0;
+  static final int VERSION_CURRENT = VERSION_START;
 
-  // Segment name is not written in the file names.
-  static final int FORMAT_NO_SEGMENT_PREFIX = -1;
-  static final int FORMAT_APPEND_FILES = -2;
-
-  static final int ENTRY_FORMAT_CURRENT = -1;
-
-  // NOTE: if you introduce a new format, make it 1 lower
-  // than the current one, and always change this if you
-  // switch to a new format!
-  /** @lucene.internal */
-  static final int FORMAT_CURRENT = FORMAT_APPEND_FILES;
+  // versioning for the .cfe file
+  static final String ENTRY_CODEC = "CompoundFileWriterEntries";
 
   private final Directory directory;
   private final Map<String, FileEntry> entries = new HashMap<String, FileEntry>();
@@ -121,7 +115,7 @@ final class CompoundFileWriter implements Closeable{
       boolean success = false;
       try {
         dataOut = directory.createOutput(dataFileName, IOContext.DEFAULT);
-        dataOut.writeVInt(FORMAT_CURRENT);
+        CodecUtil.writeHeader(dataOut, DATA_CODEC, VERSION_CURRENT);
         success = true;
       } finally {
         if (!success) {
@@ -228,7 +222,7 @@ final class CompoundFileWriter implements Closeable{
 
   protected void writeEntryTable(Collection<FileEntry> entries,
       IndexOutput entryOut) throws IOException {
-    entryOut.writeInt(ENTRY_FORMAT_CURRENT);
+    CodecUtil.writeHeader(entryOut, ENTRY_CODEC, VERSION_CURRENT);
     entryOut.writeVInt(entries.size());
     for (FileEntry fe : entries) {
       entryOut.writeString(IndexFileNames.stripSegmentName(fe.file));

lucene/core/src/test/org/apache/lucene/codecs/lucene40/TestAllFilesHaveCodecHeader.java

@@ -72,10 +72,6 @@ public class TestAllFilesHaveCodecHeader extends LuceneTestCase {
         CompoundFileDirectory cfsDir = new CompoundFileDirectory(dir, file, newIOContext(random()), false);
         checkHeaders(cfsDir); // recurse into cfs
         cfsDir.close();
-        continue; // .cfs has its own header... would be nice to fix
-      }
-      if (file.endsWith(IndexFileNames.COMPOUND_FILE_ENTRIES_EXTENSION)) {
-        continue; // .cfe has its own header... would be nice to fix
       }
       IndexInput in = null;
       boolean success = false;