From 2eef957d3686ffadbb6b235aa9201b7c941cbbf0 Mon Sep 17 00:00:00 2001 From: Michael Stack Date: Wed, 7 May 2008 04:17:09 +0000 Subject: [PATCH] HBASE-47 option to set TTL for columns in hbase git-svn-id: https://svn.apache.org/repos/asf/hadoop/hbase/trunk@653986 13f79535-47bb-0310-9956-ffa450edef68 --- CHANGES.txt | 2 + .../hadoop/hbase/HColumnDescriptor.java | 59 ++- .../org/apache/hadoop/hbase/HConstants.java | 5 + .../apache/hadoop/hbase/HTableDescriptor.java | 4 +- .../apache/hadoop/hbase/hql/AlterCommand.java | 5 +- .../org/apache/hadoop/hbase/hql/HQLParser.jj | 7 + .../apache/hadoop/hbase/hql/HelpCommand.java | 2 +- .../hbase/hql/SchemaModificationCommand.java | 6 +- .../hadoop/hbase/hql/generated/HQLParser.java | 45 ++- .../hql/generated/HQLParserConstants.java | 32 +- .../hql/generated/HQLParserTokenManager.java | 364 +++++++++--------- .../hadoop/hbase/regionserver/HStore.java | 130 ++++++- .../hadoop/hbase/regionserver/Memcache.java | 121 +++++- .../hbase/regionserver/StoreFileScanner.java | 34 +- .../apache/hadoop/hbase/thrift/Hbase.thrift | 3 +- .../hadoop/hbase/thrift/ThriftUtilities.java | 2 +- .../thrift/generated/ColumnDescriptor.java | 25 +- .../apache/hadoop/hbase/HBaseTestCase.java | 9 +- .../apache/hadoop/hbase/TestBloomFilters.java | 2 + .../org/apache/hadoop/hbase/TestToString.java | 5 +- .../hbase/regionserver/TestTimestamp.java | 3 +- 21 files changed, 585 insertions(+), 280 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 57d75833ba2..91eba96b717 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -39,6 +39,8 @@ Hbase Change Log is subclassable) (David Alves via Stack) HBASE-603 When an exception bubbles out of getRegionServerWithRetries, wrap the exception with a RetriesExhaustedException + HBASE-47 Option to set TTL for columns in hbase + (Andrew Purtell via Bryan Duxbury and Stack) Release 0.1.1 - 04/11/2008 diff --git a/src/java/org/apache/hadoop/hbase/HColumnDescriptor.java b/src/java/org/apache/hadoop/hbase/HColumnDescriptor.java index faf50632635..b524fd8ccc2 100644 --- a/src/java/org/apache/hadoop/hbase/HColumnDescriptor.java +++ b/src/java/org/apache/hadoop/hbase/HColumnDescriptor.java @@ -42,7 +42,7 @@ import org.apache.hadoop.hbase.io.TextSequence; public class HColumnDescriptor implements WritableComparable { // For future backward compatibility - private static final byte COLUMN_DESCRIPTOR_VERSION = (byte)2; + private static final byte COLUMN_DESCRIPTOR_VERSION = (byte)3; /** Legal family names can only contain 'word characters' and end in a colon. */ public static final Pattern LEGAL_FAMILY_NAME = Pattern.compile("\\w+:"); @@ -85,7 +85,12 @@ public class HColumnDescriptor implements WritableComparable { * Default maximum length of cell contents. */ public static final int DEFAULT_MAX_VALUE_LENGTH = Integer.MAX_VALUE; - + + /** + * Default time to live of cell contents. + */ + public static final int DEFAULT_TIME_TO_LIVE = HConstants.FOREVER; + /** * Default bloom filter description. */ @@ -104,6 +109,8 @@ public class HColumnDescriptor implements WritableComparable { private boolean blockCacheEnabled; // Maximum value size private int maxValueLength; + // Time to live of cell contents, in seconds from last timestamp + private int timeToLive; // True if bloom filter was specified private boolean bloomFilterSpecified; // Descriptor of bloom filter @@ -131,7 +138,8 @@ public class HColumnDescriptor implements WritableComparable { new Text(): new Text(columnName), DEFAULT_N_VERSIONS, DEFAULT_COMPRESSION_TYPE, DEFAULT_IN_MEMORY, DEFAULT_BLOCK_CACHE_ENABLED, - Integer.MAX_VALUE, DEFAULT_BLOOM_FILTER_DESCRIPTOR); + Integer.MAX_VALUE, DEFAULT_TIME_TO_LIVE, + DEFAULT_BLOOM_FILTER_DESCRIPTOR); } /** @@ -144,6 +152,8 @@ public class HColumnDescriptor implements WritableComparable { * cache * @param blockCacheEnabled If true, MapFile blocks should be cached * @param maxValueLength Restrict values to <= this value + * @param timeToLive Time-to-live of cell contents, in seconds from last timestamp + * (use HConstants.FOREVER for unlimited TTL) * @param bloomFilter Enable the specified bloom filter for this column * * @throws IllegalArgumentException if passed a family name that is made of @@ -154,7 +164,8 @@ public class HColumnDescriptor implements WritableComparable { public HColumnDescriptor(final Text name, final int maxVersions, final CompressionType compression, final boolean inMemory, final boolean blockCacheEnabled, - final int maxValueLength, final BloomFilterDescriptor bloomFilter) { + final int maxValueLength, final int timeToLive, + final BloomFilterDescriptor bloomFilter) { String familyStr = name.toString(); // Test name if not null (It can be null when deserializing after // construction but before we've read in the fields); @@ -177,12 +188,13 @@ public class HColumnDescriptor implements WritableComparable { this.inMemory = inMemory; this.blockCacheEnabled = blockCacheEnabled; this.maxValueLength = maxValueLength; + this.timeToLive = timeToLive; this.bloomFilter = bloomFilter; this.bloomFilterSpecified = this.bloomFilter == null ? false : true; this.versionNumber = COLUMN_DESCRIPTOR_VERSION; this.compressionType = compression; } - + /** @return name of column family */ public Text getName() { return name; @@ -229,7 +241,14 @@ public class HColumnDescriptor implements WritableComparable { public int getMaxValueLength() { return this.maxValueLength; } - + + /** + * @return Time to live. + */ + public int getTimeToLive() { + return this.timeToLive; + } + /** * @return True if MapFile blocks should be cached. */ @@ -253,8 +272,13 @@ public class HColumnDescriptor implements WritableComparable { ", max versions: " + maxVersions + ", compression: " + this.compressionType + ", in memory: " + inMemory + ", block cache enabled: " + blockCacheEnabled + - ", max length: " + maxValueLength + ", bloom filter: " + - (bloomFilterSpecified ? bloomFilter.toString() : "none") + "}"; + ", max length: " + maxValueLength + + ", time to live: " + + (timeToLive == HConstants.FOREVER ? "FOREVER" : + Integer.toString(timeToLive)) + + ", bloom filter: " + + (bloomFilterSpecified ? bloomFilter.toString() : "none") + + "}"; } /** {@inheritDoc} */ @@ -272,6 +296,7 @@ public class HColumnDescriptor implements WritableComparable { result ^= Boolean.valueOf(this.inMemory).hashCode(); result ^= Boolean.valueOf(this.blockCacheEnabled).hashCode(); result ^= Integer.valueOf(this.maxValueLength).hashCode(); + result ^= Integer.valueOf(this.timeToLive).hashCode(); result ^= Boolean.valueOf(this.bloomFilterSpecified).hashCode(); result ^= Byte.valueOf(this.versionNumber).hashCode(); if(this.bloomFilterSpecified) { @@ -301,6 +326,10 @@ public class HColumnDescriptor implements WritableComparable { if (this.versionNumber > 1) { this.blockCacheEnabled = in.readBoolean(); } + + if (this.versionNumber > 2) { + this.timeToLive = in.readInt(); + } } /** {@inheritDoc} */ @@ -317,7 +346,13 @@ public class HColumnDescriptor implements WritableComparable { bloomFilter.write(out); } - out.writeBoolean(this.blockCacheEnabled); + if (this.versionNumber > 1) { + out.writeBoolean(this.blockCacheEnabled); + } + + if (this.versionNumber > 2) { + out.writeInt(this.timeToLive); + } } // Comparable @@ -368,7 +403,11 @@ public class HColumnDescriptor implements WritableComparable { if(result == 0) { result = other.maxValueLength - this.maxValueLength; } - + + if(result == 0) { + result = other.timeToLive - this.timeToLive; + } + if(result == 0) { if(this.bloomFilterSpecified == other.bloomFilterSpecified) { result = 0; diff --git a/src/java/org/apache/hadoop/hbase/HConstants.java b/src/java/org/apache/hadoop/hbase/HConstants.java index e78501b6b83..fef78c53bcd 100644 --- a/src/java/org/apache/hadoop/hbase/HConstants.java +++ b/src/java/org/apache/hadoop/hbase/HConstants.java @@ -188,4 +188,9 @@ public interface HConstants { * Define for 'return-all-versions'. */ static final int ALL_VERSIONS = -1; + + /** + * Unlimited time-to-live. + */ + static final int FOREVER = -1; } diff --git a/src/java/org/apache/hadoop/hbase/HTableDescriptor.java b/src/java/org/apache/hadoop/hbase/HTableDescriptor.java index 50cf553b8b8..22b89551a04 100644 --- a/src/java/org/apache/hadoop/hbase/HTableDescriptor.java +++ b/src/java/org/apache/hadoop/hbase/HTableDescriptor.java @@ -44,14 +44,14 @@ public class HTableDescriptor implements WritableComparable { new HTableDescriptor(HConstants.ROOT_TABLE_NAME, new HColumnDescriptor(HConstants.COLUMN_FAMILY, 1, HColumnDescriptor.CompressionType.NONE, false, false, - Integer.MAX_VALUE, null)); + Integer.MAX_VALUE, HConstants.FOREVER, null)); /** table descriptor for meta table */ public static final HTableDescriptor metaTableDesc = new HTableDescriptor(HConstants.META_TABLE_NAME, new HColumnDescriptor(HConstants.COLUMN_FAMILY, 1, HColumnDescriptor.CompressionType.NONE, false, false, - Integer.MAX_VALUE, null)); + Integer.MAX_VALUE, HConstants.FOREVER, null)); private boolean rootregion; private boolean metaregion; diff --git a/src/java/org/apache/hadoop/hbase/hql/AlterCommand.java b/src/java/org/apache/hadoop/hbase/hql/AlterCommand.java index bfdeee55d2e..2674c0a030d 100644 --- a/src/java/org/apache/hadoop/hbase/hql/AlterCommand.java +++ b/src/java/org/apache/hadoop/hbase/hql/AlterCommand.java @@ -213,6 +213,8 @@ public class AlterCommand extends SchemaModificationCommand { numHash = (Integer) columnSpec.get(spec); } else if (spec.equals("NUM_ENTRIES")) { numEntries = (Integer) columnSpec.get(spec); + } else if (spec.equals("TTL")) { + timeToLive = (Integer) columnSpec.get(spec); } else { throw new IllegalArgumentException("Invalid option: " + spec); } @@ -232,7 +234,7 @@ public class AlterCommand extends SchemaModificationCommand { HColumnDescriptor columnDesc = new HColumnDescriptor(new Text(column), maxVersions, compression, inMemory, blockCacheEnabled, - maxLength, bloomFilterDesc); + maxLength, timeToLive, bloomFilterDesc); return columnDesc; } @@ -248,5 +250,6 @@ public class AlterCommand extends SchemaModificationCommand { inMemory = original.isInMemory(); blockCacheEnabled = original.isBlockCacheEnabled(); bloomFilterDesc = original.getBloomFilter(); + timeToLive = original.getTimeToLive(); } } diff --git a/src/java/org/apache/hadoop/hbase/hql/HQLParser.jj b/src/java/org/apache/hadoop/hbase/hql/HQLParser.jj index 8d6ee8582bd..8804995d78e 100644 --- a/src/java/org/apache/hadoop/hbase/hql/HQLParser.jj +++ b/src/java/org/apache/hadoop/hbase/hql/HQLParser.jj @@ -119,6 +119,7 @@ TOKEN: /** for HQL statements */ | | | + | | | | @@ -359,6 +360,12 @@ Map ColumnSpec() : { columnSpec.put("BLOCK_CACHE_ENABLED", true); } + | + + n = number() + { + columnSpec.put("TTL", n); + } | diff --git a/src/java/org/apache/hadoop/hbase/hql/HelpCommand.java b/src/java/org/apache/hadoop/hbase/hql/HelpCommand.java index 3ab653ae0c3..0a8d6d06583 100644 --- a/src/java/org/apache/hadoop/hbase/hql/HelpCommand.java +++ b/src/java/org/apache/hadoop/hbase/hql/HelpCommand.java @@ -87,7 +87,7 @@ public class HelpCommand extends BasicCommand { new String[] { "Create tables", "CREATE TABLE table_name (column_family_name [MAX_VERSIONS=n] " - + "[MAX_LENGTH=n] [COMPRESSION=NONE|RECORD|BLOCK] [IN_MEMORY] " + + "[MAX_LENGTH=n] [COMPRESSION=NONE|RECORD|BLOCK] [IN_MEMORY] [TTL=n]" + "[BLOOMFILTER=NONE|BLOOMFILTER|COUNTING_BLOOMFILTER|RETOUCHED_BLOOMFILTER " + "VECTOR_SIZE=n NUM_HASH=n], " + "...)" }); load.put("DROP", new String[] { "Drop tables", diff --git a/src/java/org/apache/hadoop/hbase/hql/SchemaModificationCommand.java b/src/java/org/apache/hadoop/hbase/hql/SchemaModificationCommand.java index b8384545413..c6d2286ed1b 100644 --- a/src/java/org/apache/hadoop/hbase/hql/SchemaModificationCommand.java +++ b/src/java/org/apache/hadoop/hbase/hql/SchemaModificationCommand.java @@ -43,6 +43,7 @@ public abstract class SchemaModificationCommand extends BasicCommand { protected int vectorSize; protected int numHash; protected int numEntries; + protected int timeToLive; public SchemaModificationCommand(Writer o) { super(o); @@ -55,6 +56,7 @@ public abstract class SchemaModificationCommand extends BasicCommand { inMemory = HColumnDescriptor.DEFAULT_IN_MEMORY; blockCacheEnabled = HColumnDescriptor.DEFAULT_BLOCK_CACHE_ENABLED; bloomFilterDesc = HColumnDescriptor.DEFAULT_BLOOM_FILTER_DESCRIPTOR; + timeToLive = HColumnDescriptor.DEFAULT_TIME_TO_LIVE; } /** @@ -89,6 +91,8 @@ public abstract class SchemaModificationCommand extends BasicCommand { numHash = (Integer) columnSpec.get(spec); } else if (spec.equals("NUM_ENTRIES")) { numEntries = (Integer) columnSpec.get(spec); + } else if (spec.equals("TTL")) { + timeToLive = (Integer) columnSpec.get(spec); } else { throw new IllegalArgumentException("Invalid option: " + spec); } @@ -108,7 +112,7 @@ public abstract class SchemaModificationCommand extends BasicCommand { HColumnDescriptor columnDesc = new HColumnDescriptor(new Text(column), maxVersions, compression, inMemory, blockCacheEnabled, - maxLength, bloomFilterDesc); + maxLength, timeToLive, bloomFilterDesc); return columnDesc; } diff --git a/src/java/org/apache/hadoop/hbase/hql/generated/HQLParser.java b/src/java/org/apache/hadoop/hbase/hql/generated/HQLParser.java index 56717713347..bf2d224b60b 100644 --- a/src/java/org/apache/hadoop/hbase/hql/generated/HQLParser.java +++ b/src/java/org/apache/hadoop/hbase/hql/generated/HQLParser.java @@ -75,7 +75,7 @@ public class HQLParser implements HQLParserConstants { case SELECT: case ENABLE: case DISABLE: - case 69: + case 70: switch ((jj_ntk==-1)?jj_ntk():jj_ntk) { case HELP: case ALTER: @@ -100,7 +100,7 @@ public class HQLParser implements HQLParserConstants { jj_la1[0] = jj_gen; ; } - jj_consume_token(69); + jj_consume_token(70); break; case 0: jj_consume_token(0); @@ -391,6 +391,7 @@ public class HQLParser implements HQLParserConstants { case COMPRESSION: case IN_MEMORY: case BLOCK_CACHE_ENABLED: + case TTL: case BLOOMFILTER: case VECTOR_SIZE: case NUM_HASH: @@ -445,6 +446,12 @@ public class HQLParser implements HQLParserConstants { jj_consume_token(BLOCK_CACHE_ENABLED); columnSpec.put("BLOCK_CACHE_ENABLED", true); break; + case TTL: + jj_consume_token(TTL); + jj_consume_token(EQUALS); + n = number(); + columnSpec.put("TTL", n); + break; case BLOOMFILTER: jj_consume_token(BLOOMFILTER); jj_consume_token(EQUALS); @@ -1085,6 +1092,16 @@ public class HQLParser implements HQLParserConstants { finally { jj_save(0, xla); } } + final private boolean jj_3R_10() { + Token xsp; + xsp = jj_scanpos; + if (jj_3R_11()) { + jj_scanpos = xsp; + if (jj_3R_12()) return true; + } + return false; + } + final private boolean jj_3_1() { if (jj_scan_token(ADD)) return true; if (jj_3R_10()) return true; @@ -1094,9 +1111,9 @@ public class HQLParser implements HQLParserConstants { final private boolean jj_3R_12() { Token xsp; xsp = jj_scanpos; - if (jj_scan_token(67)) { + if (jj_scan_token(68)) { jj_scanpos = xsp; - if (jj_scan_token(68)) return true; + if (jj_scan_token(69)) return true; } return false; } @@ -1106,16 +1123,6 @@ public class HQLParser implements HQLParserConstants { return false; } - final private boolean jj_3R_10() { - Token xsp; - xsp = jj_scanpos; - if (jj_3R_11()) { - jj_scanpos = xsp; - if (jj_3R_12()) return true; - } - return false; - } - public HQLParserTokenManager token_source; SimpleCharStream jj_input_stream; public Token token, jj_nt; @@ -1138,10 +1145,10 @@ public class HQLParser implements HQLParserConstants { jj_la1_0 = new int[] {0xf3ffe0,0xf3ffe1,0xf3ffe0,0x0,0x0,0x0,0x0,0x33dbc0,0x33dbc0,0x0,0x600,0x0,0x0,0x0,0x0,0x0,0x0,0x1000,0x0,0x80000000,0x0,0x2000000,0x0,0x3000000,0x8000000,0x3000000,0x80000000,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,}; } private static void jj_la1_1() { - jj_la1_1 = new int[] {0x0,0x0,0x0,0x80000000,0x80000000,0x80000000,0x80000000,0x80000000,0x80000000,0x80000000,0x0,0xe71c000,0xe0000,0x1c00000,0xe71c000,0x10,0x10,0x30000000,0x0,0x0,0x0,0x0,0xc0002000,0x0,0x0,0x0,0x0,0x1,0x2,0x10,0x0,0x80002000,0x80002000,0x80002000,0x0,0x80002000,0x10,0x10,0x10,0x80000000,0x0,0x80000000,}; + jj_la1_1 = new int[] {0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x1cf1c000,0xe0000,0x3800000,0x1cf1c000,0x10,0x10,0x60000000,0x0,0x0,0x0,0x0,0x80002000,0x0,0x0,0x0,0x0,0x1,0x2,0x10,0x0,0x2000,0x2000,0x2000,0x0,0x2000,0x10,0x10,0x10,0x0,0x0,0x0,}; } private static void jj_la1_2() { - jj_la1_2 = new int[] {0x0,0x20,0x0,0x0,0x3,0x3,0x18,0x0,0x0,0x18,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x18,0x0,0x18,0x0,0x19,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x18,0x1,0x19,0x1,0x18,0x19,0x0,0x0,0x0,0x1,0x18,0x18,}; + jj_la1_2 = new int[] {0x0,0x40,0x0,0x1,0x7,0x7,0x31,0x1,0x1,0x31,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x30,0x0,0x30,0x0,0x33,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x30,0x3,0x33,0x3,0x30,0x33,0x0,0x0,0x0,0x3,0x30,0x31,}; } final private JJCalls[] jj_2_rtns = new JJCalls[1]; private boolean jj_rescan = false; @@ -1318,8 +1325,8 @@ public class HQLParser implements HQLParserConstants { public ParseException generateParseException() { jj_expentries.removeAllElements(); - boolean[] la1tokens = new boolean[70]; - for (int i = 0; i < 70; i++) { + boolean[] la1tokens = new boolean[71]; + for (int i = 0; i < 71; i++) { la1tokens[i] = false; } if (jj_kind >= 0) { @@ -1341,7 +1348,7 @@ public class HQLParser implements HQLParserConstants { } } } - for (int i = 0; i < 70; i++) { + for (int i = 0; i < 71; i++) { if (la1tokens[i]) { jj_expentry = new int[1]; jj_expentry[0] = i; diff --git a/src/java/org/apache/hadoop/hbase/hql/generated/HQLParserConstants.java b/src/java/org/apache/hadoop/hbase/hql/generated/HQLParserConstants.java index 39663ec4c4b..ecffb3dd1a7 100644 --- a/src/java/org/apache/hadoop/hbase/hql/generated/HQLParserConstants.java +++ b/src/java/org/apache/hadoop/hbase/hql/generated/HQLParserConstants.java @@ -53,21 +53,22 @@ public interface HQLParserConstants { int RECORD = 51; int IN_MEMORY = 52; int BLOCK_CACHE_ENABLED = 53; - int BLOOMFILTER = 54; - int COUNTING_BLOOMFILTER = 55; - int RETOUCHED_BLOOMFILTER = 56; - int VECTOR_SIZE = 57; - int NUM_HASH = 58; - int NUM_ENTRIES = 59; - int ADD = 60; - int CHANGE = 61; - int COUNT = 62; - int ID = 63; - int INTEGER_LITERAL = 64; - int FLOATING_POINT_LITERAL = 65; - int EXPONENT = 66; - int QUOTED_IDENTIFIER = 67; - int STRING_LITERAL = 68; + int TTL = 54; + int BLOOMFILTER = 55; + int COUNTING_BLOOMFILTER = 56; + int RETOUCHED_BLOOMFILTER = 57; + int VECTOR_SIZE = 58; + int NUM_HASH = 59; + int NUM_ENTRIES = 60; + int ADD = 61; + int CHANGE = 62; + int COUNT = 63; + int ID = 64; + int INTEGER_LITERAL = 65; + int FLOATING_POINT_LITERAL = 66; + int EXPONENT = 67; + int QUOTED_IDENTIFIER = 68; + int STRING_LITERAL = 69; int DEFAULT = 0; @@ -126,6 +127,7 @@ public interface HQLParserConstants { "\"record\"", "\"in_memory\"", "\"block_cache_enabled\"", + "\"ttl\"", "\"bloomfilter\"", "\"counting_bloomfilter\"", "\"retouched_bloomfilter\"", diff --git a/src/java/org/apache/hadoop/hbase/hql/generated/HQLParserTokenManager.java b/src/java/org/apache/hadoop/hbase/hql/generated/HQLParserTokenManager.java index 14b8c371cf5..fb97bcd7332 100644 --- a/src/java/org/apache/hadoop/hbase/hql/generated/HQLParserTokenManager.java +++ b/src/java/org/apache/hadoop/hbase/hql/generated/HQLParserTokenManager.java @@ -40,95 +40,95 @@ private final int jjStopStringLiteralDfa_0(int pos, long active0, long active1) switch (pos) { case 0: - if ((active0 & 0x7fffcc0fffffffe0L) != 0L) + if ((active0 & 0xffffcc0fffffffe0L) != 0L) { - jjmatchedKind = 63; + jjmatchedKind = 64; return 1; } return -1; case 1: - if ((active0 & 0x10080800064000L) != 0L) - return 1; - if ((active0 & 0x7fefc407fff9bfe0L) != 0L) + if ((active0 & 0xffefc407fff9bfe0L) != 0L) { if (jjmatchedPos != 1) { - jjmatchedKind = 63; + jjmatchedKind = 64; jjmatchedPos = 1; } return 1; } + if ((active0 & 0x10080800064000L) != 0L) + return 1; return -1; case 2: - if ((active0 & 0x1000040410008000L) != 0L) + if ((active0 & 0x2040040410008000L) != 0L) return 1; - if ((active0 & 0x6fffc003efff3fe0L) != 0L) + if ((active0 & 0xdfbfc003efff3fe0L) != 0L) { - jjmatchedKind = 63; + jjmatchedKind = 64; jjmatchedPos = 2; return 1; } return -1; case 3: - if ((active0 & 0x6ffdc003ebfa28c0L) != 0L) + if ((active0 & 0x2000004051720L) != 0L) + return 1; + if ((active0 & 0xdfbdc003ebfa28c0L) != 0L) { if (jjmatchedPos != 3) { - jjmatchedKind = 63; + jjmatchedKind = 64; jjmatchedPos = 3; } return 1; } - if ((active0 & 0x2000004051720L) != 0L) - return 1; return -1; case 4: - if ((active0 & 0x2f59c001e1f22a00L) != 0L) + if ((active0 & 0x812400020a0800c0L) != 0L) + return 1; + if ((active0 & 0x5e99c001e1f22a00L) != 0L) { if (jjmatchedPos != 4) { - jjmatchedKind = 63; + jjmatchedKind = 64; jjmatchedPos = 4; } return 1; } - if ((active0 & 0x40a400020a0800c0L) != 0L) - return 1; return -1; case 5: - if ((active0 & 0xff1c001c1802200L) != 0L) + if ((active0 & 0x4008000020720800L) != 0L) + return 1; + if ((active0 & 0x1fb1c001c1802200L) != 0L) { - jjmatchedKind = 63; + jjmatchedKind = 64; jjmatchedPos = 5; return 1; } - if ((active0 & 0x2008000020720800L) != 0L) - return 1; return -1; case 6: - if ((active0 & 0xff1c001c1002200L) != 0L) + if ((active0 & 0x800000L) != 0L) + return 1; + if ((active0 & 0x1fb1c001c1002200L) != 0L) { - jjmatchedKind = 63; + jjmatchedKind = 64; jjmatchedPos = 6; return 1; } - if ((active0 & 0x800000L) != 0L) - return 1; return -1; case 7: - if ((active0 & 0xbf1c001c0000000L) != 0L) + if ((active0 & 0x17b1c001c0000000L) != 0L) { - jjmatchedKind = 63; + jjmatchedKind = 64; jjmatchedPos = 7; return 1; } - if ((active0 & 0x400000001002200L) != 0L) + if ((active0 & 0x800000001002200L) != 0L) return 1; return -1; case 8: - if ((active0 & 0xbe1c00140000000L) != 0L) + if ((active0 & 0x17a1c00140000000L) != 0L) { - jjmatchedKind = 63; + jjmatchedKind = 64; jjmatchedPos = 8; return 1; } @@ -136,21 +136,21 @@ private final int jjStopStringLiteralDfa_0(int pos, long active0, long active1) return 1; return -1; case 9: - if ((active0 & 0x800000000000L) != 0L) - return 1; - if ((active0 & 0xbe1400140000000L) != 0L) + if ((active0 & 0x17a1400140000000L) != 0L) { - jjmatchedKind = 63; + jjmatchedKind = 64; jjmatchedPos = 9; return 1; } + if ((active0 & 0x800000000000L) != 0L) + return 1; return -1; case 10: - if ((active0 & 0xa41000000000000L) != 0L) + if ((active0 & 0x1481000000000000L) != 0L) return 1; - if ((active0 & 0x1a0400140000000L) != 0L) + if ((active0 & 0x320400140000000L) != 0L) { - jjmatchedKind = 63; + jjmatchedKind = 64; jjmatchedPos = 10; return 1; } @@ -158,17 +158,17 @@ private final int jjStopStringLiteralDfa_0(int pos, long active0, long active1) case 11: if ((active0 & 0x400100000000L) != 0L) return 1; - if ((active0 & 0x1a0000040000000L) != 0L) + if ((active0 & 0x320000040000000L) != 0L) { - jjmatchedKind = 63; + jjmatchedKind = 64; jjmatchedPos = 11; return 1; } return -1; case 12: - if ((active0 & 0x1a0000040000000L) != 0L) + if ((active0 & 0x320000040000000L) != 0L) { - jjmatchedKind = 63; + jjmatchedKind = 64; jjmatchedPos = 12; return 1; } @@ -176,49 +176,49 @@ private final int jjStopStringLiteralDfa_0(int pos, long active0, long active1) case 13: if ((active0 & 0x40000000L) != 0L) return 1; - if ((active0 & 0x1a0000000000000L) != 0L) + if ((active0 & 0x320000000000000L) != 0L) { - jjmatchedKind = 63; + jjmatchedKind = 64; jjmatchedPos = 13; return 1; } return -1; case 14: - if ((active0 & 0x1a0000000000000L) != 0L) + if ((active0 & 0x320000000000000L) != 0L) { - jjmatchedKind = 63; + jjmatchedKind = 64; jjmatchedPos = 14; return 1; } return -1; case 15: - if ((active0 & 0x1a0000000000000L) != 0L) + if ((active0 & 0x320000000000000L) != 0L) { - jjmatchedKind = 63; + jjmatchedKind = 64; jjmatchedPos = 15; return 1; } return -1; case 16: - if ((active0 & 0x1a0000000000000L) != 0L) + if ((active0 & 0x320000000000000L) != 0L) { - jjmatchedKind = 63; + jjmatchedKind = 64; jjmatchedPos = 16; return 1; } return -1; case 17: - if ((active0 & 0x1a0000000000000L) != 0L) + if ((active0 & 0x320000000000000L) != 0L) { - jjmatchedKind = 63; + jjmatchedKind = 64; jjmatchedPos = 17; return 1; } return -1; case 18: - if ((active0 & 0x180000000000000L) != 0L) + if ((active0 & 0x300000000000000L) != 0L) { - jjmatchedKind = 63; + jjmatchedKind = 64; jjmatchedPos = 18; return 1; } @@ -226,13 +226,13 @@ private final int jjStopStringLiteralDfa_0(int pos, long active0, long active1) return 1; return -1; case 19: - if ((active0 & 0x100000000000000L) != 0L) + if ((active0 & 0x200000000000000L) != 0L) { - jjmatchedKind = 63; + jjmatchedKind = 64; jjmatchedPos = 19; return 1; } - if ((active0 & 0x80000000000000L) != 0L) + if ((active0 & 0x100000000000000L) != 0L) return 1; return -1; default : @@ -272,7 +272,7 @@ private final int jjMoveStringLiteralDfa0_0() case 44: return jjStopAtPos(0, 36); case 59: - return jjStopAtPos(0, 69); + return jjStopAtPos(0, 70); case 60: return jjStopAtPos(0, 41); case 61: @@ -281,13 +281,13 @@ private final int jjMoveStringLiteralDfa0_0() return jjStopAtPos(0, 40); case 65: case 97: - return jjMoveStringLiteralDfa1_0(0x1000000400000040L); + return jjMoveStringLiteralDfa1_0(0x2000000400000040L); case 66: case 98: - return jjMoveStringLiteralDfa1_0(0x64000000000000L); + return jjMoveStringLiteralDfa1_0(0xa4000000000000L); case 67: case 99: - return jjMoveStringLiteralDfa1_0(0x6081000040000880L); + return jjMoveStringLiteralDfa1_0(0xc101000040000880L); case 68: case 100: return jjMoveStringLiteralDfa1_0(0x901600L); @@ -314,25 +314,25 @@ private final int jjMoveStringLiteralDfa0_0() return jjMoveStringLiteralDfa1_0(0xc00000000000L); case 78: case 110: - return jjMoveStringLiteralDfa1_0(0xc02040100000000L); + return jjMoveStringLiteralDfa1_0(0x1802040100000000L); case 79: case 111: return jjMoveStringLiteralDfa1_0(0x800000000L); case 82: case 114: - return jjMoveStringLiteralDfa1_0(0x108000010000000L); + return jjMoveStringLiteralDfa1_0(0x208000010000000L); case 83: case 115: return jjMoveStringLiteralDfa1_0(0x1200100L); case 84: case 116: - return jjMoveStringLiteralDfa1_0(0x80082000L); + return jjMoveStringLiteralDfa1_0(0x40000080082000L); case 85: case 117: return jjMoveStringLiteralDfa1_0(0x8000000L); case 86: case 118: - return jjMoveStringLiteralDfa1_0(0x200000020000000L); + return jjMoveStringLiteralDfa1_0(0x400000020000000L); case 87: case 119: return jjMoveStringLiteralDfa1_0(0x2000000L); @@ -358,19 +358,19 @@ private final int jjMoveStringLiteralDfa1_0(long active0) return jjMoveStringLiteralDfa2_0(active0, 0xc00020088000L); case 68: case 100: - return jjMoveStringLiteralDfa2_0(active0, 0x1000000000000000L); + return jjMoveStringLiteralDfa2_0(active0, 0x2000000000000000L); case 69: case 101: - return jjMoveStringLiteralDfa2_0(active0, 0x308000000300620L); + return jjMoveStringLiteralDfa2_0(active0, 0x608000000300620L); case 72: case 104: - return jjMoveStringLiteralDfa2_0(active0, 0x2000000002000100L); + return jjMoveStringLiteralDfa2_0(active0, 0x4000000002000100L); case 73: case 105: return jjMoveStringLiteralDfa2_0(active0, 0x280800000L); case 76: case 108: - return jjMoveStringLiteralDfa2_0(active0, 0x640000000000c0L); + return jjMoveStringLiteralDfa2_0(active0, 0xa40000000000c0L); case 78: case 110: if ((active0 & 0x80000000000L) != 0L) @@ -381,7 +381,7 @@ private final int jjMoveStringLiteralDfa1_0(long active0) return jjMoveStringLiteralDfa2_0(active0, 0x10000408460000L); case 79: case 111: - return jjMoveStringLiteralDfa2_0(active0, 0x4083040050000000L); + return jjMoveStringLiteralDfa2_0(active0, 0x8103040050000000L); case 82: case 114: if ((active0 & 0x800000000L) != 0L) @@ -394,10 +394,10 @@ private final int jjMoveStringLiteralDfa1_0(long active0) break; case 84: case 116: - return jjMoveStringLiteralDfa2_0(active0, 0x1000000L); + return jjMoveStringLiteralDfa2_0(active0, 0x40000001000000L); case 85: case 117: - return jjMoveStringLiteralDfa2_0(active0, 0xc00000100000000L); + return jjMoveStringLiteralDfa2_0(active0, 0x1800000100000000L); case 88: case 120: return jjMoveStringLiteralDfa2_0(active0, 0x10000L); @@ -421,19 +421,19 @@ private final int jjMoveStringLiteralDfa2_0(long old0, long active0) return jjMoveStringLiteralDfa3_0(active0, 0x10000000000000L); case 65: case 97: - return jjMoveStringLiteralDfa3_0(active0, 0x2000000001400000L); + return jjMoveStringLiteralDfa3_0(active0, 0x4000000001400000L); case 66: case 98: return jjMoveStringLiteralDfa3_0(active0, 0x80000L); case 67: case 99: - return jjMoveStringLiteralDfa3_0(active0, 0x208000000000000L); + return jjMoveStringLiteralDfa3_0(active0, 0x408000000000000L); case 68: case 100: if ((active0 & 0x400000000L) != 0L) return jjStartNfaWithStates_0(2, 34, 1); - else if ((active0 & 0x1000000000000000L) != 0L) - return jjStartNfaWithStates_0(2, 60, 1); + else if ((active0 & 0x2000000000000000L) != 0L) + return jjStartNfaWithStates_0(2, 61, 1); break; case 69: case 101: @@ -443,16 +443,18 @@ private final int jjMoveStringLiteralDfa2_0(long old0, long active0) return jjMoveStringLiteralDfa3_0(active0, 0x10000L); case 76: case 108: + if ((active0 & 0x40000000000000L) != 0L) + return jjStartNfaWithStates_0(2, 54, 1); return jjMoveStringLiteralDfa3_0(active0, 0x60300020L); case 77: case 109: - return jjMoveStringLiteralDfa3_0(active0, 0xc01000380000000L); + return jjMoveStringLiteralDfa3_0(active0, 0x1801000380000000L); case 78: case 110: return jjMoveStringLiteralDfa3_0(active0, 0x2000000000000L); case 79: case 111: - return jjMoveStringLiteralDfa3_0(active0, 0x64000004001100L); + return jjMoveStringLiteralDfa3_0(active0, 0xa4000004001100L); case 82: case 114: if ((active0 & 0x8000L) != 0L) @@ -465,10 +467,10 @@ private final int jjMoveStringLiteralDfa2_0(long old0, long active0) case 116: if ((active0 & 0x40000000000L) != 0L) return jjStartNfaWithStates_0(2, 42, 1); - return jjMoveStringLiteralDfa3_0(active0, 0x100000008040040L); + return jjMoveStringLiteralDfa3_0(active0, 0x200000008040040L); case 85: case 117: - return jjMoveStringLiteralDfa3_0(active0, 0x4080000000002000L); + return jjMoveStringLiteralDfa3_0(active0, 0x8100000000002000L); case 87: case 119: if ((active0 & 0x10000000L) != 0L) @@ -494,7 +496,7 @@ private final int jjMoveStringLiteralDfa3_0(long old0, long active0) switch(curChar) { case 95: - return jjMoveStringLiteralDfa4_0(active0, 0xc00c00100000000L); + return jjMoveStringLiteralDfa4_0(active0, 0x1800c00100000000L); case 65: case 97: return jjMoveStringLiteralDfa4_0(active0, 0x800880L); @@ -527,12 +529,12 @@ private final int jjMoveStringLiteralDfa3_0(long old0, long active0) return jjMoveStringLiteralDfa4_0(active0, 0x10000000000000L); case 78: case 110: - return jjMoveStringLiteralDfa4_0(active0, 0x6080000000002000L); + return jjMoveStringLiteralDfa4_0(active0, 0xc100000000002000L); case 79: case 111: if ((active0 & 0x40000L) != 0L) return jjStartNfaWithStates_0(3, 18, 1); - return jjMoveStringLiteralDfa4_0(active0, 0x148000000000000L); + return jjMoveStringLiteralDfa4_0(active0, 0x288000000000000L); case 80: case 112: if ((active0 & 0x20L) != 0L) @@ -547,7 +549,7 @@ private final int jjMoveStringLiteralDfa3_0(long old0, long active0) case 116: if ((active0 & 0x10000L) != 0L) return jjStartNfaWithStates_0(3, 16, 1); - return jjMoveStringLiteralDfa4_0(active0, 0x200000000000000L); + return jjMoveStringLiteralDfa4_0(active0, 0x400000000000000L); case 85: case 117: return jjMoveStringLiteralDfa4_0(active0, 0x60000000L); @@ -584,13 +586,13 @@ private final int jjMoveStringLiteralDfa4_0(long old0, long active0) return jjStartNfaWithStates_0(4, 19, 1); else if ((active0 & 0x2000000L) != 0L) return jjStartNfaWithStates_0(4, 25, 1); - return jjMoveStringLiteralDfa5_0(active0, 0x810000020000000L); + return jjMoveStringLiteralDfa5_0(active0, 0x1010000020000000L); case 71: case 103: - return jjMoveStringLiteralDfa5_0(active0, 0x2000000000000000L); + return jjMoveStringLiteralDfa5_0(active0, 0x4000000000000000L); case 72: case 104: - return jjMoveStringLiteralDfa5_0(active0, 0x400000000000000L); + return jjMoveStringLiteralDfa5_0(active0, 0x800000000000000L); case 75: case 107: if ((active0 & 0x4000000000000L) != 0L) @@ -606,10 +608,10 @@ private final int jjMoveStringLiteralDfa4_0(long old0, long active0) return jjMoveStringLiteralDfa5_0(active0, 0x800000400000L); case 77: case 109: - return jjMoveStringLiteralDfa5_0(active0, 0x40000040000000L); + return jjMoveStringLiteralDfa5_0(active0, 0x80000040000000L); case 79: case 111: - return jjMoveStringLiteralDfa5_0(active0, 0x200000000000000L); + return jjMoveStringLiteralDfa5_0(active0, 0x400000000000000L); case 82: case 114: if ((active0 & 0x40L) != 0L) @@ -624,15 +626,15 @@ private final int jjMoveStringLiteralDfa4_0(long old0, long active0) case 116: if ((active0 & 0x200000000L) != 0L) return jjStartNfaWithStates_0(4, 33, 1); - else if ((active0 & 0x4000000000000000L) != 0L) + else if ((active0 & 0x8000000000000000L) != 0L) { - jjmatchedKind = 62; + jjmatchedKind = 63; jjmatchedPos = 4; } - return jjMoveStringLiteralDfa5_0(active0, 0x80000001100800L); + return jjMoveStringLiteralDfa5_0(active0, 0x100000001100800L); case 85: case 117: - return jjMoveStringLiteralDfa5_0(active0, 0x100000000000000L); + return jjMoveStringLiteralDfa5_0(active0, 0x200000000000000L); case 86: case 118: return jjMoveStringLiteralDfa5_0(active0, 0x400100000000L); @@ -656,10 +658,10 @@ private final int jjMoveStringLiteralDfa5_0(long old0, long active0) return jjMoveStringLiteralDfa6_0(active0, 0x20000000000000L); case 65: case 97: - return jjMoveStringLiteralDfa6_0(active0, 0x400000000002000L); + return jjMoveStringLiteralDfa6_0(active0, 0x800000000002000L); case 67: case 99: - return jjMoveStringLiteralDfa6_0(active0, 0x100000000000000L); + return jjMoveStringLiteralDfa6_0(active0, 0x200000000000000L); case 68: case 100: if ((active0 & 0x8000000000000L) != 0L) @@ -673,15 +675,15 @@ private final int jjMoveStringLiteralDfa5_0(long old0, long active0) return jjStartNfaWithStates_0(5, 20, 1); else if ((active0 & 0x400000L) != 0L) return jjStartNfaWithStates_0(5, 22, 1); - else if ((active0 & 0x2000000000000000L) != 0L) - return jjStartNfaWithStates_0(5, 61, 1); + else if ((active0 & 0x4000000000000000L) != 0L) + return jjStartNfaWithStates_0(5, 62, 1); return jjMoveStringLiteralDfa6_0(active0, 0x1c00100000000L); case 70: case 102: - return jjMoveStringLiteralDfa6_0(active0, 0x40000000000000L); + return jjMoveStringLiteralDfa6_0(active0, 0x80000000000000L); case 73: case 105: - return jjMoveStringLiteralDfa6_0(active0, 0x80000001000200L); + return jjMoveStringLiteralDfa6_0(active0, 0x100000001000200L); case 76: case 108: return jjMoveStringLiteralDfa6_0(active0, 0x800000L); @@ -690,10 +692,10 @@ private final int jjMoveStringLiteralDfa5_0(long old0, long active0) return jjMoveStringLiteralDfa6_0(active0, 0x10000000000000L); case 78: case 110: - return jjMoveStringLiteralDfa6_0(active0, 0x800000040000000L); + return jjMoveStringLiteralDfa6_0(active0, 0x1000000040000000L); case 82: case 114: - return jjMoveStringLiteralDfa6_0(active0, 0x200000000000000L); + return jjMoveStringLiteralDfa6_0(active0, 0x400000000000000L); case 83: case 115: if ((active0 & 0x20000000L) != 0L) @@ -723,7 +725,7 @@ private final int jjMoveStringLiteralDfa6_0(long old0, long active0) switch(curChar) { case 95: - return jjMoveStringLiteralDfa7_0(active0, 0x200000000000000L); + return jjMoveStringLiteralDfa7_0(active0, 0x400000000000000L); case 65: case 97: return jjMoveStringLiteralDfa7_0(active0, 0x80000000L); @@ -743,13 +745,13 @@ private final int jjMoveStringLiteralDfa6_0(long old0, long active0) return jjMoveStringLiteralDfa7_0(active0, 0x40000000L); case 72: case 104: - return jjMoveStringLiteralDfa7_0(active0, 0x100000000000000L); + return jjMoveStringLiteralDfa7_0(active0, 0x200000000000000L); case 73: case 105: - return jjMoveStringLiteralDfa7_0(active0, 0x40000000000000L); + return jjMoveStringLiteralDfa7_0(active0, 0x80000000000000L); case 78: case 110: - return jjMoveStringLiteralDfa7_0(active0, 0x80800001000000L); + return jjMoveStringLiteralDfa7_0(active0, 0x100800001000000L); case 79: case 111: return jjMoveStringLiteralDfa7_0(active0, 0x10000000000000L); @@ -758,10 +760,10 @@ private final int jjMoveStringLiteralDfa6_0(long old0, long active0) return jjMoveStringLiteralDfa7_0(active0, 0x400100000000L); case 83: case 115: - return jjMoveStringLiteralDfa7_0(active0, 0x401000000000000L); + return jjMoveStringLiteralDfa7_0(active0, 0x801000000000000L); case 84: case 116: - return jjMoveStringLiteralDfa7_0(active0, 0x800000000002000L); + return jjMoveStringLiteralDfa7_0(active0, 0x1000000000002000L); default : break; } @@ -787,29 +789,29 @@ private final int jjMoveStringLiteralDfa7_0(long old0, long active0) return jjStartNfaWithStates_0(7, 9, 1); else if ((active0 & 0x2000L) != 0L) return jjStartNfaWithStates_0(7, 13, 1); - return jjMoveStringLiteralDfa8_0(active0, 0x100000000000000L); + return jjMoveStringLiteralDfa8_0(active0, 0x200000000000000L); case 71: case 103: if ((active0 & 0x1000000L) != 0L) return jjStartNfaWithStates_0(7, 24, 1); - return jjMoveStringLiteralDfa8_0(active0, 0x80800000000000L); + return jjMoveStringLiteralDfa8_0(active0, 0x100800000000000L); case 72: case 104: - if ((active0 & 0x400000000000000L) != 0L) - return jjStartNfaWithStates_0(7, 58, 1); + if ((active0 & 0x800000000000000L) != 0L) + return jjStartNfaWithStates_0(7, 59, 1); break; case 76: case 108: - return jjMoveStringLiteralDfa8_0(active0, 0x40000000000000L); + return jjMoveStringLiteralDfa8_0(active0, 0x80000000000000L); case 77: case 109: return jjMoveStringLiteralDfa8_0(active0, 0x80000000L); case 82: case 114: - return jjMoveStringLiteralDfa8_0(active0, 0x810000000000000L); + return jjMoveStringLiteralDfa8_0(active0, 0x1010000000000000L); case 83: case 115: - return jjMoveStringLiteralDfa8_0(active0, 0x201400100000000L); + return jjMoveStringLiteralDfa8_0(active0, 0x401400100000000L); default : break; } @@ -827,16 +829,16 @@ private final int jjMoveStringLiteralDfa8_0(long old0, long active0) switch(curChar) { case 95: - return jjMoveStringLiteralDfa9_0(active0, 0x80000000000000L); + return jjMoveStringLiteralDfa9_0(active0, 0x100000000000000L); case 67: case 99: return jjMoveStringLiteralDfa9_0(active0, 0x20000000000000L); case 68: case 100: - return jjMoveStringLiteralDfa9_0(active0, 0x100000000000000L); + return jjMoveStringLiteralDfa9_0(active0, 0x200000000000000L); case 73: case 105: - return jjMoveStringLiteralDfa9_0(active0, 0xa01400100000000L); + return jjMoveStringLiteralDfa9_0(active0, 0x1401400100000000L); case 77: case 109: return jjMoveStringLiteralDfa9_0(active0, 0x40000000L); @@ -847,7 +849,7 @@ private final int jjMoveStringLiteralDfa8_0(long old0, long active0) break; case 84: case 116: - return jjMoveStringLiteralDfa9_0(active0, 0x40800000000000L); + return jjMoveStringLiteralDfa9_0(active0, 0x80800000000000L); case 89: case 121: if ((active0 & 0x10000000000000L) != 0L) @@ -870,13 +872,13 @@ private final int jjMoveStringLiteralDfa9_0(long old0, long active0) switch(curChar) { case 95: - return jjMoveStringLiteralDfa10_0(active0, 0x100000000000000L); + return jjMoveStringLiteralDfa10_0(active0, 0x200000000000000L); case 66: case 98: - return jjMoveStringLiteralDfa10_0(active0, 0x80000000000000L); + return jjMoveStringLiteralDfa10_0(active0, 0x100000000000000L); case 69: case 101: - return jjMoveStringLiteralDfa10_0(active0, 0x840000000000000L); + return jjMoveStringLiteralDfa10_0(active0, 0x1080000000000000L); case 72: case 104: if ((active0 & 0x800000000000L) != 0L) @@ -890,7 +892,7 @@ private final int jjMoveStringLiteralDfa9_0(long old0, long active0) return jjMoveStringLiteralDfa10_0(active0, 0x1400100000000L); case 90: case 122: - return jjMoveStringLiteralDfa10_0(active0, 0x200000000000000L); + return jjMoveStringLiteralDfa10_0(active0, 0x400000000000000L); default : break; } @@ -909,15 +911,15 @@ private final int jjMoveStringLiteralDfa10_0(long old0, long active0) { case 66: case 98: - return jjMoveStringLiteralDfa11_0(active0, 0x100000000000000L); + return jjMoveStringLiteralDfa11_0(active0, 0x200000000000000L); case 69: case 101: - if ((active0 & 0x200000000000000L) != 0L) - return jjStartNfaWithStates_0(10, 57, 1); + if ((active0 & 0x400000000000000L) != 0L) + return jjStartNfaWithStates_0(10, 58, 1); return jjMoveStringLiteralDfa11_0(active0, 0x20000000000000L); case 76: case 108: - return jjMoveStringLiteralDfa11_0(active0, 0x80000040000000L); + return jjMoveStringLiteralDfa11_0(active0, 0x100000040000000L); case 78: case 110: if ((active0 & 0x1000000000000L) != 0L) @@ -925,13 +927,13 @@ private final int jjMoveStringLiteralDfa10_0(long old0, long active0) return jjMoveStringLiteralDfa11_0(active0, 0x400100000000L); case 82: case 114: - if ((active0 & 0x40000000000000L) != 0L) - return jjStartNfaWithStates_0(10, 54, 1); + if ((active0 & 0x80000000000000L) != 0L) + return jjStartNfaWithStates_0(10, 55, 1); break; case 83: case 115: - if ((active0 & 0x800000000000000L) != 0L) - return jjStartNfaWithStates_0(10, 59, 1); + if ((active0 & 0x1000000000000000L) != 0L) + return jjStartNfaWithStates_0(10, 60, 1); break; default : break; @@ -956,10 +958,10 @@ private final int jjMoveStringLiteralDfa11_0(long old0, long active0) return jjMoveStringLiteralDfa12_0(active0, 0x40000000L); case 76: case 108: - return jjMoveStringLiteralDfa12_0(active0, 0x100000000000000L); + return jjMoveStringLiteralDfa12_0(active0, 0x200000000000000L); case 79: case 111: - return jjMoveStringLiteralDfa12_0(active0, 0x80000000000000L); + return jjMoveStringLiteralDfa12_0(active0, 0x100000000000000L); case 83: case 115: if ((active0 & 0x100000000L) != 0L) @@ -988,7 +990,7 @@ private final int jjMoveStringLiteralDfa12_0(long old0, long active0) return jjMoveStringLiteralDfa13_0(active0, 0x20000040000000L); case 79: case 111: - return jjMoveStringLiteralDfa13_0(active0, 0x180000000000000L); + return jjMoveStringLiteralDfa13_0(active0, 0x300000000000000L); default : break; } @@ -1007,13 +1009,13 @@ private final int jjMoveStringLiteralDfa13_0(long old0, long active0) { case 77: case 109: - return jjMoveStringLiteralDfa14_0(active0, 0x80000000000000L); + return jjMoveStringLiteralDfa14_0(active0, 0x100000000000000L); case 78: case 110: return jjMoveStringLiteralDfa14_0(active0, 0x20000000000000L); case 79: case 111: - return jjMoveStringLiteralDfa14_0(active0, 0x100000000000000L); + return jjMoveStringLiteralDfa14_0(active0, 0x200000000000000L); case 83: case 115: if ((active0 & 0x40000000L) != 0L) @@ -1040,10 +1042,10 @@ private final int jjMoveStringLiteralDfa14_0(long old0, long active0) return jjMoveStringLiteralDfa15_0(active0, 0x20000000000000L); case 70: case 102: - return jjMoveStringLiteralDfa15_0(active0, 0x80000000000000L); + return jjMoveStringLiteralDfa15_0(active0, 0x100000000000000L); case 77: case 109: - return jjMoveStringLiteralDfa15_0(active0, 0x100000000000000L); + return jjMoveStringLiteralDfa15_0(active0, 0x200000000000000L); default : break; } @@ -1065,10 +1067,10 @@ private final int jjMoveStringLiteralDfa15_0(long old0, long active0) return jjMoveStringLiteralDfa16_0(active0, 0x20000000000000L); case 70: case 102: - return jjMoveStringLiteralDfa16_0(active0, 0x100000000000000L); + return jjMoveStringLiteralDfa16_0(active0, 0x200000000000000L); case 73: case 105: - return jjMoveStringLiteralDfa16_0(active0, 0x80000000000000L); + return jjMoveStringLiteralDfa16_0(active0, 0x100000000000000L); default : break; } @@ -1087,10 +1089,10 @@ private final int jjMoveStringLiteralDfa16_0(long old0, long active0) { case 73: case 105: - return jjMoveStringLiteralDfa17_0(active0, 0x100000000000000L); + return jjMoveStringLiteralDfa17_0(active0, 0x200000000000000L); case 76: case 108: - return jjMoveStringLiteralDfa17_0(active0, 0xa0000000000000L); + return jjMoveStringLiteralDfa17_0(active0, 0x120000000000000L); default : break; } @@ -1112,10 +1114,10 @@ private final int jjMoveStringLiteralDfa17_0(long old0, long active0) return jjMoveStringLiteralDfa18_0(active0, 0x20000000000000L); case 76: case 108: - return jjMoveStringLiteralDfa18_0(active0, 0x100000000000000L); + return jjMoveStringLiteralDfa18_0(active0, 0x200000000000000L); case 84: case 116: - return jjMoveStringLiteralDfa18_0(active0, 0x80000000000000L); + return jjMoveStringLiteralDfa18_0(active0, 0x100000000000000L); default : break; } @@ -1139,10 +1141,10 @@ private final int jjMoveStringLiteralDfa18_0(long old0, long active0) break; case 69: case 101: - return jjMoveStringLiteralDfa19_0(active0, 0x80000000000000L); + return jjMoveStringLiteralDfa19_0(active0, 0x100000000000000L); case 84: case 116: - return jjMoveStringLiteralDfa19_0(active0, 0x100000000000000L); + return jjMoveStringLiteralDfa19_0(active0, 0x200000000000000L); default : break; } @@ -1161,11 +1163,11 @@ private final int jjMoveStringLiteralDfa19_0(long old0, long active0) { case 69: case 101: - return jjMoveStringLiteralDfa20_0(active0, 0x100000000000000L); + return jjMoveStringLiteralDfa20_0(active0, 0x200000000000000L); case 82: case 114: - if ((active0 & 0x80000000000000L) != 0L) - return jjStartNfaWithStates_0(19, 55, 1); + if ((active0 & 0x100000000000000L) != 0L) + return jjStartNfaWithStates_0(19, 56, 1); break; default : break; @@ -1185,8 +1187,8 @@ private final int jjMoveStringLiteralDfa20_0(long old0, long active0) { case 82: case 114: - if ((active0 & 0x100000000000000L) != 0L) - return jjStartNfaWithStates_0(20, 56, 1); + if ((active0 & 0x200000000000000L) != 0L) + return jjStartNfaWithStates_0(20, 57, 1); break; default : break; @@ -1248,14 +1250,14 @@ private final int jjMoveNfa_0(int startState, int curPos) case 0: if ((0x3ff000000000000L & l) != 0L) { - if (kind > 64) - kind = 64; + if (kind > 65) + kind = 65; jjCheckNAddStates(0, 6); } else if ((0x400e00000000000L & l) != 0L) { - if (kind > 63) - kind = 63; + if (kind > 64) + kind = 64; jjCheckNAdd(1); } else if (curChar == 39) @@ -1268,8 +1270,8 @@ private final int jjMoveNfa_0(int startState, int curPos) case 1: if ((0x7ffe00000000000L & l) == 0L) break; - if (kind > 63) - kind = 63; + if (kind > 64) + kind = 64; jjCheckNAdd(1); break; case 2: @@ -1279,8 +1281,8 @@ private final int jjMoveNfa_0(int startState, int curPos) case 3: if ((0x3ff000000000000L & l) == 0L) break; - if (kind > 65) - kind = 65; + if (kind > 66) + kind = 66; jjCheckNAddTwoStates(3, 4); break; case 5: @@ -1290,8 +1292,8 @@ private final int jjMoveNfa_0(int startState, int curPos) case 6: if ((0x3ff000000000000L & l) == 0L) break; - if (kind > 65) - kind = 65; + if (kind > 66) + kind = 66; jjCheckNAdd(6); break; case 7: @@ -1303,8 +1305,8 @@ private final int jjMoveNfa_0(int startState, int curPos) jjCheckNAddTwoStates(8, 9); break; case 9: - if (curChar == 34 && kind > 67) - kind = 67; + if (curChar == 34 && kind > 68) + kind = 68; break; case 10: if (curChar == 39) @@ -1327,21 +1329,21 @@ private final int jjMoveNfa_0(int startState, int curPos) jjCheckNAddStates(10, 12); break; case 15: - if (curChar == 39 && kind > 68) - kind = 68; + if (curChar == 39 && kind > 69) + kind = 69; break; case 16: if ((0x3ff000000000000L & l) == 0L) break; - if (kind > 64) - kind = 64; + if (kind > 65) + kind = 65; jjCheckNAddStates(0, 6); break; case 17: if ((0x3ff000000000000L & l) == 0L) break; - if (kind > 64) - kind = 64; + if (kind > 65) + kind = 65; jjCheckNAdd(17); break; case 18: @@ -1355,8 +1357,8 @@ private final int jjMoveNfa_0(int startState, int curPos) case 20: if ((0x3ff000000000000L & l) == 0L) break; - if (kind > 65) - kind = 65; + if (kind > 66) + kind = 66; jjCheckNAddTwoStates(20, 21); break; case 22: @@ -1366,8 +1368,8 @@ private final int jjMoveNfa_0(int startState, int curPos) case 23: if ((0x3ff000000000000L & l) == 0L) break; - if (kind > 65) - kind = 65; + if (kind > 66) + kind = 66; jjCheckNAdd(23); break; case 24: @@ -1381,15 +1383,15 @@ private final int jjMoveNfa_0(int startState, int curPos) case 27: if ((0x3ff000000000000L & l) == 0L) break; - if (kind > 65) - kind = 65; + if (kind > 66) + kind = 66; jjCheckNAdd(27); break; case 28: if ((0x3ff000000000000L & l) == 0L) break; - if (kind > 65) - kind = 65; + if (kind > 66) + kind = 66; jjCheckNAddTwoStates(28, 29); break; case 30: @@ -1399,8 +1401,8 @@ private final int jjMoveNfa_0(int startState, int curPos) case 31: if ((0x3ff000000000000L & l) == 0L) break; - if (kind > 65) - kind = 65; + if (kind > 66) + kind = 66; jjCheckNAdd(31); break; default : break; @@ -1418,8 +1420,8 @@ private final int jjMoveNfa_0(int startState, int curPos) case 1: if ((0x47fffffe87fffffeL & l) == 0L) break; - if (kind > 63) - kind = 63; + if (kind > 64) + kind = 64; jjCheckNAdd(1); break; case 4: @@ -1498,12 +1500,12 @@ null, null, null, null, null, null, null, null, null, null, null, null, null, nu null, null, null, null, null, null, null, null, null, "\54", "\50", "\51", "\75", "\76", "\74", null, null, "\41\75", "\52", null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, -null, null, "\73", }; +null, null, null, "\73", }; public static final String[] lexStateNames = { "DEFAULT", }; static final long[] jjtoToken = { - 0xffffffffffffffe1L, 0x3bL, + 0xffffffffffffffe1L, 0x77L, }; static final long[] jjtoSkip = { 0x1eL, 0x0L, diff --git a/src/java/org/apache/hadoop/hbase/regionserver/HStore.java b/src/java/org/apache/hadoop/hbase/regionserver/HStore.java index e4221e6c038..287d873cfa4 100644 --- a/src/java/org/apache/hadoop/hbase/regionserver/HStore.java +++ b/src/java/org/apache/hadoop/hbase/regionserver/HStore.java @@ -85,7 +85,7 @@ public class HStore implements HConstants { private static final String BLOOMFILTER_FILE_NAME = "filter"; - protected final Memcache memcache = new Memcache(); + protected final Memcache memcache; private final Path basedir; private final HRegionInfo info; private final HColumnDescriptor family; @@ -94,6 +94,7 @@ public class HStore implements HConstants { private final HBaseConfiguration conf; private final Path filterDir; final Filter bloomFilter; + protected long ttl; private final long desiredMaxFileSize; private volatile long storeSize; @@ -173,7 +174,10 @@ public class HStore implements HConstants { this.family = family; this.fs = fs; this.conf = conf; - + this.ttl = family.getTimeToLive(); + if (ttl != HConstants.FOREVER) + this.ttl *= 1000; + this.memcache = new Memcache(this.ttl); this.compactionDir = HRegion.getCompactionDir(basedir); this.storeName = new Text(this.info.getEncodedName() + "/" + this.family.getFamilyName()); @@ -613,6 +617,7 @@ public class HStore implements HConstants { // flush to list of store files. Add cleanup of anything put on filesystem // if we fail. synchronized(flushLock) { + long now = System.currentTimeMillis(); // A. Write the Maps out to the disk HStoreFile flushedFile = new HStoreFile(conf, fs, basedir, info.getEncodedName(), family.getFamilyName(), -1L, null); @@ -637,9 +642,17 @@ public class HStore implements HConstants { byte[] bytes = es.getValue(); TextSequence f = HStoreKey.extractFamily(curkey.getColumn()); if (f.equals(this.family.getFamilyName())) { - entries++; - out.append(curkey, new ImmutableBytesWritable(bytes)); - flushed += HRegion.getEntrySize(curkey, bytes); + if (ttl == HConstants.FOREVER || + now < curkey.getTimestamp() + ttl) { + entries++; + out.append(curkey, new ImmutableBytesWritable(bytes)); + flushed += HRegion.getEntrySize(curkey, bytes); + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("internalFlushCache: " + curkey + + ": expired, skipped"); + } + } } } } finally { @@ -858,6 +871,7 @@ public class HStore implements HConstants { } } + long now = System.currentTimeMillis(); int timesSeen = 0; Text lastRow = new Text(); Text lastColumn = new Text(); @@ -903,7 +917,13 @@ public class HStore implements HConstants { if (sk.getRow().getLength() != 0 && sk.getColumn().getLength() != 0) { // Only write out objects which have a non-zero length key and // value - compactedOut.append(sk, vals[smallestKey]); + if (ttl == HConstants.FOREVER || now < sk.getTimestamp() + ttl) { + compactedOut.append(sk, vals[smallestKey]); + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("compactHStoreFiles: " + sk + ": expired, deleted"); + } + } } } @@ -1125,6 +1145,8 @@ public class HStore implements HConstants { Set columns, Map deletes, Map results) throws IOException { synchronized(map) { + long now = System.currentTimeMillis(); + // seek back to the beginning map.reset(); @@ -1159,11 +1181,18 @@ public class HStore implements HConstants { // there aren't any pending deletes. if (!(deletes.containsKey(readcol) && deletes.get(readcol).longValue() >= readkey.getTimestamp())) { - results.put(new Text(readcol), - new Cell(readval.get(), readkey.getTimestamp())); - // need to reinstantiate the readval so we can reuse it, - // otherwise next iteration will destroy our result - readval = new ImmutableBytesWritable(); + if (ttl == HConstants.FOREVER || + now < readkey.getTimestamp() + ttl) { + results.put(new Text(readcol), + new Cell(readval.get(), readkey.getTimestamp())); + // need to reinstantiate the readval so we can reuse it, + // otherwise next iteration will destroy our result + readval = new ImmutableBytesWritable(); + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("getFullFromMapFile: " + readkey + ": expired, skipped"); + } + } } } } else if(key.getRow().compareTo(readkey.getRow()) < 0) { @@ -1197,6 +1226,7 @@ public class HStore implements HConstants { } this.lock.readLock().lock(); + long now = System.currentTimeMillis(); try { // Check the memcache List results = this.memcache.get(key, numVersions); @@ -1233,7 +1263,14 @@ public class HStore implements HConstants { continue; } if (!isDeleted(readkey, readval.get(), true, deletes)) { - results.add(new Cell(readval.get(), readkey.getTimestamp())); + if (ttl == HConstants.FOREVER || + now < readkey.getTimestamp() + ttl) { + results.add(new Cell(readval.get(), readkey.getTimestamp())); + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("get: " + readkey + ": expired, skipped"); + } + } // Perhaps only one version is wanted. I could let this // test happen later in the for loop test but it would cost // the allocation of an ImmutableBytesWritable. @@ -1247,7 +1284,14 @@ public class HStore implements HConstants { !hasEnoughVersions(numVersions, results); readval = new ImmutableBytesWritable()) { if (!isDeleted(readkey, readval.get(), true, deletes)) { - results.add(new Cell(readval.get(), readkey.getTimestamp())); + if (ttl == HConstants.FOREVER || + now < readkey.getTimestamp() + ttl) { + results.add(new Cell(readval.get(), readkey.getTimestamp())); + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("get: " + readkey + ": expired, skipped"); + } + } } } } @@ -1292,6 +1336,7 @@ public class HStore implements HConstants { // This code below is very close to the body of the get method. this.lock.readLock().lock(); + long now = System.currentTimeMillis(); try { MapFile.Reader[] maparray = getReaders(); for(int i = maparray.length - 1; i >= 0; i--) { @@ -1319,7 +1364,15 @@ public class HStore implements HConstants { // in the memcache if (!isDeleted(readkey, readval.get(), false, null) && !keys.contains(readkey)) { - keys.add(new HStoreKey(readkey)); + if (ttl == HConstants.FOREVER || + now < readkey.getTimestamp() + ttl) { + keys.add(new HStoreKey(readkey)); + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("getKeys: " + readkey + + ": expired, skipped"); + } + } // if we've collected enough versions, then exit the loop. if (versions != ALL_VERSIONS && keys.size() >= versions) { @@ -1397,7 +1450,9 @@ public class HStore implements HConstants { if (!map.next(readkey, readval)) { return; } - + + long now = System.currentTimeMillis(); + // if there aren't any candidate keys yet, we'll do some things slightly // different if (candidateKeys.isEmpty()) { @@ -1426,8 +1481,16 @@ public class HStore implements HConstants { // as a candidate key if (readkey.getRow().equals(row)) { if (!HLogEdit.isDeleted(readval.get())) { - candidateKeys.put(stripTimestamp(readkey), - new Long(readkey.getTimestamp())); + if (ttl == HConstants.FOREVER || + now < readkey.getTimestamp() + ttl) { + candidateKeys.put(stripTimestamp(readkey), + new Long(readkey.getTimestamp())); + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("rowAtOrBeforeFromMapFile:" + readkey + + ": expired, skipped"); + } + } } } else if (readkey.getRow().compareTo(row) > 0 ) { // if the row key we just read is beyond the key we're searching for, @@ -1438,8 +1501,16 @@ public class HStore implements HConstants { // we're seeking yet, so this row is a candidate for closest // (assuming that it isn't a delete). if (!HLogEdit.isDeleted(readval.get())) { - candidateKeys.put(stripTimestamp(readkey), - new Long(readkey.getTimestamp())); + if (ttl == HConstants.FOREVER || + now < readkey.getTimestamp() + ttl) { + candidateKeys.put(stripTimestamp(readkey), + new Long(readkey.getTimestamp())); + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("rowAtOrBeforeFromMapFile:" + readkey + + ": expired, skipped"); + } + } } } } while(map.next(readkey, readval)); @@ -1490,7 +1561,16 @@ public class HStore implements HConstants { if (readkey.getRow().equals(row)) { strippedKey = stripTimestamp(readkey); if (!HLogEdit.isDeleted(readval.get())) { - candidateKeys.put(strippedKey, new Long(readkey.getTimestamp())); + if (ttl == HConstants.FOREVER || + now < readkey.getTimestamp() + ttl) { + candidateKeys.put(strippedKey, + new Long(readkey.getTimestamp())); + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("rowAtOrBeforeFromMapFile: " + readkey + + ": expired, skipped"); + } + } } else { // if the candidate keys contain any that might match by timestamp, // then check for a match and remove it if it's too young to @@ -1514,7 +1594,15 @@ public class HStore implements HConstants { // we're seeking yet, so this row is a candidate for closest // (assuming that it isn't a delete). if (!HLogEdit.isDeleted(readval.get())) { - candidateKeys.put(strippedKey, readkey.getTimestamp()); + if (ttl == HConstants.FOREVER || + now < readkey.getTimestamp() + ttl) { + candidateKeys.put(strippedKey, readkey.getTimestamp()); + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("rowAtOrBeforeFromMapFile: " + readkey + + ": expired, skipped"); + } + } } else { // if the candidate keys contain any that might match by timestamp, // then check for a match and remove it if it's too young to diff --git a/src/java/org/apache/hadoop/hbase/regionserver/Memcache.java b/src/java/org/apache/hadoop/hbase/regionserver/Memcache.java index 30c67ad811f..6720aece185 100644 --- a/src/java/org/apache/hadoop/hbase/regionserver/Memcache.java +++ b/src/java/org/apache/hadoop/hbase/regionserver/Memcache.java @@ -17,7 +17,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - + package org.apache.hadoop.hbase.regionserver; import java.io.IOException; @@ -53,6 +53,8 @@ import org.apache.hadoop.io.WritableComparable; class Memcache { private final Log LOG = LogFactory.getLog(this.getClass().getName()); + private long ttl; + // Note that since these structures are always accessed with a lock held, // so no additional synchronization is required. @@ -66,6 +68,21 @@ class Memcache { private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); + /** + * Default constructor. Used for tests. + */ + public Memcache() + { + ttl = HConstants.FOREVER; + } + + /** + * Constructor. + * @param ttl The TTL for cache entries, in milliseconds. + */ + public Memcache(long ttl) { + this.ttl = ttl; + } /* * Utility method. @@ -229,7 +246,7 @@ class Memcache { if (itKey.getRow().compareTo(row) <= 0) { continue; } - // Note: Not suppressing deletes. + // Note: Not suppressing deletes or expired cells. result = itKey.getRow(); break; } @@ -268,7 +285,9 @@ class Memcache { return; } + List victims = new ArrayList(); SortedMap tailMap = map.tailMap(key); + long now = System.currentTimeMillis(); for (Map.Entry es: tailMap.entrySet()) { HStoreKey itKey = es.getKey(); Text itCol = itKey.getColumn(); @@ -282,14 +301,27 @@ class Memcache { deletes.put(new Text(itCol), Long.valueOf(itKey.getTimestamp())); } } else if (!(deletes.containsKey(itCol) - && deletes.get(itCol).longValue() >= itKey.getTimestamp())) { - results.put(new Text(itCol), new Cell(val, itKey.getTimestamp())); + && deletes.get(itCol).longValue() >= itKey.getTimestamp())) { + // Skip expired cells + if (ttl == HConstants.FOREVER || + now < itKey.getTimestamp() + ttl) { + results.put(new Text(itCol), + new Cell(val, itKey.getTimestamp())); + } else { + victims.add(itKey); + if (LOG.isDebugEnabled()) { + LOG.debug("internalGetFull: " + itKey + ": expired, skipped"); + } + } } } } else if (key.getRow().compareTo(itKey.getRow()) < 0) { break; } } + // Remove expired victims from the map. + for (HStoreKey v: victims) + map.remove(v); } /** @@ -323,7 +355,8 @@ class Memcache { Iterator key_iterator = null; HStoreKey found_key = null; - + ArrayList victims = new ArrayList(); + long now = System.currentTimeMillis(); // get all the entries that come equal or after our search key SortedMap tailMap = map.tailMap(search_key); @@ -348,8 +381,16 @@ class Memcache { } } } else { - candidateKeys.put(strippedKey, - new Long(found_key.getTimestamp())); + if (ttl == HConstants.FOREVER || + now < found_key.getTimestamp() + ttl) { + candidateKeys.put(strippedKey, + new Long(found_key.getTimestamp())); + } else { + victims.add(found_key); + if (LOG.isDebugEnabled()) { + LOG.debug(":" + found_key + ": expired, skipped"); + } + } } } } while (found_key.getRow().compareTo(key) <= 0 @@ -385,9 +426,18 @@ class Memcache { // take note of the row of this candidate so that we'll know when // we cross the row boundary into the previous row. if (!HLogEdit.isDeleted(headMap.get(thisKey))) { - lastRowFound = thisKey.getRow(); - candidateKeys.put(stripTimestamp(thisKey), - new Long(thisKey.getTimestamp())); + if (ttl == HConstants.FOREVER || + now < found_key.getTimestamp() + ttl) { + lastRowFound = thisKey.getRow(); + candidateKeys.put(stripTimestamp(thisKey), + new Long(thisKey.getTimestamp())); + } else { + victims.add(found_key); + if (LOG.isDebugEnabled()) { + LOG.debug("internalGetRowKeyAtOrBefore: " + found_key + + ": expired, skipped"); + } + } } } } else { @@ -413,12 +463,24 @@ class Memcache { } } } else { - candidateKeys.put(stripTimestamp(found_key), - Long.valueOf(found_key.getTimestamp())); + if (ttl == HConstants.FOREVER || + now < found_key.getTimestamp() + ttl) { + candidateKeys.put(stripTimestamp(found_key), + Long.valueOf(found_key.getTimestamp())); + } else { + victims.add(found_key); + if (LOG.isDebugEnabled()) { + LOG.debug("internalGetRowKeyAtOrBefore: " + found_key + + ": expired, skipped"); + } + } } } while (key_iterator.hasNext()); } } + // Remove expired victims from the map. + for (HStoreKey victim: victims) + map.remove(victim); } static HStoreKey stripTimestamp(HStoreKey key) { @@ -442,21 +504,37 @@ class Memcache { final int numVersions) { ArrayList result = new ArrayList(); + // TODO: If get is of a particular version -- numVersions == 1 -- we // should be able to avoid all of the tailmap creations and iterations // below. + long now = System.currentTimeMillis(); + List victims = new ArrayList(); SortedMap tailMap = map.tailMap(key); for (Map.Entry es: tailMap.entrySet()) { HStoreKey itKey = es.getKey(); if (itKey.matchesRowCol(key)) { if (!HLogEdit.isDeleted(es.getValue())) { - result.add(new Cell(tailMap.get(itKey), itKey.getTimestamp())); + // Filter out expired results + if (ttl == HConstants.FOREVER || + now < itKey.getTimestamp() + ttl) { + result.add(new Cell(tailMap.get(itKey), itKey.getTimestamp())); + } else { + victims.add(itKey); + if (LOG.isDebugEnabled()) { + LOG.debug("internalGet: " + itKey + ": expired, skipped"); + } + } } } if (numVersions > 0 && result.size() >= numVersions) { break; } } + // Remove expired victims from the map. + for (HStoreKey v: victims) + map.remove(v); + return result; } @@ -500,7 +578,9 @@ class Memcache { private List internalGetKeys(final SortedMap map, final HStoreKey origin, final int versions) { + long now = System.currentTimeMillis(); List result = new ArrayList(); + List victims = new ArrayList(); SortedMap tailMap = map.tailMap(origin); for (Map.Entry es: tailMap.entrySet()) { HStoreKey key = es.getKey(); @@ -525,15 +605,26 @@ class Memcache { break; } } - if (!HLogEdit.isDeleted(es.getValue())) { - result.add(key); + if (ttl == HConstants.FOREVER || now < key.getTimestamp() + ttl) { + result.add(key); + } else { + victims.add(key); + if (LOG.isDebugEnabled()) { + LOG.debug("internalGetKeys: " + key + ": expired, skipped"); + } + } if (versions != HConstants.ALL_VERSIONS && result.size() >= versions) { // We have enough results. Return. break; } } } + + // Clean expired victims from the map. + for (HStoreKey v: victims) + map.remove(v); + return result; } diff --git a/src/java/org/apache/hadoop/hbase/regionserver/StoreFileScanner.java b/src/java/org/apache/hadoop/hbase/regionserver/StoreFileScanner.java index 0aa16b50867..96bb2622aea 100644 --- a/src/java/org/apache/hadoop/hbase/regionserver/StoreFileScanner.java +++ b/src/java/org/apache/hadoop/hbase/regionserver/StoreFileScanner.java @@ -24,6 +24,7 @@ import java.io.IOException; import java.util.SortedMap; import java.util.concurrent.locks.ReentrantReadWriteLock; +import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HStoreKey; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.hadoop.io.MapFile; @@ -216,6 +217,8 @@ implements ChangedReadersObserver { // Find the next viable row label (and timestamp). Text viableRow = null; long viableTimestamp = -1; + long now = System.currentTimeMillis(); + long ttl = store.ttl; for(int i = 0; i < keys.length; i++) { if((keys[i] != null) && (columnMatch(i)) @@ -224,8 +227,14 @@ implements ChangedReadersObserver { || (keys[i].getRow().compareTo(viableRow) < 0) || ((keys[i].getRow().compareTo(viableRow) == 0) && (keys[i].getTimestamp() > viableTimestamp)))) { - viableRow = new Text(keys[i].getRow()); - viableTimestamp = keys[i].getTimestamp(); + if (ttl == HConstants.FOREVER || now < keys[i].getTimestamp() + ttl) { + viableRow = new Text(keys[i].getRow()); + viableTimestamp = keys[i].getTimestamp(); + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("getNextViableRow :" + keys[i] + ": expired, skipped"); + } + } } } return new ViableRow(viableRow, viableTimestamp); @@ -248,6 +257,13 @@ implements ChangedReadersObserver { closeSubScanner(i); return true; } + long now = System.currentTimeMillis(); + long ttl = store.ttl; + if (ttl != HConstants.FOREVER && now >= firstKey.getTimestamp() + ttl) { + // Didn't find it. Close the scanner and return TRUE + closeSubScanner(i); + return true; + } this.vals[i] = ibw.get(); keys[i].setRow(firstKey.getRow()); keys[i].setColumn(firstKey.getColumn()); @@ -264,15 +280,23 @@ implements ChangedReadersObserver { boolean getNext(int i) throws IOException { boolean result = false; ImmutableBytesWritable ibw = new ImmutableBytesWritable(); + long now = System.currentTimeMillis(); + long ttl = store.ttl; while (true) { if (!readers[i].next(keys[i], ibw)) { closeSubScanner(i); break; } if (keys[i].getTimestamp() <= this.timestamp) { - vals[i] = ibw.get(); - result = true; - break; + if (ttl == HConstants.FOREVER || now < keys[i].getTimestamp() + ttl) { + vals[i] = ibw.get(); + result = true; + break; + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("getNext: " + keys[i] + ": expired, skipped"); + } + } } } return result; diff --git a/src/java/org/apache/hadoop/hbase/thrift/Hbase.thrift b/src/java/org/apache/hadoop/hbase/thrift/Hbase.thrift index 2a246a68402..6dcad2cb148 100644 --- a/src/java/org/apache/hadoop/hbase/thrift/Hbase.thrift +++ b/src/java/org/apache/hadoop/hbase/thrift/Hbase.thrift @@ -65,7 +65,8 @@ struct ColumnDescriptor { 6:string bloomFilterType = "NONE", 7:i32 bloomFilterVectorSize = 0, 8:i32 bloomFilterNbHashes = 0, - 9:bool blockCacheEnabled = 0 + 9:bool blockCacheEnabled = 0, + 10:i32 timeToLive = -1 } /** diff --git a/src/java/org/apache/hadoop/hbase/thrift/ThriftUtilities.java b/src/java/org/apache/hadoop/hbase/thrift/ThriftUtilities.java index be96ae956d1..029c1ed5a93 100644 --- a/src/java/org/apache/hadoop/hbase/thrift/ThriftUtilities.java +++ b/src/java/org/apache/hadoop/hbase/thrift/ThriftUtilities.java @@ -59,7 +59,7 @@ public class ThriftUtilities { } HColumnDescriptor col = new HColumnDescriptor(new Text(in.name), in.maxVersions, comp, in.inMemory, in.blockCacheEnabled, - in.maxValueLength, bloom); + in.maxValueLength, in.timeToLive, bloom); return col; } diff --git a/src/java/org/apache/hadoop/hbase/thrift/generated/ColumnDescriptor.java b/src/java/org/apache/hadoop/hbase/thrift/generated/ColumnDescriptor.java index 25b73209ebd..b05740e3695 100644 --- a/src/java/org/apache/hadoop/hbase/thrift/generated/ColumnDescriptor.java +++ b/src/java/org/apache/hadoop/hbase/thrift/generated/ColumnDescriptor.java @@ -47,6 +47,7 @@ public class ColumnDescriptor implements TBase, java.io.Serializable { public int bloomFilterVectorSize; public int bloomFilterNbHashes; public boolean blockCacheEnabled; + public int timeToLive; public final Isset __isset = new Isset(); public static final class Isset { @@ -59,6 +60,7 @@ public class ColumnDescriptor implements TBase, java.io.Serializable { public boolean bloomFilterVectorSize = false; public boolean bloomFilterNbHashes = false; public boolean blockCacheEnabled = false; + public boolean timeToLive = false; } public ColumnDescriptor() { @@ -78,6 +80,8 @@ public class ColumnDescriptor implements TBase, java.io.Serializable { this.blockCacheEnabled = false; + this.timeToLive = -1; + } public ColumnDescriptor( @@ -89,7 +93,8 @@ public class ColumnDescriptor implements TBase, java.io.Serializable { String bloomFilterType, int bloomFilterVectorSize, int bloomFilterNbHashes, - boolean blockCacheEnabled) + boolean blockCacheEnabled, + int timeToLive) { this(); this.name = name; @@ -110,6 +115,8 @@ public class ColumnDescriptor implements TBase, java.io.Serializable { this.__isset.bloomFilterNbHashes = true; this.blockCacheEnabled = blockCacheEnabled; this.__isset.blockCacheEnabled = true; + this.timeToLive = timeToLive; + this.__isset.timeToLive = true; } public void read(TProtocol iprot) throws TException { @@ -195,6 +202,14 @@ public class ColumnDescriptor implements TBase, java.io.Serializable { TProtocolUtil.skip(iprot, field.type); } break; + case 10: + if (field.type == TType.I32) { + this.timeToLive = iprot.readI32(); + this.__isset.timeToLive = true; + } else { + TProtocolUtil.skip(iprot, field.type); + } + break; default: TProtocolUtil.skip(iprot, field.type); break; @@ -268,6 +283,12 @@ public class ColumnDescriptor implements TBase, java.io.Serializable { oprot.writeFieldBegin(field); oprot.writeBool(this.blockCacheEnabled); oprot.writeFieldEnd(); + field.name = "timeToLive"; + field.type = TType.I32; + field.id = 10; + oprot.writeFieldBegin(field); + oprot.writeI32(this.timeToLive); + oprot.writeFieldEnd(); oprot.writeFieldStop(); oprot.writeStructEnd(); } @@ -292,6 +313,8 @@ public class ColumnDescriptor implements TBase, java.io.Serializable { sb.append(this.bloomFilterNbHashes); sb.append(",blockCacheEnabled:"); sb.append(this.blockCacheEnabled); + sb.append(",timeToLive:"); + sb.append(this.timeToLive); sb.append(")"); return sb.toString(); } diff --git a/src/test/org/apache/hadoop/hbase/HBaseTestCase.java b/src/test/org/apache/hadoop/hbase/HBaseTestCase.java index cfc0f3927a1..c70b35e704d 100644 --- a/src/test/org/apache/hadoop/hbase/HBaseTestCase.java +++ b/src/test/org/apache/hadoop/hbase/HBaseTestCase.java @@ -186,11 +186,14 @@ public abstract class HBaseTestCase extends TestCase { final int versions) { HTableDescriptor htd = new HTableDescriptor(name); htd.addFamily(new HColumnDescriptor(new Text(COLFAMILY_NAME1), versions, - CompressionType.NONE, false, false, Integer.MAX_VALUE, null)); + CompressionType.NONE, false, false, Integer.MAX_VALUE, + HConstants.FOREVER, null)); htd.addFamily(new HColumnDescriptor(new Text(COLFAMILY_NAME2), versions, - CompressionType.NONE, false, false, Integer.MAX_VALUE, null)); + CompressionType.NONE, false, false, Integer.MAX_VALUE, + HConstants.FOREVER, null)); htd.addFamily(new HColumnDescriptor(new Text(COLFAMILY_NAME3), versions, - CompressionType.NONE, false, false, Integer.MAX_VALUE, null)); + CompressionType.NONE, false, false, Integer.MAX_VALUE, + HConstants.FOREVER, null)); return htd; } diff --git a/src/test/org/apache/hadoop/hbase/TestBloomFilters.java b/src/test/org/apache/hadoop/hbase/TestBloomFilters.java index 46dbf1d0b9c..4a0dcc584b4 100644 --- a/src/test/org/apache/hadoop/hbase/TestBloomFilters.java +++ b/src/test/org/apache/hadoop/hbase/TestBloomFilters.java @@ -174,6 +174,7 @@ public class TestBloomFilters extends HBaseClusterTestCase { HColumnDescriptor.DEFAULT_IN_MEMORY, // not in memory HColumnDescriptor.DEFAULT_BLOCK_CACHE_ENABLED, HColumnDescriptor.DEFAULT_MAX_VALUE_LENGTH, + HColumnDescriptor.DEFAULT_TIME_TO_LIVE, bloomFilter ) ); @@ -240,6 +241,7 @@ public class TestBloomFilters extends HBaseClusterTestCase { HColumnDescriptor.DEFAULT_IN_MEMORY, // not in memory HColumnDescriptor.DEFAULT_BLOCK_CACHE_ENABLED, HColumnDescriptor.DEFAULT_MAX_VALUE_LENGTH, + HColumnDescriptor.DEFAULT_TIME_TO_LIVE, bloomFilter ) ); diff --git a/src/test/org/apache/hadoop/hbase/TestToString.java b/src/test/org/apache/hadoop/hbase/TestToString.java index 07ea01a8453..665751f77e3 100644 --- a/src/test/org/apache/hadoop/hbase/TestToString.java +++ b/src/test/org/apache/hadoop/hbase/TestToString.java @@ -46,7 +46,7 @@ public class TestToString extends TestCase { assertEquals("Table descriptor", "name: -ROOT-, families: {info:={name: " + "info, max versions: 1, compression: NONE, in memory: false, " + "block cache enabled: false, max length: 2147483647, " + - "bloom filter: none}}", htd.toString()); + "time to live: FOREVER, bloom filter: none}}", htd.toString()); } /** @@ -59,6 +59,7 @@ public class TestToString extends TestCase { "regionname: -ROOT-,,0, startKey: <>, endKey: <>, encodedName: 70236052, tableDesc: " + "{name: -ROOT-, families: {info:={name: info, max versions: 1, " + "compression: NONE, in memory: false, block cache enabled: false, " + - "max length: 2147483647, bloom filter: none}}}", hri.toString()); + "max length: 2147483647, time to live: FOREVER, bloom filter: none}}}", + hri.toString()); } } diff --git a/src/test/org/apache/hadoop/hbase/regionserver/TestTimestamp.java b/src/test/org/apache/hadoop/hbase/regionserver/TestTimestamp.java index 445124e8e84..ad3dffc60bf 100644 --- a/src/test/org/apache/hadoop/hbase/regionserver/TestTimestamp.java +++ b/src/test/org/apache/hadoop/hbase/regionserver/TestTimestamp.java @@ -83,7 +83,8 @@ public class TestTimestamp extends HBaseClusterTestCase { private HRegion createRegion() throws IOException { HTableDescriptor htd = createTableDescriptor(getName()); htd.addFamily(new HColumnDescriptor(COLUMN, VERSIONS, - CompressionType.NONE, false, false, Integer.MAX_VALUE, null)); + CompressionType.NONE, false, false, Integer.MAX_VALUE, + HConstants.FOREVER, null)); return createNewHRegion(htd, null, null); } }