Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

LUCENE-5098: Broadword utility methods. 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1502690 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Adrien Grand 2013-07-12 21:05:20 +00:00
parent 412895a266
commit 7983841e88
4 changed files with 290 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
lucene/CHANGES.txt
View File

@ -54,6 +54,9 @@ New features
* LUCENE-5081: Added WAH8DocIdSet, an in-memory doc id set implementation based * LUCENE-5081: Added WAH8DocIdSet, an in-memory doc id set implementation based
on word-aligned hybrid encoding. (Adrien Grand) on word-aligned hybrid encoding. (Adrien Grand)
* LUCENE-5098: New broadword utility methods in oal.util.BroadWord.
(Paul Elschot via Adrien Grand, Dawid Weiss)
======================= Lucene 4.4.0 ======================= ======================= Lucene 4.4.0 =======================
Changes in backwards compatibility policy Changes in backwards compatibility policy

150
lucene/core/src/java/org/apache/lucene/util/BroadWord.java Normal file
View File

@ -0,0 +1,150 @@
package org.apache.lucene.util;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* Methods and constants inspired by the article
* "Broadword Implementation of Rank/Select Queries" by Sebastiano Vigna, January 30, 2012:
* <ul>
* <li>algorithm 1: {@link #rank9(long)}, count of set bits in a <code>long</code>
* <li>algorithm 2: {@link #select9(long, int)}, selection of a set bit in a <code>long</code>,
* <li>bytewise signed smaller &lt;<sub><small>8</small></sub> operator: {@link #smallerUpTo7_8(long,long)}.
* <li>shortwise signed smaller &lt;<sub><small>16</small></sub> operator: {@link #smallerUpto15_16(long,long)}.
* <li>some of the Lk and Hk constants that are used by the above:
* L8 {@link #L8_L}, H8 {@link #H8_L}, L9 {@link #L9_L}, L16 {@link #L16_L}and H16 {@link #H8_L}.
* </ul>
* @lucene.internal
*/
public final class BroadWord {
// TBD: test smaller8 and smaller16 separately.
private BroadWord() {} // no instance
/** Bit count of a long.
* Only here to compare the implementation with {@link #select9(long,int)},
* normally {@link Long#bitCount} is preferable.
* @return The total number of 1 bits in x.
*/
static int rank9(long x) {
// Step 0 leaves in each pair of bits the number of ones originally contained in that pair:
x = x - ((x & 0xAAAAAAAAAAAAAAAAL) >>> 1);
// Step 1, idem for each nibble:
x = (x & 0x3333333333333333L) + ((x >>> 2) & 0x3333333333333333L);
// Step 2, idem for each byte:
x = (x + (x >>> 4)) & 0x0F0F0F0F0F0F0F0FL;
// Multiply to sum them all into the high byte, and return the high byte:
return (int) ((x * L8_L) >>> 56);
}
/** Select a 1-bit from a long.
* @return The index of the r-th 1 bit in x, or if no such bit exists, 72.
*/
public static int select9(long x, int r) {
long s = x - ((x & 0xAAAAAAAAAAAAAAAAL) >>> 1); // Step 0, pairwise bitsums
// Correct a small mistake in algorithm 2:
// Use s instead of x the second time in right shift 2, compare to Algorithm 1 in rank9 above.
s = (s & 0x3333333333333333L) + ((s >>> 2) & 0x3333333333333333L); // Step 1, nibblewise bitsums
s = ((s + (s >>> 4)) & 0x0F0F0F0F0F0F0F0FL) * L8_L; // Step 2, bytewise bitsums
long b = ((smallerUpTo7_8(s, (r * L8_L)) >>> 7) * L8_L) >>> 53; // & (~7L); // Step 3, side ways addition for byte number times 8
long l = r - (((s << 8) >>> b) & 0xFFL); // Step 4, byte wise rank, subtract the rank with byte at b-8, or zero for b=0;
assert 0L <= l : l;
//assert l < 8 : l; //fails when bit r is not available.
// Select bit l from byte (x >>> b):
long spr = (((x >>> b) & 0xFFL) * L8_L) & L9_L; // spread the 8 bits of the byte at b over the long at L9 positions
// long spr_bigger8_zero = smaller8(0L, spr); // inlined smaller8 with 0L argument:
// FIXME: replace by biggerequal8_one formula from article page 6, line 9. four operators instead of five here.
long spr_bigger8_zero = ( ( H8_L - (spr & (~H8_L)) ) ^ (~spr) ) & H8_L;
s = (spr_bigger8_zero >>> 7) * L8_L; // Step 5, sideways byte add the 8 bits towards the high byte
int res = (int) (b + (((smallerUpTo7_8(s, (l * L8_L)) >>> 7) * L8_L) >>> 56)); // Step 6
return res;
}
/** A signed bytewise smaller &lt;<sub><small>8</small></sub> operator, for operands 0L<= x, y <=0x7L.
* This uses the following numbers of basic long operations: 1 or, 2 and, 2 xor, 1 minus, 1 not.
* @return A long with bits set in the {@link #H8_L} positions corresponding to each input signed byte pair that compares smaller.
*/
public static long smallerUpTo7_8(long x, long y) {
// See section 4, page 5, line 14 of the Vigna article:
return ( ( (x | H8_L) - (y & (~H8_L)) ) ^ x ^ ~y) & H8_L;
}
/** An unsigned bytewise smaller &lt;<sub><small>8</small></sub> operator.
* This uses the following numbers of basic long operations: 3 or, 2 and, 2 xor, 1 minus, 1 not.
* @return A long with bits set in the {@link #H8_L} positions corresponding to each input unsigned byte pair that compares smaller.
*/
public static long smalleru_8(long x, long y) {
// See section 4, 8th line from the bottom of the page 5, of the Vigna article:
return ( ( ( (x | H8_L) - (y & ~H8_L) ) | x ^ y) ^ (x | ~y) ) & H8_L;
}
/** An unsigned bytewise not equals 0 operator.
* This uses the following numbers of basic long operations: 2 or, 1 and, 1 minus.
* @return A long with bits set in the {@link #H8_L} positions corresponding to each unsigned byte that does not equal 0.
*/
public static long notEquals0_8(long x) {
// See section 4, line 6-8 on page 6, of the Vigna article:
return (((x | H8_L) - L8_L) | x) & H8_L;
}
/** A bytewise smaller &lt;<sub><small>16</small></sub> operator.
* This uses the following numbers of basic long operations: 1 or, 2 and, 2 xor, 1 minus, 1 not.
* @return A long with bits set in the {@link #H16_L} positions corresponding to each input signed short pair that compares smaller.
*/
public static long smallerUpto15_16(long x, long y) {
return ( ( (x | H16_L) - (y & (~H16_L)) ) ^ x ^ ~y) & H16_L;
}
/** Lk denotes the constant whose ones are in position 0, k, 2k, . . .
* These contain the low bit of each group of k bits.
* The suffix _L indicates the long implementation.
*/
public final static long L8_L = 0x0101010101010101L;
public final static long L9_L = 0x8040201008040201L;
public final static long L16_L = 0x0001000100010001L;
/** Hk = Lk << (k-1) .
* These contain the high bit of each group of k bits.
* The suffix _L indicates the long implementation.
*/
public final static long H8_L = L8_L << 7;
public final static long H16_L = L16_L << 15;
/**
* Naive implementation of {@link #select9(long,int)}, using {@link Long#numberOfTrailingZeros} repetitively.
* @return The index of the r-th 1 bit in x, or if no such bit exists, 72.
*/
public static int selectNaive(long x, int r) {
assert r >= 1;
int s = -1;
while ((x != 0L) && (r > 0)) {
int ntz = Long.numberOfTrailingZeros(x);
x >>>= (ntz + 1);
s += (ntz + 1);
r -= 1;
}
int res = (r > 0) ? 72 : s;
return res;
}
}

10
lucene/core/src/java/org/apache/lucene/util/ToStringUtils.java
View File

@ -43,4 +43,14 @@ public final class ToStringUtils {
} }
} }
private final static char [] HEX = "0123456789abcdef".toCharArray();
public static String longHex(long x) {
char [] asHex = new char [16];
for (int i = 16; --i >= 0; x >>>= 4) {
asHex[i] = HEX[(int) x & 0x0F];
}
return "0x" + new String(asHex);
}
} }

127
lucene/core/src/test/org/apache/lucene/util/TestBroadWord.java Normal file
View File

@ -0,0 +1,127 @@
package org.apache.lucene.util;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
public class TestBroadWord extends LuceneTestCase {
private void tstRank(long x) {
assertEquals("rank9(" + x + ")", Long.bitCount(x), BroadWord.rank9(x));
}
public void testRank1() {
tstRank(0L);
tstRank(1L);
tstRank(3L);
tstRank(0x100L);
tstRank(0x300L);
tstRank(0x8000000000000001L);
}
private void tstSelect(long x, int r, int exp) {
assertEquals("selectNaive(" + x + "," + r + ")", exp, BroadWord.selectNaive(x, r));
assertEquals("select9(" + x + "," + r + ")", exp, BroadWord.select9(x, r));
}
public void testSelectFromZero() {
tstSelect(0L,1,72);
}
public void testSelectSingleBit() {
for (int i = 0; i < 64; i++) {
tstSelect((1L << i),1,i);
}
}
public void testSelectTwoBits() {
for (int i = 0; i < 64; i++) {
for (int j = i+1; j < 64; j++) {
long x = (1L << i) | (1L << j);
//System.out.println(getName() + " i: " + i + " j: " + j);
tstSelect(x,1,i);
tstSelect(x,2,j);
tstSelect(x,3,72);
}
}
}
public void testSelectThreeBits() {
for (int i = 0; i < 64; i++) {
for (int j = i+1; j < 64; j++) {
for (int k = j+1; k < 64; k++) {
long x = (1L << i) | (1L << j) | (1L << k);
tstSelect(x,1,i);
tstSelect(x,2,j);
tstSelect(x,3,k);
tstSelect(x,4,72);
}
}
}
}
public void testSelectAllBits() {
for (int i = 0; i < 64; i++) {
tstSelect(0xFFFFFFFFFFFFFFFFL,i+1,i);
}
}
public void testPerfSelectAllBitsBroad() {
for (int j = 0; j < 100000; j++) { // 1000000 for real perf test
for (int i = 0; i < 64; i++) {
assertEquals(i, BroadWord.select9(0xFFFFFFFFFFFFFFFFL, i+1));
}
}
}
public void testPerfSelectAllBitsNaive() {
for (int j = 0; j < 10000; j++) { // real perftest: 1000000
for (int i = 0; i < 64; i++) {
assertEquals(i, BroadWord.selectNaive(0xFFFFFFFFFFFFFFFFL, i+1));
}
}
}
public void testSmalleru_87_01() {
// 0 <= arguments < 2 ** (k-1), k=8, see paper
for (long i = 0x0L; i <= 0x7FL; i++) {
for (long j = 0x0L; i <= 0x7FL; i++) {
long ii = i * BroadWord.L8_L;
long jj = j * BroadWord.L8_L;
assertEquals(ToStringUtils.longHex(ii) + " < " + ToStringUtils.longHex(jj),
ToStringUtils.longHex((i<j) ? (0x80L * BroadWord.L8_L) : 0x0L),
ToStringUtils.longHex(BroadWord.smallerUpTo7_8(ii,jj)));
}
}
}
public void testSmalleru_8_01() {
// 0 <= arguments < 2 ** k, k=8, see paper
for (long i = 0x0L; i <= 0xFFL; i++) {
for (long j = 0x0L; i <= 0xFFL; i++) {
long ii = i * BroadWord.L8_L;
long jj = j * BroadWord.L8_L;
assertEquals(ToStringUtils.longHex(ii) + " < " + ToStringUtils.longHex(jj),
ToStringUtils.longHex((i<j) ? (0x80L * BroadWord.L8_L): 0x0L),
ToStringUtils.longHex(BroadWord.smalleru_8(ii,jj)));
}
}
}
public void testNotEquals0_8() {
// 0 <= arguments < 2 ** k, k=8, see paper
for (long i = 0x0L; i <= 0xFFL; i++) {
long ii = i * BroadWord.L8_L;
assertEquals(ToStringUtils.longHex(ii) + " <> 0",
ToStringUtils.longHex((i != 0L) ? (0x80L * BroadWord.L8_L) : 0x0L),
ToStringUtils.longHex(BroadWord.notEquals0_8(ii)));
}
}
}