Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

LUCENE-9235: upgrade all python to python3 

Die, python2, die.

Some generated .java files change (parameterized automata for
spell-correction).

This is because the order of python dictionaries was not well-defined
previously. A sort() was added so that the python code now generates
reproducible output (Thanks @mikemccand).

So we'll suffer a change once, but the automata are equivalent. If you
run the script again you should not see source code changes.

The relevant unit tests are exhaustive (if you trust the paper!), so we can
be confident it does not break things, even though it looks very scary.
This commit is contained in:
Robert Muir 2020-02-20 21:27:38 -05:00
parent 79966132fc
commit 9302eee1e0
No known key found for this signature in database
GPG Key ID: 817AE1DD322D7ECA
10 changed files with 246 additions and 247 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
gradle/generation/util.gradle
View File

@ -33,7 +33,7 @@ configure(rootProject) {
task installMoman(type: Download) { task installMoman(type: Download) {
def momanZip = file("${momanDir}/moman.zip") def momanZip = file("${momanDir}/moman.zip")
src "https://bitbucket.org/jpbarrette/moman/get/5c5c2a1e4dea.zip" src "https://github.com/jpbarrette/moman/archive/497c90e34e412b6494db6dabf0d95db8034bd325.zip"
dest momanZip dest momanZip
onlyIfModified true onlyIfModified true
@ -57,7 +57,7 @@ configure(project(":lucene:core")) {
logger.lifecycle("Executing: ${prog} in ${targetDir}") logger.lifecycle("Executing: ${prog} in ${targetDir}")
project.exec { project.exec {
workingDir targetDir workingDir targetDir
executable "python2.7" executable "python3"
args = ['-B', "${prog}"] args = ['-B', "${prog}"]
} }
} }
@ -82,7 +82,7 @@ configure(project(":lucene:core")) {
['True', 'False'].each { transpose -> ['True', 'False'].each { transpose ->
project.exec { project.exec {
workingDir targetDir workingDir targetDir
executable "python2.7" executable "python3"
args = ['-B', 'createLevAutomata.py', num, transpose, "${momanDir}/finenight/python"] args = ['-B', 'createLevAutomata.py', num, transpose, "${momanDir}/finenight/python"]
} }
} }

2
lucene/analysis/common/build.xml
View File

@ -45,7 +45,7 @@
<target name="generate-jflex-html-char-entities"> <target name="generate-jflex-html-char-entities">
<exec dir="src/java/org/apache/lucene/analysis/charfilter" <exec dir="src/java/org/apache/lucene/analysis/charfilter"
output="src/java/org/apache/lucene/analysis/charfilter/HTMLCharacterEntities.jflex" output="src/java/org/apache/lucene/analysis/charfilter/HTMLCharacterEntities.jflex"
executable="${python2.exe}" failonerror="true" logerror="true"> executable="${python3.exe}" failonerror="true" logerror="true">
<!-- Tell Python not to write any bytecode cache into the filesystem: --> <!-- Tell Python not to write any bytecode cache into the filesystem: -->
<arg value="-B"/> <arg value="-B"/>
<arg value="htmlentity.py"/> <arg value="htmlentity.py"/>

2
lucene/common-build.xml
View File

@ -248,8 +248,6 @@
<property name="git.exe" value="git" /> <property name="git.exe" value="git" />
<property name="perl.exe" value="perl" /> <property name="perl.exe" value="perl" />
<!-- we default to python2.7 because not all OSs (e.g. mac) have a python2 link -->
<property name="python2.exe" value="python2.7" />
<property name="python3.exe" value="python3" /> <property name="python3.exe" value="python3" />
<property name="gpg.exe" value="gpg" /> <property name="gpg.exe" value="gpg" />

12
lucene/core/build.xml
View File

@ -26,8 +26,8 @@
<property name="javac.profile.args" value="-profile compact1"/> <property name="javac.profile.args" value="-profile compact1"/>
<import file="../common-build.xml"/> <import file="../common-build.xml"/>
<property name="moman.commit-hash" value="5c5c2a1e4dea" /> <property name="moman.commit-hash" value="497c90e34e412b6494db6dabf0d95db8034bd325" />
<property name="moman.url" value="https://bitbucket.org/jpbarrette/moman/get/${moman.commit-hash}.zip" /> <property name="moman.url" value="https://github.com/jpbarrette/moman/archive/${moman.commit-hash}.zip" />
<path id="classpath"/> <path id="classpath"/>
@ -69,7 +69,7 @@
<attribute name="n"/> <attribute name="n"/>
<sequential> <sequential>
<exec dir="src/java/org/apache/lucene/util/automaton" <exec dir="src/java/org/apache/lucene/util/automaton"
executable="${python2.exe}" failonerror="true"> executable="${python3.exe}" failonerror="true">
<!-- Tell Python not to write any bytecode cache into the filesystem: --> <!-- Tell Python not to write any bytecode cache into the filesystem: -->
<arg value="-B"/> <arg value="-B"/>
<arg value="createLevAutomata.py"/> <arg value="createLevAutomata.py"/>
@ -79,7 +79,7 @@
<arg value="../../../../../../../../build/core/moman/finenight/python"/> <arg value="../../../../../../../../build/core/moman/finenight/python"/>
</exec> </exec>
<exec dir="src/java/org/apache/lucene/util/automaton" <exec dir="src/java/org/apache/lucene/util/automaton"
executable="${python2.exe}" failonerror="true"> executable="${python3.exe}" failonerror="true">
<!-- Tell Python not to write any bytecode cache into the filesystem: --> <!-- Tell Python not to write any bytecode cache into the filesystem: -->
<arg value="-B"/> <arg value="-B"/>
<arg value="createLevAutomata.py"/> <arg value="createLevAutomata.py"/>
@ -94,13 +94,13 @@
<target name="createPackedIntSources"> <target name="createPackedIntSources">
<exec dir="src/java/org/apache/lucene/util/packed" <exec dir="src/java/org/apache/lucene/util/packed"
executable="${python2.exe}" failonerror="true"> executable="${python3.exe}" failonerror="true">
<!-- Tell Python not to write any bytecode cache into the filesystem: --> <!-- Tell Python not to write any bytecode cache into the filesystem: -->
<arg value="-B"/> <arg value="-B"/>
<arg value="gen_BulkOperation.py"/> <arg value="gen_BulkOperation.py"/>
</exec> </exec>
<exec dir="src/java/org/apache/lucene/util/packed" <exec dir="src/java/org/apache/lucene/util/packed"
executable="${python2.exe}" failonerror="true"> executable="${python3.exe}" failonerror="true">
<!-- Tell Python not to write any bytecode cache into the filesystem: --> <!-- Tell Python not to write any bytecode cache into the filesystem: -->
<arg value="-B"/> <arg value="-B"/>
<arg value="gen_Packed64SingleBlock.py"/> <arg value="gen_Packed64SingleBlock.py"/>

12
lucene/core/src/java/org/apache/lucene/util/automaton/Lev1ParametricDescription.java
View File

@ -88,26 +88,26 @@ class Lev1ParametricDescription extends ParametricDescription {
// 4 vectors; 5 states per vector; array length = 20 // 4 vectors; 5 states per vector; array length = 20
private final static long[] toStates2 = new long[] /*3 bits per value */ { private final static long[] toStates2 = new long[] /*3 bits per value */ {
0x69a292450428003L 0x4da292442420003L
}; };
private final static long[] offsetIncrs2 = new long[] /*2 bits per value */ { private final static long[] offsetIncrs2 = new long[] /*2 bits per value */ {
0x5555588000L 0x5555528000L
}; };
// 8 vectors; 5 states per vector; array length = 40 // 8 vectors; 5 states per vector; array length = 40
private final static long[] toStates3 = new long[] /*3 bits per value */ { private final static long[] toStates3 = new long[] /*3 bits per value */ {
0x1690a82152018003L,0xb1a2d346448a49L 0x14d0812112018003L,0xb1a29b46d48a49L
}; };
private final static long[] offsetIncrs3 = new long[] /*2 bits per value */ { private final static long[] offsetIncrs3 = new long[] /*2 bits per value */ {
0x555555b8220f0000L,0x5555L 0x555555e80a0f0000L,0x5555L
}; };
// state map // state map
// 0 -> [(0, 0)] // 0 -> [(0, 0)]
// 1 -> [(0, 1)] // 1 -> [(0, 1)]
// 2 -> [(0, 1), (1, 1)] // 2 -> [(0, 1), (1, 1)]
// 3 -> [(0, 1), (2, 1)] // 3 -> [(0, 1), (1, 1), (2, 1)]
// 4 -> [(0, 1), (1, 1), (2, 1)] // 4 -> [(0, 1), (2, 1)]
public Lev1ParametricDescription(int w) { public Lev1ParametricDescription(int w) {

14
lucene/core/src/java/org/apache/lucene/util/automaton/Lev1TParametricDescription.java
View File

@ -89,27 +89,27 @@ class Lev1TParametricDescription extends ParametricDescription {
// 4 vectors; 6 states per vector; array length = 24 // 4 vectors; 6 states per vector; array length = 24
private final static long[] toStates2 = new long[] /*3 bits per value */ { private final static long[] toStates2 = new long[] /*3 bits per value */ {
0x3453491482140003L,0x6dL 0xb45a491412180003L,0x69L
}; };
private final static long[] offsetIncrs2 = new long[] /*2 bits per value */ { private final static long[] offsetIncrs2 = new long[] /*2 bits per value */ {
0x555555a20000L 0x5555558a0000L
}; };
// 8 vectors; 6 states per vector; array length = 48 // 8 vectors; 6 states per vector; array length = 48
private final static long[] toStates3 = new long[] /*3 bits per value */ { private final static long[] toStates3 = new long[] /*3 bits per value */ {
0x21520854900c0003L,0x5b4d19a24534916dL,0xda34L 0xa1904864900c0003L,0x5a6d196a45a49169L,0x9634L
}; };
private final static long[] offsetIncrs3 = new long[] /*2 bits per value */ { private final static long[] offsetIncrs3 = new long[] /*2 bits per value */ {
0x5555ae0a20fc0000L,0x55555555L 0x5555ba08a0fc0000L,0x55555555L
}; };
// state map // state map
// 0 -> [(0, 0)] // 0 -> [(0, 0)]
// 1 -> [(0, 1)] // 1 -> [(0, 1)]
// 2 -> [(0, 1), (1, 1)] // 2 -> [(0, 1), (1, 1)]
// 3 -> [(0, 1), (2, 1)] // 3 -> [(0, 1), (1, 1), (2, 1)]
// 4 -> [t(0, 1), (0, 1), (1, 1), (2, 1)] // 4 -> [(0, 1), (2, 1)]
// 5 -> [(0, 1), (1, 1), (2, 1)] // 5 -> [t(0, 1), (0, 1), (1, 1), (2, 1)]
public Lev1TParametricDescription(int w) { public Lev1TParametricDescription(int w) {

148
lucene/core/src/java/org/apache/lucene/util/automaton/Lev2ParametricDescription.java
View File

@ -84,7 +84,7 @@ class Lev2ParametricDescription extends ParametricDescription {
// 1 vectors; 3 states per vector; array length = 3 // 1 vectors; 3 states per vector; array length = 3
private final static long[] toStates0 = new long[] /*2 bits per value */ { private final static long[] toStates0 = new long[] /*2 bits per value */ {
0x23L 0xeL
}; };
private final static long[] offsetIncrs0 = new long[] /*1 bits per value */ { private final static long[] offsetIncrs0 = new long[] /*1 bits per value */ {
0x0L 0x0L
@ -92,7 +92,7 @@ class Lev2ParametricDescription extends ParametricDescription {
// 2 vectors; 5 states per vector; array length = 10 // 2 vectors; 5 states per vector; array length = 10
private final static long[] toStates1 = new long[] /*3 bits per value */ { private final static long[] toStates1 = new long[] /*3 bits per value */ {
0x13688b44L 0x1a688a2cL
}; };
private final static long[] offsetIncrs1 = new long[] /*1 bits per value */ { private final static long[] offsetIncrs1 = new long[] /*1 bits per value */ {
0x3e0L 0x3e0L
@ -100,41 +100,41 @@ class Lev2ParametricDescription extends ParametricDescription {
// 4 vectors; 11 states per vector; array length = 44 // 4 vectors; 11 states per vector; array length = 44
private final static long[] toStates2 = new long[] /*4 bits per value */ { private final static long[] toStates2 = new long[] /*4 bits per value */ {
0x26a09a0a0520a504L,0x2323523321a260a2L,0x354235543213L 0x3a07603570707054L,0x522323232103773aL,0x352254543213L
}; };
private final static long[] offsetIncrs2 = new long[] /*2 bits per value */ { private final static long[] offsetIncrs2 = new long[] /*2 bits per value */ {
0x5555520280000800L,0x555555L 0x5555520880080000L,0x555555L
}; };
// 8 vectors; 21 states per vector; array length = 168 // 8 vectors; 21 states per vector; array length = 168
private final static long[] toStates3 = new long[] /*5 bits per value */ { private final static long[] toStates3 = new long[] /*5 bits per value */ {
0x380e014a051404L,0xe28245009451140L,0x8a26880098a6268cL,0x180a288ca0246213L, 0x7000a560180380a4L,0xc015a0180a0194aL,0x8032c58318a301c0L,0x9d8350d403980318L,
0x494053284a1080e1L,0x510265a89c311940L,0x4218c41188a6509cL,0x6340c4211c4710dL, 0x3006028ca73a8602L,0xc51462640b21a807L,0x2310c4100c62194eL,0xce35884218ce248dL,
0xa168398471882a12L,0x104c841c683a0425L,0x3294472904351483L,0xe6290620a84a20d0L, 0xa9285a0691882358L,0x1046b5a86b1252b5L,0x2110a33892521483L,0xe62906208d63394eL,
0x1441a0ea2896a4a0L,0x32L 0xd6a29c4921d6a4a0L,0x1aL
}; };
private final static long[] offsetIncrs3 = new long[] /*2 bits per value */ { private final static long[] offsetIncrs3 = new long[] /*2 bits per value */ {
0x33300230c0000800L,0x220ca080a00fc330L,0x555555f832823380L,0x5555555555555555L, 0xf0c000c8c0080000L,0xca808822003f303L,0x5555553fa02f0880L,0x5555555555555555L,
0x5555555555555555L,0x5555L 0x5555555555555555L,0x5555L
}; };
// 16 vectors; 30 states per vector; array length = 480 // 16 vectors; 30 states per vector; array length = 480
private final static long[] toStates4 = new long[] /*5 bits per value */ { private final static long[] toStates4 = new long[] /*5 bits per value */ {
0x380e014a051404L,0xaa015452940L,0x55014501000000L,0x1843ddc771085c07L, 0x7000a560180380a4L,0xa000000280e0294aL,0x6c0b00e029000000L,0x8c4350c59cdc6039L,
0x7141200040108405L,0x52b44004c5313460L,0x401080200063115cL,0x85314c4d181c5048L, 0x600ad00c03380601L,0x2962c18c5180e00L,0x18c4000c6028c4L,0x8a314603801802b4L,
0x1440190a3e5c7828L,0x28a232809100a21L,0xa028ca2a84203846L,0xca0240010800108aL, 0x6328c4520c59c5L,0x60d43500e600c651L,0x280e339cea180a7L,0x4039800000a318c6L,
0xc7b4205c1580a508L,0x1021090251846b6L,0x4cb513862328090L,0x210863128ca2b8a2L, 0xd57be96039ec3d0dL,0xc0338d6358c4352L,0x28c4c81643500e60L,0x3194a028c4339d8aL,
0x4e188ca024402940L,0xa6b6c7c520532d4L,0x8c41101451150219L,0xa0c4211c4710d421L, 0x590d403980018c4L,0xc4522d57b68e3132L,0xc4100c6510d6538L,0x9884218ce248d231L,
0x2108421094e15063L,0x8f13c43708631044L,0x18274d908c611631L,0x1cc238c411098263L, 0x318ce318c6398d83L,0xa3609c370c431046L,0xea3ad6958568f7beL,0x2d0348c411d47560L,
0x450e3a1d0212d0b4L,0x31050242048108c6L,0xfa318b42d07308eL,0xa8865182356907c6L, 0x9ad43989295ad494L,0x3104635ad431ad63L,0x8f73a6b5250b40d2L,0x57350eab9d693956L,
0x1ca410d4520c4140L,0x2954e13883a0ca51L,0x3714831044229442L,0x93946116b58f2c84L, 0x8ce24948520c411dL,0x294a398d85608442L,0x5694831046318ce5L,0x958460f7b623609cL,
0xc41109a5631a574dL,0x1d4512d4941cc520L,0x52848294c643883aL,0xb525073148310502L, 0xc411d475616258d6L,0x9243ad4941cc520L,0x5ad4529ce39ad456L,0xb525073148310463L,
0xa5356939460f7358L,0x409ca651L 0x27656939460f7358L,0x1d573516L
}; };
private final static long[] offsetIncrs4 = new long[] /*3 bits per value */ { private final static long[] offsetIncrs4 = new long[] /*3 bits per value */ {
0x20c0600000010000L,0x2000040000000001L,0x209204a40209L,0x301b6c0618018618L, 0x610600010000000L,0x2040000000001000L,0x1044209245200L,0x80d86d86006d80c0L,
0x207206186000186cL,0x1200061b8e06dc0L,0x480492080612010L,0xa20204a040048000L, 0x2001b6030000006dL,0x8200011b6237237L,0x12490612400410L,0x2449001040208000L,
0x1061a0000129124L,0x1848349b680612L,0xd26da0204a041868L,0x2492492492496128L, 0x4d80820001044925L,0x6da4906da400L,0x9252369001360208L,0x24924924924911b6L,
0x9249249249249249L,0x4924924924924924L,0x2492492492492492L,0x9249249249249249L, 0x9249249249249249L,0x4924924924924924L,0x2492492492492492L,0x9249249249249249L,
0x4924924924924924L,0x2492492492492492L,0x9249249249249249L,0x4924924924924924L, 0x4924924924924924L,0x2492492492492492L,0x9249249249249249L,0x4924924924924924L,
0x2492492492492492L,0x9249249249249249L,0x24924924L 0x2492492492492492L,0x9249249249249249L,0x24924924L
@ -142,33 +142,33 @@ class Lev2ParametricDescription extends ParametricDescription {
// 32 vectors; 30 states per vector; array length = 960 // 32 vectors; 30 states per vector; array length = 960
private final static long[] toStates5 = new long[] /*5 bits per value */ { private final static long[] toStates5 = new long[] /*5 bits per value */ {
0x380e014a051404L,0xaa015452940L,0x8052814501000000L,0xb80a515450000e03L, 0x7000a560180380a4L,0xa000000280e0294aL,0x580600e029000000L,0x80e0600e529c0029L,
0x5140410842108426L,0x71dc421701c01540L,0x100421014610f7L,0x85c0700550145010L, 0x380a418c6388c631L,0x316737180e5b02c0L,0x300ce01806310d4L,0xc60396c0b00e0290L,
0x94a271843ddc7710L,0x1346071412108a22L,0x3115c52b44004c53L,0xc504840108020006L, 0xca328c4350c59cdL,0x80e00600ad194656L,0x28c402962c18c51L,0x802b40018c4000c6L,
0x54d1001314c4d181L,0x9081204239c4a71L,0x14c5313460714124L,0x51006428f971e0a2L, 0xe58b06314603801L,0x8d6b48c6b580e348L,0x28c5180e00600ad1L,0x18ca31148316716L,
0x4d181c5048402884L,0xa3e5c782885314cL,0x2809409482a8a239L,0x2a84203846028a23L, 0x3801802b4031944L,0xc4520c59c58a3146L,0xe61956748cab38L,0x39cea180a760d435L,
0x10800108aa028caL,0xe1180a288ca0240L,0x98c6b80e3294a108L,0x2942328091098c10L, 0xa318c60280e3L,0x6029d8350d403980L,0x6b5a80e060d873a8L,0xf43500e618c638dL,
0x11adb1ed08170560L,0xa024004084240946L,0x7b4205c1580a508cL,0xa8c2968c71846b6cL, 0x10d4b55efa580e7bL,0x3980300ce358d63L,0x57be96039ec3d0d4L,0x4656567598c4352dL,
0x4cb5138623280910L,0x10863128ca2b8a20L,0xe188ca0244029402L,0x4e3294e288132d44L, 0x8c4c81643500e619L,0x194a028c4339d8a2L,0x590d403980018c43L,0xe348d87628a31320L,
0x809409ad1218c39cL,0xf14814cb51386232L,0x514454086429adb1L,0x32d44e188ca02440L, 0xe618d6b4d6b1880L,0x5eda38c4c8164350L,0x19443594e31148b5L,0x31320590d4039803L,
0x8c390a6b6c7c5205L,0xd4218c41409cd2aaL,0x5063a0c4211c4710L,0x10442108421094e1L, 0x7160c4522d57b68eL,0xd2310c41195674d6L,0x8d839884218ce248L,0x1046318ce318c639L,
0x31084711c4350863L,0xbdef7bddf05918f2L,0xc4f10dc218c41ef7L,0x9d3642318458c63L, 0x2108633892348c43L,0xdebfbdef0f63b0f6L,0xd8270dc310c41f7bL,0x8eb5a5615a3defa8L,
0x70863104426098c6L,0x8c6116318f13c43L,0x41ef75dd6b5de4d9L,0xd0212d0b41cc238cL, 0x70c43104751d583aL,0x58568f7bea3609c3L,0x41f77ddb7bbeed69L,0x9295ad4942d0348cL,
0x2048108c6450e3a1L,0x42d07308e3105024L,0xdb591938f274084bL,0xc238c41f77deefbbL, 0xad431ad639ad4398L,0x5250b40d23104635L,0xce0f6bd0f624a56bL,0x348c41f7b9cd7bdL,
0x1f183e8c62d0b41cL,0x502a2194608d5a4L,0xa318b42d07308e31L,0xed675db56907c60fL, 0xe55a3dce9ad4942dL,0x4755cd43aae75a4L,0x73a6b5250b40d231L,0xbd7bbcdd6939568fL,
0xa410d4520c41f773L,0x54e13883a0ca511cL,0x1483104422944229L,0x20f2329447290435L, 0xe24948520c41f779L,0x4a398d856084428cL,0x14831046318ce529L,0xb16c2110a3389252L,
0x1ef6f7ef6f7df05cL,0xad63cb210dc520c4L,0x58c695d364e51845L,0xc843714831044269L, 0x1f7bdebe739c8f63L,0xed88d82715a520c4L,0x58589635a561183dL,0x9c569483104751dL,
0xe4d93946116b58f2L,0x520c41ef717d6b17L,0x83a1d4512d4941ccL,0x50252848294c6438L, 0xc56958460f7b6236L,0x520c41f77ddb6719L,0x45609243ad4941ccL,0x4635ad4529ce39adL,
0x144b525073148310L,0xefaf7b591c20f275L,0x941cc520c41f777bL,0xd5a4e5183dcd62d4L, 0x90eb525073148310L,0xd6737b8f6bd16c24L,0x941cc520c41f7b9cL,0x95a4e5183dcd62d4L,
0x4831050272994694L,0x460f7358b5250731L,0xf779bd6717b56939L 0x483104755cd4589dL,0x460f7358b5250731L,0xf779bd6717b56939L
}; };
private final static long[] offsetIncrs5 = new long[] /*3 bits per value */ { private final static long[] offsetIncrs5 = new long[] /*3 bits per value */ {
0x20c0600000010000L,0x40000000001L,0xb6db6d4830180L,0x4812900824800010L, 0x610600010000000L,0x40000000001000L,0xb6d56da184180L,0x824914800810000L,
0x2092000040000082L,0x618000b659254a40L,0x86c301b6c0618018L,0xdb01860061860001L, 0x2002040000000411L,0xc0000b2c5659245L,0x6d80d86d86006d8L,0x1b61801b60300000L,
0x81861800075baed6L,0x186e381b70081cL,0xe56dc02072061860L,0x61201001200075b8L, 0x6d80c0000b5b76b6L,0x46d88dc8dc800L,0x6372372001b60300L,0x400410082000b1b7L,
0x480000480492080L,0x52b5248201848040L,0x880812810012000bL,0x4004800004a4492L, 0x2080000012490612L,0x6d49241849001040L,0x912400410082000bL,0x402080004112494L,
0xb529124a20204aL,0x49b68061201061a0L,0x8480418680018483L,0x1a000752ad26da01L, 0xb2c49252449001L,0x4906da4004d80820L,0x136020800006daL,0x82000b5b69241b69L,
0x4a349b6808128106L,0xa0204a0418680018L,0x492492497528d26dL,0x2492492492492492L, 0x6da4948da4004d80L,0x3690013602080004L,0x49249249b1b69252L,0x2492492492492492L,
0x9249249249249249L,0x4924924924924924L,0x2492492492492492L,0x9249249249249249L, 0x9249249249249249L,0x4924924924924924L,0x2492492492492492L,0x9249249249249249L,
0x4924924924924924L,0x2492492492492492L,0x9249249249249249L,0x4924924924924924L, 0x4924924924924924L,0x2492492492492492L,0x9249249249249249L,0x4924924924924924L,
0x2492492492492492L,0x9249249249249249L,0x4924924924924924L,0x2492492492492492L, 0x2492492492492492L,0x9249249249249249L,0x4924924924924924L,0x2492492492492492L,
@ -179,38 +179,38 @@ class Lev2ParametricDescription extends ParametricDescription {
// state map // state map
// 0 -> [(0, 0)] // 0 -> [(0, 0)]
// 1 -> [(0, 2)] // 1 -> [(0, 1)]
// 2 -> [(0, 1)] // 2 -> [(0, 2)]
// 3 -> [(0, 1), (1, 1)] // 3 -> [(0, 1), (1, 1)]
// 4 -> [(0, 2), (1, 2)] // 4 -> [(0, 2), (1, 2)]
// 5 -> [(0, 2), (2, 1)] // 5 -> [(0, 1), (1, 1), (2, 1)]
// 6 -> [(0, 1), (2, 2)] // 6 -> [(0, 2), (1, 2), (2, 2)]
// 7 -> [(0, 2), (2, 2)] // 7 -> [(0, 1), (2, 1)]
// 8 -> [(0, 1), (1, 1), (2, 1)] // 8 -> [(0, 1), (2, 2)]
// 9 -> [(0, 2), (1, 2), (2, 2)] // 9 -> [(0, 2), (2, 1)]
// 10 -> [(0, 1), (2, 1)] // 10 -> [(0, 2), (2, 2)]
// 11 -> [(0, 2), (3, 2)] // 11 -> [(0, 2), (1, 2), (2, 2), (3, 2)]
// 12 -> [(0, 2), (1, 2), (3, 2)] // 12 -> [(0, 1), (1, 1), (3, 2)]
// 13 -> [(0, 2), (1, 2), (2, 2), (3, 2)] // 13 -> [(0, 1), (2, 2), (3, 2)]
// 14 -> [(0, 1), (2, 2), (3, 2)] // 14 -> [(0, 1), (3, 2)]
// 15 -> [(0, 2), (3, 1)] // 15 -> [(0, 2), (1, 2), (3, 1)]
// 16 -> [(0, 1), (3, 2)] // 16 -> [(0, 2), (1, 2), (3, 2)]
// 17 -> [(0, 1), (1, 1), (3, 2)] // 17 -> [(0, 2), (2, 1), (3, 1)]
// 18 -> [(0, 2), (1, 2), (3, 1)] // 18 -> [(0, 2), (2, 2), (3, 2)]
// 19 -> [(0, 2), (2, 2), (3, 2)] // 19 -> [(0, 2), (3, 1)]
// 20 -> [(0, 2), (2, 1), (3, 1)] // 20 -> [(0, 2), (3, 2)]
// 21 -> [(0, 2), (2, 1), (4, 2)] // 21 -> [(0, 2), (1, 2), (2, 2), (3, 2), (4, 2)]
// 22 -> [(0, 2), (1, 2), (4, 2)] // 22 -> [(0, 2), (1, 2), (2, 2), (4, 2)]
// 23 -> [(0, 2), (1, 2), (3, 2), (4, 2)] // 23 -> [(0, 2), (1, 2), (3, 2), (4, 2)]
// 24 -> [(0, 2), (2, 2), (3, 2), (4, 2)] // 24 -> [(0, 2), (1, 2), (4, 2)]
// 25 -> [(0, 2), (3, 2), (4, 2)] // 25 -> [(0, 2), (2, 1), (4, 2)]
// 26 -> [(0, 2), (1, 2), (2, 2), (4, 2)] // 26 -> [(0, 2), (2, 2), (3, 2), (4, 2)]
// 27 -> [(0, 2), (1, 2), (2, 2), (3, 2), (4, 2)] // 27 -> [(0, 2), (2, 2), (4, 2)]
// 28 -> [(0, 2), (4, 2)] // 28 -> [(0, 2), (3, 2), (4, 2)]
// 29 -> [(0, 2), (2, 2), (4, 2)] // 29 -> [(0, 2), (4, 2)]
public Lev2ParametricDescription(int w) { public Lev2ParametricDescription(int w) {
super(w, 2, new int[] {0,2,1,0,1,-1,0,0,-1,0,-1,-1,-1,-1,-1,-2,-1,-1,-2,-1,-2,-2,-2,-2,-2,-2,-2,-2,-2,-2}); super(w, 2, new int[] {0,1,2,0,1,-1,0,-1,0,-1,0,-1,-1,-1,-1,-2,-1,-2,-1,-2,-1,-2,-2,-2,-2,-2,-2,-2,-2,-2});
} }
} }

228
lucene/core/src/java/org/apache/lucene/util/automaton/Lev2TParametricDescription.java
View File

@ -85,7 +85,7 @@ class Lev2TParametricDescription extends ParametricDescription {
// 1 vectors; 3 states per vector; array length = 3 // 1 vectors; 3 states per vector; array length = 3
private final static long[] toStates0 = new long[] /*2 bits per value */ { private final static long[] toStates0 = new long[] /*2 bits per value */ {
0x23L 0xeL
}; };
private final static long[] offsetIncrs0 = new long[] /*1 bits per value */ { private final static long[] offsetIncrs0 = new long[] /*1 bits per value */ {
0x0L 0x0L
@ -93,7 +93,7 @@ class Lev2TParametricDescription extends ParametricDescription {
// 2 vectors; 5 states per vector; array length = 10 // 2 vectors; 5 states per vector; array length = 10
private final static long[] toStates1 = new long[] /*3 bits per value */ { private final static long[] toStates1 = new long[] /*3 bits per value */ {
0x13688b44L 0x1a688a2cL
}; };
private final static long[] offsetIncrs1 = new long[] /*1 bits per value */ { private final static long[] offsetIncrs1 = new long[] /*1 bits per value */ {
0x3e0L 0x3e0L
@ -101,51 +101,51 @@ class Lev2TParametricDescription extends ParametricDescription {
// 4 vectors; 13 states per vector; array length = 52 // 4 vectors; 13 states per vector; array length = 52
private final static long[] toStates2 = new long[] /*4 bits per value */ { private final static long[] toStates2 = new long[] /*4 bits per value */ {
0x60dbb0b05200b504L,0x5233217627062227L,0x2355543214323235L,0x4354L 0xdc0703570707054L,0x2323213a03dd3a3aL,0x2254543215435223L,0x5435L
}; };
private final static long[] offsetIncrs2 = new long[] /*2 bits per value */ { private final static long[] offsetIncrs2 = new long[] /*2 bits per value */ {
0x555080a800002000L,0x5555555555L 0x5558208800080000L,0x5555555555L
}; };
// 8 vectors; 28 states per vector; array length = 224 // 8 vectors; 28 states per vector; array length = 224
private final static long[] toStates3 = new long[] /*5 bits per value */ { private final static long[] toStates3 = new long[] /*5 bits per value */ {
0xe701c02940059404L,0xa010162000a50000L,0xb02c8c40a1416288L,0xa821032310858c0L, 0x700a5701c0380a4L,0x180a000ca529c0L,0xc5498e60a80af180L,0x8c4300e85a546398L,
0x314423980d28b201L,0x5281e528847788e0L,0xa23980d308c2280eL,0x1e3294b1a962278cL, 0xd8d43501ac18c601L,0x51976d6a863500adL,0xc3501ac28ca0180aL,0x76dda8a5b0c5be16L,
0x8c41309e2288e528L,0x11444409021aca21L,0x11a4624886b1086bL,0x2a6258941d6240c4L, 0xc41294a018c4519L,0x1086520ce248d231L,0x13946358ce31ac42L,0x6732d4942d0348c4L,
0x5024a50b489074adL,0x14821aca520c411aL,0x5888b5890b594a44L,0x941d6520c411a465L, 0xd635ad4b1ad224a5L,0xce24948520c4139L,0x58ce729d22110a52L,0x941cc520c41394e3L,
0x8b589075ad6a62d4L,0x1a5055a4L 0x4729d22490e732d4L,0x39ce35adL
}; };
private final static long[] offsetIncrs3 = new long[] /*2 bits per value */ { private final static long[] offsetIncrs3 = new long[] /*2 bits per value */ {
0x30c30200002000L,0x2a0030f3c3fc333cL,0x233a00328282a820L,0x5555555532b283a8L, 0xc0c83000080000L,0x2200fcff300f3c30L,0x3c2200a8caa00a08L,0x55555555a8fea00aL,
0x5555555555555555L,0x5555555555555555L,0x5555555555555555L 0x5555555555555555L,0x5555555555555555L,0x5555555555555555L
}; };
// 16 vectors; 45 states per vector; array length = 720 // 16 vectors; 45 states per vector; array length = 720
private final static long[] toStates4 = new long[] /*6 bits per value */ { private final static long[] toStates4 = new long[] /*6 bits per value */ {
0x3801450002c5004L,0xc500014b00000e38L,0x51451401402L,0x0L, 0x1453803801c0144L,0xc000514514700038L,0x1400001401L,0x140000L,
0x518000b14010000L,0x9f1c20828e20230L,0x219f0df0830a70c2L,0x8200008208208200L, 0x6301f00700510000L,0xa186178301f00d1L,0xc20c30c20ca0c3L,0xc00c00cd0c30030cL,
0x805050160800800L,0x3082098602602643L,0x4564014250508064L,0x850051420000831L, 0x4c054014f0c00c30L,0x55150c34c30944c3L,0x430c014308300550L,0xc30850c00050c31L,
0x4140582085002082L,0x456180980990c201L,0x8316d0c50a01051L,0x21451420050df0e0L, 0x50053c50c3143000L,0x850d30c25130d301L,0xc21441430a08608L,0x2145003143142145L,
0xd14214014508214L,0x3c21c01850821c60L,0x1cb1403cb142087L,0x800821451851822cL, 0x4c1431451400c314L,0x28014d6c32832803L,0x1c50c76cd34a0c3L,0x430c30c31c314014L,
0x20020820800020L,0xd006182087180345L,0xcb0a81cb24976b09L,0x8b1a60e624709d1L, 0xc30050000001431L,0xd36d0e40ca00d303L,0xcb2abb2c90b0e400L,0x2c32ca2c70c20ca1L,
0x249082082249089L,0xc31421c600d2c024L,0x3c31451515454423L,0x31853c22c21cb140L, 0x31c00c00cd2c70cbL,0x558328034c2c32cL,0x6cd6ca14558309b7L,0x51c51401430850c7L,
0x4514500b2c208214L,0x8718034508b0051L,0xb2cb45515108f0c5L,0xe824715d1cb0a810L, 0xc30871430c714L,0xca00d3071451450L,0xb9071560c26dc156L,0xc70c21441cb2abb2L,
0x1422cb14908b0e60L,0x30812c22c02cb145L,0x842022020cb1420cL,0x5c20ce0820ce0850L, 0x1421c70cb1c51ca1L,0x30811c51c51c00c3L,0xc51031c224324308L,0x5c33830d70820820L,
0x208208208b0d70c2L,0x4208508214214208L,0x920834050830c20L,0xc6134dc613653592L, 0x30c30c30c33850c3L,0x451450c30c30c31cL,0xda0920d20c20c20L,0x365961145145914fL,
0xd309341c6dc4db4dL,0x6424d90854d34d34L,0x92072c22030814c2L,0x4220724b24a30930L, 0xd964365351965865L,0x51964364365a6590L,0x920b203243081505L,0xd72422492c718b28L,
0x2470d72025c920e2L,0x92c92d70975c9082L,0xcb0880c204924e08L,0x45739728c24c2481L, 0x2cb3872c35cb28b0L,0xb0c32cb2972c30d7L,0xc80c90c204e1c75cL,0x4504171c62ca2482L,
0xc6da4db5da6174daL,0x4b5d35d75d30971dL,0x1030815c93825ce2L,0x51442051020cb145L, 0x33976585d65d9610L,0x4b5ca5d70d95cb5dL,0x1030813873975c36L,0x41451031c2245105L,
0xc538210e2c220e2cL,0x851421452cb0d70L,0x204b085085145142L,0x921560834051440cL, 0xc35c338714e24208L,0x1c51c51451453851L,0x20451450c70c30c3L,0x4f0da09214f1440cL,
0x4d660e4da60e6595L,0x94d914e41c6dc658L,0x826426591454d365L,0x2892072c51030813L, 0x6533944d04513d41L,0xe15450551350e658L,0x551938364365a50L,0x2892071851030815L,
0xe2c22072cb2ca30bL,0x452c70d720538910L,0x8b2cb2d708e3891L,0x81cb1440c204b24eL, 0x714e2422441c718bL,0x4e1c73871c35cb28L,0x5c70c32cb28e1c51L,0x81c61440c204e1c7L,
0xda44e38e28c2ca24L,0x1dc6da6585d660e4L,0xe2cb5d338e5d914eL,0x38938238L 0xd04503ce1c62ca24L,0x39338e6585d63944L,0x364b5ca38e154387L,0x38739738L
}; };
private final static long[] offsetIncrs4 = new long[] /*3 bits per value */ { private final static long[] offsetIncrs4 = new long[] /*3 bits per value */ {
0x3002000000080000L,0x20c060L,0x8149000004000000L,0x4024924110824824L, 0xc0000010000000L,0x40000060061L,0x8001000800000000L,0x8229048249248a4L,
0xdb6030c360002082L,0x6c36c06c301b0d80L,0xb01861b0000db0dbL,0x1b7036209188e06dL, 0x6c360300002092L,0x6db6036db61b6c30L,0x361b0180000db6c0L,0xdb11b71b91b72000L,
0x800920006d86db7L,0x4920c2402402490L,0x49000208249009L,0x4908128128124804L, 0x100820006db6236L,0x2492490612480012L,0x8041000248200049L,0x4924a48924000900L,
0x34800104124a44a2L,0xc30930900d24020cL,0x40009a0924c24d24L,0x4984a069201061aL, 0x2080012510822492L,0x9241b69200048360L,0x4000926806da4924L,0x291b49000241b010L,
0x494d049271269262L,0x2492492492492492L,0x9249249249249249L,0x4924924924924924L, 0x494934236d249249L,0x2492492492492492L,0x9249249249249249L,0x4924924924924924L,
0x2492492492492492L,0x9249249249249249L,0x4924924924924924L,0x2492492492492492L, 0x2492492492492492L,0x9249249249249249L,0x4924924924924924L,0x2492492492492492L,
0x9249249249249249L,0x4924924924924924L,0x2492492492492492L,0x9249249249249249L, 0x9249249249249249L,0x4924924924924924L,0x2492492492492492L,0x9249249249249249L,
0x4924924924924924L,0x2492492492492492L,0x9249249249249249L,0x4924924924924924L, 0x4924924924924924L,0x2492492492492492L,0x9249249249249249L,0x4924924924924924L,
@ -154,51 +154,51 @@ class Lev2TParametricDescription extends ParametricDescription {
// 32 vectors; 45 states per vector; array length = 1440 // 32 vectors; 45 states per vector; array length = 1440
private final static long[] toStates5 = new long[] /*6 bits per value */ { private final static long[] toStates5 = new long[] /*6 bits per value */ {
0x3801450002c5004L,0xc500014b00000e38L,0x51451401402L,0x0L, 0x1453803801c0144L,0xc000514514700038L,0x1400001401L,0x140000L,
0x514000b14010000L,0x550000038e00e0L,0x264518500600b180L,0x8208208208208208L, 0x4e00e00700510000L,0x3451451c000e0051L,0x30cd00000d015000L,0xc30c30d40c30c30cL,
0x2c50040820820L,0x70820a38808c0146L,0xc37c20c29c30827cL,0x20820820800867L, 0x7c01c01440c30c30L,0x185e0c07c03458c0L,0x830c30832830c286L,0x33430c00c30030L,
0xb140102002002080L,0x828e202300518000L,0x830a70c209f1c20L,0x51451450853df0dfL, 0x70051030030c3003L,0x8301f00d16301f00L,0xc20ca0c30a18617L,0xb1450c51431420c3L,
0x1614214214508214L,0x6026026430805050L,0x2505080643082098L,0x4200008314564014L, 0x4f14314514314315L,0x4c30944c34c05401L,0x30830055055150c3L,0xc00050c31430c014L,
0x850020820850051L,0x80990c2014140582L,0x8201920208261809L,0x892051990060941L, 0xc31430000c30850L,0x25130d30150053c5L,0xc03541545430d30cL,0x1cb2cd0c300d0c90L,
0x22492492c22cb242L,0x430805050162492cL,0x8041451586026026L,0x37c38020c5b43142L, 0x72c30cb2c91cb0c3L,0xc34c054014f1cb2cL,0x8218221434c30944L,0x50851430851050c2L,
0x4208508514508014L,0x141405850850051L,0x51456180980990c2L,0xe008316d0c50a010L, 0x30c50851400c50cL,0x150053c50c51450L,0x8850d30c25130d3L,0x450c21441430a086L,
0x2c52cb2c508b21f0L,0x600d2c92c22cb249L,0x873c21c01850821cL,0x2c01cb1403cb1420L, 0x1c91c70c51cb1c21L,0x34c1cb1c71c314bL,0xc328014d6c328328L,0x1401c50c76cd34a0L,
0x2080082145185182L,0x4500200208208000L,0x870061420871803L,0x740500f5050821cfL, 0x31430c30c31c3140L,0x30c300500000014L,0x535b0ca0ca00d3L,0x514369b34d2830caL,
0x934d964618609000L,0x4c24d34d30824d30L,0x1860821c600d642L,0xc2a072c925dac274L, 0x5965965a0c500d01L,0x6435030c30d46546L,0xdb4390328034c659L,0xcaaecb242c390034L,
0x2c69839891c27472L,0x9242082089242242L,0x8208718034b00900L,0x1cb24976b09d0061L, 0xcb28b1c30832872L,0x700300334b1c32cbL,0xe40ca00d30b0cb0cL,0xb2c90b0e400d36d0L,
0x60e624709d1cb0a8L,0xd31455d71574ce3eL,0x1c600d3825c25d74L,0x51515454423c3142L, 0xa2c70c20ca1cb2abL,0x4315b5ce6575d95cL,0x28034c5d95c53831L,0xa14558309b705583L,
0xc22c21cb1403c314L,0xb2c20821431853L,0x34508b005145145L,0x5515108f0c508718L, 0x401430850c76cd6cL,0x871430c71451c51L,0xd3071451450000c3L,0x560c26dc1560ca00L,
0x8740500f2051454L,0xe2534d920618f090L,0x493826596592c238L,0x4423c31421c600d6L, 0xc914369b35b2851L,0x465939451a14500dL,0x945075030cb2c939L,0x9b70558328034c3L,
0x72c2a042cb2d1545L,0x422c3983a091c574L,0xb2c514508b2c52L,0xf0c508718034b08bL, 0x72caaecae41c5583L,0xc71472871c308510L,0x1470030c50871c32L,0xc1560ca00d307147L,
0xa810b2cb45515108L,0x2260e824715d1cb0L,0xe6592c538e2d74ceL,0x420c308138938238L, 0xabb2b9071560c26dL,0x38a1c70c21441cb2L,0x314b1c938e657394L,0x4308308139438738L,
0x850842022020cb1L,0x70c25c20ce0820ceL,0x4208208208208b0dL,0xc20420850821421L, 0x820c51031c22432L,0x50c35c33830d7082L,0xc31c30c30c30c338L,0xc20451450c30c30L,
0x21080880832c5083L,0xa50838820838c214L,0xaaaaaaaaa9c39430L,0x1aaa7eaa9fa9faaaL, 0x31440c70890c90c2L,0xea0df0c3a8208208L,0xa28a28a28a231430L,0x1861868a28a28a1eL,
0x824820d01420c308L,0x7184d37184d94d64L,0x34c24d071b7136d3L,0x990936421534d34dL, 0xc368248348308308L,0x4d96584514516453L,0x36590d94d4659619L,0x546590d90d969964L,
0x834050830c20530L,0x34dc613653592092L,0xa479c6dc4db4dc61L,0x920a9f924924924aL, 0x920d20c20c20541L,0x961145145914f0daL,0xe89d351965865365L,0x9e89e89e99e7a279L,
0x72c220308192a82aL,0x724b24a30930920L,0xd72025c920e2422L,0x92d70975c9082247L, 0xb203243081821827L,0x422492c718b28920L,0x3872c35cb28b0d72L,0x32cb2972c30d72cbL,
0x880c204924e0892cL,0x2c928c24c2481cb0L,0x80a5248889088749L,0x6a861b2aaac74394L, 0xc90c204e1c75cb0cL,0x24b1c62ca2482c80L,0xb0ea2e42c3a89089L,0xa4966a289669a31cL,
0x81b2ca6ab27b278L,0xa3093092072c2203L,0xd76985d36915ce5cL,0x5d74c25c771b6936L, 0x8175e7a59a8a269L,0x718b28920b203243L,0x175976584114105cL,0x5c36572d74ce5d96L,
0x724e0973892d74d7L,0x4c2481cb0880c205L,0x6174da45739728c2L,0x4aa175c6da4db5daL, 0xe1ce5d70d92d7297L,0xca2482c80c90c204L,0x5d96104504171c62L,0x79669533976585d6L,
0x6a869b2786486186L,0xcb14510308186caL,0x220e2c5144205102L,0xcb0d70c538210e2cL, 0x659689e6964965a2L,0x24510510308175e7L,0xe2420841451031c2L,0x453851c35c338714L,
0x1451420851421452L,0x51440c204b085085L,0xcb1451081440832cL,0x94316208488b0888L, 0xc30c31c51c51451L,0x41440c20451450c7L,0x821051440c708914L,0x1470ea0df1c58c90L,
0xfaaa7dfa9f7e79c3L,0x30819ea7ea7df7dL,0x6564855820d01451L,0x9613598393698399L, 0x8a1e85e861861863L,0x30818618687a8a2L,0x5053c36824853c51L,0x96194ce51341144fL,
0xd965364539071b71L,0x4e0990996451534L,0x21560834051440c2L,0xd660e4da60e65959L, 0x943855141544d439L,0x5415464e0d90d96L,0xf0da09214f1440c2L,0x533944d04513d414L,
0x9207e979c6dc6584L,0xa82a8207df924820L,0x892072c5103081a6L,0x2c22072cb2ca30b2L, 0x86082181350e6586L,0x18277689e89e981dL,0x8920718510308182L,0x14e2422441c718b2L,
0x52c70d720538910eL,0x8b2cb2d708e38914L,0x1cb1440c204b24e0L,0x874b2cb28c2ca248L, 0xe1c73871c35cb287L,0xc70c32cb28e1c514L,0x1c61440c204e1c75L,0x90891071c62ca248L,
0x4394816224488b08L,0x9e786aa69b1f7e77L,0x51030819eca6a9e7L,0x8e38a30b2892072cL, 0xa31c70ea2e41c58cL,0xa269a475e86175e7L,0x510308175e7a57a8L,0xf38718b28920718L,
0x6996175983936913L,0x74ce39764538771bL,0xc204e24e08e38b2dL,0x28c2ca2481cb1440L, 0x39961758e5134114L,0x728e38550e1ce4ceL,0xc204e1ce5ce0d92dL,0x1c62ca2481c61440L,
0x85d660e4da44e38eL,0x698607e975c6da65L,0xa6ca6aa699e7864aL 0x85d63944d04503ceL,0x5d86075e75338e65L,0x75e7657689e69647L
}; };
private final static long[] offsetIncrs5 = new long[] /*3 bits per value */ { private final static long[] offsetIncrs5 = new long[] /*3 bits per value */ {
0x3002000000080000L,0x20c060L,0x100000004000000L,0xdb6db6db50603018L, 0xc0000010000000L,0x40000060061L,0x6000000800000000L,0xdb6ab6db6b003080L,
0xa480000200002db6L,0x1249208841241240L,0x4000010000104120L,0x2492c42092092052L, 0x80040000002db6L,0x1148241249245240L,0x4002000000104904L,0xa4b2592492292000L,
0xc30d800096592d9L,0xb01b0c06c36036d8L,0x186c00036c36db0dL,0xad860361b01b6c06L, 0xd80c00009649658L,0x80db6d86db0c001bL,0xc06000036db01b6dL,0x6db6c36d86000d86L,
0x360001b75b6dd6ddL,0xc412311c0db6030cL,0xdb0db6e36e06L,0x9188e06db01861bL, 0x300001b6ddadb6edL,0xe37236e40006c360L,0xdb6c46db6236L,0xb91b72000361b018L,
0x6dd6db71b72b62L,0x4024024900800920L,0x20824900904920c2L,0x1201248040049000L, 0x6db7636dbb1b71L,0x6124800120100820L,0x2482000492492490L,0x9240009008041000L,
0x5524ad4aa4906120L,0x4092402002480015L,0x9252251248409409L,0x4920100124000820L, 0x555b6a4924924830L,0x2000480402080012L,0x8411249249252449L,0x24020104000928L,
0x29128924204a04a0L,0x900830d200055549L,0x934930c24c24034L,0x418690002682493L, 0x5892492492922490L,0x120d808200049456L,0x6924924906da4800L,0x6c041000249a01bL,
0x9a49861261201a48L,0xc348001355249d4L,0x24c40930940d2402L,0x1a40009a0924e24dL, 0x924924836d240009L,0x6020800124d5adb4L,0x2492523692000483L,0x104000926846da49L,
0x6204984a06920106L,0x92494d5492712692L,0x4924924924924924L,0x2492492492492492L, 0x49291b49000241b0L,0x92494935636d2492L,0x4924924924924924L,0x2492492492492492L,
0x9249249249249249L,0x4924924924924924L,0x2492492492492492L,0x9249249249249249L, 0x9249249249249249L,0x4924924924924924L,0x2492492492492492L,0x9249249249249249L,
0x4924924924924924L,0x2492492492492492L,0x9249249249249249L,0x4924924924924924L, 0x4924924924924924L,0x2492492492492492L,0x9249249249249249L,0x4924924924924924L,
0x2492492492492492L,0x9249249249249249L,0x4924924924924924L,0x2492492492492492L, 0x2492492492492492L,0x9249249249249249L,0x4924924924924924L,0x2492492492492492L,
@ -211,53 +211,53 @@ class Lev2TParametricDescription extends ParametricDescription {
// state map // state map
// 0 -> [(0, 0)] // 0 -> [(0, 0)]
// 1 -> [(0, 2)] // 1 -> [(0, 1)]
// 2 -> [(0, 1)] // 2 -> [(0, 2)]
// 3 -> [(0, 1), (1, 1)] // 3 -> [(0, 1), (1, 1)]
// 4 -> [(0, 2), (1, 2)] // 4 -> [(0, 2), (1, 2)]
// 5 -> [t(0, 2), (0, 2), (1, 2), (2, 2)] // 5 -> [(0, 1), (1, 1), (2, 1)]
// 6 -> [(0, 2), (2, 1)] // 6 -> [(0, 2), (1, 2), (2, 2)]
// 7 -> [(0, 1), (2, 2)] // 7 -> [(0, 1), (2, 1)]
// 8 -> [(0, 2), (2, 2)] // 8 -> [(0, 1), (2, 2)]
// 9 -> [(0, 1), (1, 1), (2, 1)] // 9 -> [(0, 2), (2, 1)]
// 10 -> [(0, 2), (1, 2), (2, 2)] // 10 -> [(0, 2), (2, 2)]
// 11 -> [(0, 1), (2, 1)] // 11 -> [t(0, 1), (0, 1), (1, 1), (2, 1)]
// 12 -> [t(0, 1), (0, 1), (1, 1), (2, 1)] // 12 -> [t(0, 2), (0, 2), (1, 2), (2, 2)]
// 13 -> [(0, 2), (1, 2), (2, 2), (3, 2)] // 13 -> [(0, 2), (1, 2), (2, 2), (3, 2)]
// 14 -> [t(0, 2), (0, 2), (1, 2), (2, 2), (3, 2)] // 14 -> [(0, 1), (1, 1), (3, 2)]
// 15 -> [(0, 2), t(1, 2), (1, 2), (2, 2), (3, 2)] // 15 -> [(0, 1), (2, 2), (3, 2)]
// 16 -> [(0, 2), (2, 1), (3, 1)] // 16 -> [(0, 1), (3, 2)]
// 17 -> [(0, 1), t(1, 2), (2, 2), (3, 2)] // 17 -> [(0, 1), t(1, 2), (2, 2), (3, 2)]
// 18 -> [(0, 2), (3, 2)] // 18 -> [(0, 2), (1, 2), (3, 1)]
// 19 -> [(0, 2), (1, 2), t(1, 2), (2, 2), (3, 2)] // 19 -> [(0, 2), (1, 2), (3, 2)]
// 20 -> [t(0, 2), (0, 2), (1, 2), (3, 1)] // 20 -> [(0, 2), (1, 2), t(1, 2), (2, 2), (3, 2)]
// 21 -> [(0, 1), (1, 1), (3, 2)] // 21 -> [(0, 2), (2, 1), (3, 1)]
// 22 -> [(0, 2), (2, 2), (3, 2)] // 22 -> [(0, 2), (2, 2), (3, 2)]
// 23 -> [(0, 2), (1, 2), (3, 1)] // 23 -> [(0, 2), (3, 1)]
// 24 -> [(0, 2), (1, 2), (3, 2)] // 24 -> [(0, 2), (3, 2)]
// 25 -> [(0, 1), (2, 2), (3, 2)] // 25 -> [(0, 2), t(1, 2), (1, 2), (2, 2), (3, 2)]
// 26 -> [(0, 2), (3, 1)] // 26 -> [t(0, 2), (0, 2), (1, 2), (2, 2), (3, 2)]
// 27 -> [(0, 1), (3, 2)] // 27 -> [t(0, 2), (0, 2), (1, 2), (3, 1)]
// 28 -> [(0, 2), (2, 1), (4, 2)] // 28 -> [(0, 2), (1, 2), (2, 2), (3, 2), (4, 2)]
// 29 -> [(0, 2), t(1, 2), (1, 2), (2, 2), (3, 2), (4, 2)] // 29 -> [(0, 2), (1, 2), (2, 2), (4, 2)]
// 30 -> [(0, 2), (1, 2), (4, 2)] // 30 -> [(0, 2), (1, 2), (2, 2), t(2, 2), (3, 2), (4, 2)]
// 31 -> [(0, 2), (1, 2), (3, 2), (4, 2)] // 31 -> [(0, 2), (1, 2), (3, 2), (4, 2)]
// 32 -> [(0, 2), (2, 2), (3, 2), (4, 2)] // 32 -> [(0, 2), (1, 2), (4, 2)]
// 33 -> [(0, 2), (1, 2), t(2, 2), (2, 2), (3, 2), (4, 2)] // 33 -> [(0, 2), (1, 2), t(1, 2), (2, 2), (3, 2), (4, 2)]
// 34 -> [(0, 2), (1, 2), (2, 2), t(2, 2), (3, 2), (4, 2)] // 34 -> [(0, 2), (1, 2), t(2, 2), (2, 2), (3, 2), (4, 2)]
// 35 -> [(0, 2), (3, 2), (4, 2)] // 35 -> [(0, 2), (2, 1), (4, 2)]
// 36 -> [(0, 2), t(2, 2), (2, 2), (3, 2), (4, 2)] // 36 -> [(0, 2), (2, 2), (3, 2), (4, 2)]
// 37 -> [t(0, 2), (0, 2), (1, 2), (2, 2), (4, 2)] // 37 -> [(0, 2), (2, 2), (4, 2)]
// 38 -> [(0, 2), (1, 2), (2, 2), (4, 2)] // 38 -> [(0, 2), (3, 2), (4, 2)]
// 39 -> [t(0, 2), (0, 2), (1, 2), (2, 2), (3, 2), (4, 2)] // 39 -> [(0, 2), (4, 2)]
// 40 -> [(0, 2), (1, 2), (2, 2), (3, 2), (4, 2)] // 40 -> [(0, 2), t(1, 2), (1, 2), (2, 2), (3, 2), (4, 2)]
// 41 -> [(0, 2), (4, 2)] // 41 -> [(0, 2), t(2, 2), (2, 2), (3, 2), (4, 2)]
// 42 -> [t(0, 2), (0, 2), (1, 2), (2, 2), t(2, 2), (3, 2), (4, 2)] // 42 -> [t(0, 2), (0, 2), (1, 2), (2, 2), (3, 2), (4, 2)]
// 43 -> [(0, 2), (2, 2), (4, 2)] // 43 -> [t(0, 2), (0, 2), (1, 2), (2, 2), (4, 2)]
// 44 -> [(0, 2), (1, 2), t(1, 2), (2, 2), (3, 2), (4, 2)] // 44 -> [t(0, 2), (0, 2), (1, 2), (2, 2), t(2, 2), (3, 2), (4, 2)]
public Lev2TParametricDescription(int w) { public Lev2TParametricDescription(int w) {
super(w, 2, new int[] {0,2,1,0,1,0,-1,0,0,-1,0,-1,-1,-1,-1,-1,-2,-1,-1,-1,-2,-1,-1,-2,-1,-1,-2,-1,-2,-2,-2,-2,-2,-2,-2,-2,-2,-2,-2,-2,-2,-2,-2,-2,-2}); super(w, 2, new int[] {0,1,2,0,1,-1,0,-1,0,-1,0,-1,0,-1,-1,-1,-1,-1,-2,-1,-1,-2,-1,-2,-1,-1,-1,-2,-2,-2,-2,-2,-2,-2,-2,-2,-2,-2,-2,-2,-2,-2,-2,-2,-2});
} }
} }

41
lucene/core/src/java/org/apache/lucene/util/automaton/createLevAutomata.py
View File

@ -42,7 +42,7 @@ class LineOutput:
self._indent = self._indent[:-2] self._indent = self._indent[:-2]
if indent != 0: if indent != 0:
indent0 = ' ' * (len(self._indent) / 2 + indent) indent0 = ' ' * (len(self._indent) // 2 + indent)
else: else:
indent0 = self._indent indent0 = self._indent
@ -92,11 +92,11 @@ def charVarNumber(charVar):
def main(): def main():
if len(sys.argv) != 4: if len(sys.argv) != 4:
print print()
print 'Usage: python -u %s N <True/False> path_to_moman_dir' % sys.argv[0] print('Usage: python -u %s N <True/False> path_to_moman_dir' % sys.argv[0])
print print()
print 'NOTE: the resulting .java file is created in the current working dir!' print('NOTE: the resulting .java file is created in the current working dir!')
print print()
sys.exit(1) sys.exit(1)
n = int(sys.argv[1]) n = int(sys.argv[1])
@ -184,7 +184,7 @@ def main():
if i != 0 and MODE == 'switch': if i != 0 and MODE == 'switch':
w('switch(vector) {') w('switch(vector) {')
l = map.items() l = list(map.items())
l.sort() l.sort()
numCasesPerVector = None numCasesPerVector = None
@ -203,7 +203,8 @@ def main():
w('case %s: // <%s>' % (charVarNumber(charVar), ','.join([str(x) for x in charVar]))) w('case %s: // <%s>' % (charVarNumber(charVar), ','.join([str(x) for x in charVar])))
w.indent() w.indent()
l = states.items() l = list(states.items())
l.sort()
byFromState = {} byFromState = {}
@ -307,12 +308,12 @@ def main():
l, nbits = pack(toStateArray) l, nbits = pack(toStateArray)
subs.append(('NBITSSTATES%d' % i, str(nbits))) subs.append(('NBITSSTATES%d' % i, str(nbits)))
w(' private final static long[] toStates%d = new long[] /*%d bits per value */ %s;' % \ w(' private final static long[] toStates%d = new long[] /*%d bits per value */ %s;' % \
(i, nbits, renderList([hex(long(x)) for x in l]))) (i, nbits, renderList([(hex(int(x)) + "L") for x in l])))
l, nbits = pack(toOffsetIncrsArray) l, nbits = pack(toOffsetIncrsArray)
subs.append(('NBITSOFFSET%d' % i, str(nbits))) subs.append(('NBITSOFFSET%d' % i, str(nbits)))
w(' private final static long[] offsetIncrs%d = new long[] /*%d bits per value */ %s;' % \ w(' private final static long[] offsetIncrs%d = new long[] /*%d bits per value */ %s;' % \
(i, nbits, renderList([hex(long(x)) for x in l]))) (i, nbits, renderList([(hex(int(x)) + "L") for x in l])))
else: else:
w(' private final static int[] toStates%d = new int[] %s;' % \ w(' private final static int[] toStates%d = new int[] %s;' % \
(i, renderList([str(x) for x in toStateArray]))) (i, renderList([str(x) for x in toStateArray])))
@ -325,7 +326,7 @@ def main():
w('// state map') w('// state map')
sum = 0 sum = 0
minErrors = [] minErrors = []
for i in xrange(len(stateMap2) - 1): for i in range(len(stateMap2) - 1):
w('// %s -> %s' % (i, stateMap2[i])) w('// %s -> %s' % (i, stateMap2[i]))
# we replace t-notation as it's not relevant here # we replace t-notation as it's not relevant here
st = stateMap2[i].replace('t', '') st = stateMap2[i].replace('t', '')
@ -416,14 +417,14 @@ def main():
for sub, repl in subs: for sub, repl in subs:
s = s.replace(sub, repl) s = s.replace(sub, repl)
open(fileOut, 'wb').write(s) open(fileOut, 'w').write(s)
print 'Wrote %s [%d lines; %.1f KB]' % \ print('Wrote %s [%d lines; %.1f KB]' % \
(fileOut, len(w.l), os.path.getsize(fileOut) / 1024.) (fileOut, len(w.l), os.path.getsize(fileOut) / 1024.))
def renderList(l): def renderList(l):
lx = [' '] lx = [' ']
for i in xrange(len(l)): for i in range(len(l)):
if i > 0: if i > 0:
lx.append(',') lx.append(',')
if i % 4 == 0: if i % 4 == 0:
@ -433,7 +434,7 @@ def renderList(l):
MASKS = [] MASKS = []
v = 2 v = 2
for i in xrange(63): for i in range(63):
MASKS.append(v - 1) MASKS.append(v - 1)
v *= 2 v *= 2
@ -446,7 +447,7 @@ def pack(l):
pendingValue = 0 pendingValue = 0
packed = [] packed = []
for i in xrange(len(l)): for i in range(len(l)):
v = l[i] v = l[i]
if pendingValue > 0: if pendingValue > 0:
bitsUsed = math.ceil(math.log(pendingValue) / math.log(2.0)) bitsUsed = math.ceil(math.log(pendingValue) / math.log(2.0))
@ -495,8 +496,8 @@ def unpack(data, index, bitsPerValue):
if __name__ == '__main__': if __name__ == '__main__':
if not __debug__: if not __debug__:
print print()
print 'ERROR: please run without -O' print('ERROR: please run without -O')
print print()
sys.exit(1) sys.exit(1)
main() main()

28
lucene/core/src/java/org/apache/lucene/util/packed/gen_BulkOperation.py
View File

@ -122,10 +122,10 @@ def get_type(bits):
def block_value_count(bpv, bits=64): def block_value_count(bpv, bits=64):
blocks = bpv blocks = bpv
values = blocks * bits / bpv values = blocks * bits // bpv
while blocks % 2 == 0 and values % 2 == 0: while blocks % 2 == 0 and values % 2 == 0:
blocks /= 2 blocks //= 2
values /= 2 values //= 2
assert values * bpv == bits * blocks, "%d values, %d blocks, %d bits per value" % (values, blocks, bpv) assert values * bpv == bits * blocks, "%d values, %d blocks, %d bits per value" % (values, blocks, bpv)
return (blocks, values) return (blocks, values)
@ -181,8 +181,8 @@ def p64_decode(bpv, f, bits):
f.write(" values[valuesOffset++] = %s(block >>> shift) & %d%s;\n" % (cast_start, mask, cast_end)) f.write(" values[valuesOffset++] = %s(block >>> shift) & %d%s;\n" % (cast_start, mask, cast_end))
f.write(" }\n") f.write(" }\n")
else: else:
for i in xrange(0, values): for i in range(0, values):
block_offset = i * bpv / 64 block_offset = i * bpv // 64
bit_offset = (i * bpv) % 64 bit_offset = (i * bpv) % 64
if bit_offset == 0: if bit_offset == 0:
# start of block # start of block
@ -214,7 +214,7 @@ def p64_decode(bpv, f, bits):
if is_power_of_two(bpv) and bpv < 8: if is_power_of_two(bpv) and bpv < 8:
f.write(" for (int j = 0; j < iterations; ++j) {\n") f.write(" for (int j = 0; j < iterations; ++j) {\n")
f.write(" final byte block = blocks[blocksOffset++];\n") f.write(" final byte block = blocks[blocksOffset++];\n")
for shift in xrange(8 - bpv, 0, -bpv): for shift in range(8 - bpv, 0, -bpv):
f.write(" values[valuesOffset++] = (block >>> %d) & %d;\n" % (shift, mask)) f.write(" values[valuesOffset++] = (block >>> %d) & %d;\n" % (shift, mask))
f.write(" values[valuesOffset++] = block & %d;\n" % mask) f.write(" values[valuesOffset++] = block & %d;\n" % mask)
f.write(" }\n") f.write(" }\n")
@ -226,21 +226,21 @@ def p64_decode(bpv, f, bits):
f.write(" for (int j = 0; j < iterations; ++j) {\n") f.write(" for (int j = 0; j < iterations; ++j) {\n")
m = bits <= 32 and "0xFF" or "0xFFL" m = bits <= 32 and "0xFF" or "0xFFL"
f.write(" values[valuesOffset++] =") f.write(" values[valuesOffset++] =")
for i in xrange(bpv / 8 - 1): for i in range(bpv // 8 - 1):
f.write(" ((blocks[blocksOffset++] & %s) << %d) |" % (m, bpv - 8)) f.write(" ((blocks[blocksOffset++] & %s) << %d) |" % (m, bpv - 8))
f.write(" (blocks[blocksOffset++] & %s);\n" % m) f.write(" (blocks[blocksOffset++] & %s);\n" % m)
f.write(" }\n") f.write(" }\n")
else: else:
f.write(" for (int i = 0; i < iterations; ++i) {\n") f.write(" for (int i = 0; i < iterations; ++i) {\n")
for i in xrange(0, byte_values): for i in range(0, byte_values):
byte_start = i * bpv / 8 byte_start = i * bpv // 8
bit_start = (i * bpv) % 8 bit_start = (i * bpv) % 8
byte_end = ((i + 1) * bpv - 1) / 8 byte_end = ((i + 1) * bpv - 1) // 8
bit_end = ((i + 1) * bpv - 1) % 8 bit_end = ((i + 1) * bpv - 1) % 8
shift = lambda b: 8 * (byte_end - b - 1) + 1 + bit_end shift = lambda b: 8 * (byte_end - b - 1) + 1 + bit_end
if bit_start == 0: if bit_start == 0:
f.write(" final %s byte%d = blocks[blocksOffset++] & 0xFF;\n" % (typ, byte_start)) f.write(" final %s byte%d = blocks[blocksOffset++] & 0xFF;\n" % (typ, byte_start))
for b in xrange(byte_start + 1, byte_end + 1): for b in range(byte_start + 1, byte_end + 1):
f.write(" final %s byte%d = blocks[blocksOffset++] & 0xFF;\n" % (typ, b)) f.write(" final %s byte%d = blocks[blocksOffset++] & 0xFF;\n" % (typ, b))
f.write(" values[valuesOffset++] =") f.write(" values[valuesOffset++] =")
if byte_start == byte_end: if byte_start == byte_end:
@ -259,7 +259,7 @@ def p64_decode(bpv, f, bits):
f.write(" (byte%d << %d)" % (byte_start, shift(byte_start))) f.write(" (byte%d << %d)" % (byte_start, shift(byte_start)))
else: else:
f.write(" ((byte%d & %d) << %d)" % (byte_start, 2 ** (8 - bit_start) - 1, shift(byte_start))) f.write(" ((byte%d & %d) << %d)" % (byte_start, 2 ** (8 - bit_start) - 1, shift(byte_start)))
for b in xrange(byte_start + 1, byte_end): for b in range(byte_start + 1, byte_end):
f.write(" | (byte%d << %d)" % (b, shift(b))) f.write(" | (byte%d << %d)" % (b, shift(b)))
if bit_end == 7: if bit_end == 7:
f.write(" | byte%d" % byte_end) f.write(" | byte%d" % byte_end)
@ -280,7 +280,7 @@ if __name__ == '__main__':
f.write('abstract class BulkOperation implements PackedInts.Decoder, PackedInts.Encoder {\n') f.write('abstract class BulkOperation implements PackedInts.Decoder, PackedInts.Encoder {\n')
f.write(' private static final BulkOperation[] packedBulkOps = new BulkOperation[] {\n') f.write(' private static final BulkOperation[] packedBulkOps = new BulkOperation[] {\n')
for bpv in xrange(1, 65): for bpv in range(1, 65):
if bpv > MAX_SPECIALIZED_BITS_PER_VALUE: if bpv > MAX_SPECIALIZED_BITS_PER_VALUE:
f.write(' new BulkOperationPacked(%d),\n' % bpv) f.write(' new BulkOperationPacked(%d),\n' % bpv)
continue continue
@ -304,7 +304,7 @@ if __name__ == '__main__':
f.write(' // NOTE: this is sparse (some entries are null):\n') f.write(' // NOTE: this is sparse (some entries are null):\n')
f.write(' private static final BulkOperation[] packedSingleBlockBulkOps = new BulkOperation[] {\n') f.write(' private static final BulkOperation[] packedSingleBlockBulkOps = new BulkOperation[] {\n')
for bpv in xrange(1, max(PACKED_64_SINGLE_BLOCK_BPV) + 1): for bpv in range(1, max(PACKED_64_SINGLE_BLOCK_BPV) + 1):
if bpv in PACKED_64_SINGLE_BLOCK_BPV: if bpv in PACKED_64_SINGLE_BLOCK_BPV:
f.write(' new BulkOperationPackedSingleBlock(%d),\n' % bpv) f.write(' new BulkOperationPackedSingleBlock(%d),\n' % bpv)
else: else: