From 26d2ed7c4ddd2b8c714c2742dc9c42393bdb5d31 Mon Sep 17 00:00:00 2001 From: Chris Hostetter Date: Fri, 30 Jun 2017 10:39:28 -0700 Subject: [PATCH 01/10] SOLR-10989: Randomize PointFields and general cleanup in schema files where some Trie fields were unused --- solr/CHANGES.txt | 1 + .../conf/bad-schema-dup-fieldType.xml | 4 +- .../collection1/conf/schema-binaryfield.xml | 36 -------------- .../conf/schema-copyfield-test.xml | 34 ++------------ .../collection1/conf/schema-customfield.xml | 29 +----------- .../conf/schema-non-stored-docvalues.xml | 47 ++----------------- .../conf/schema-not-required-unique-key.xml | 1 - .../collection1/conf/schema-replication1.xml | 1 - .../collection1/conf/schema-replication2.xml | 1 - .../conf/schema-required-fields.xml | 34 ++------------ .../solr/collection1/conf/schema-reversed.xml | 5 +- .../conf/schema-tokenizer-test.xml | 21 +-------- .../collection1/conf/schema-version-dv.xml | 5 +- .../conf/schema-version-indexed.xml | 5 +- 14 files changed, 26 insertions(+), 198 deletions(-) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 0d2226f3b81..aadc6db6f8b 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -364,6 +364,7 @@ Other Changes - SOLR-10971: Randomize PointFields in CdcrBootstrapTest (hossman) - SOLR-10977: Randomize the usage of Points based numerics in schema15.xml and all impacted tests (hossman) - SOLR-10979: Randomize PointFields in schema-docValues*.xml and all affected tests (hossman) + - SOLR-10989: Randomize PointFields and general cleanup in schema files where some Trie fields were unused (hossman) * SOLR-6807: Changed requestDispatcher's handleSelect to default to false, thus ignoring "qt". Simplified configs to not refer to handleSelect or "qt". Switch all tests that assumed true to assume false diff --git a/solr/core/src/test-files/solr/collection1/conf/bad-schema-dup-fieldType.xml b/solr/core/src/test-files/solr/collection1/conf/bad-schema-dup-fieldType.xml index cd60df5c5e1..ac48c515ae2 100644 --- a/solr/core/src/test-files/solr/collection1/conf/bad-schema-dup-fieldType.xml +++ b/solr/core/src/test-files/solr/collection1/conf/bad-schema-dup-fieldType.xml @@ -22,8 +22,8 @@ - - + + diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-binaryfield.xml b/solr/core/src/test-files/solr/collection1/conf/schema-binaryfield.xml index 26a428d599d..f0f216b2b12 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema-binaryfield.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema-binaryfield.xml @@ -27,46 +27,10 @@ - - - - - - - - - - - - - - - - - - - diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-copyfield-test.xml b/solr/core/src/test-files/solr/collection1/conf/schema-copyfield-test.xml index 96adf2dd623..adea3c19a64 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema-copyfield-test.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema-copyfield-test.xml @@ -27,34 +27,10 @@ - - - - - - - - - - - - - + + + + @@ -94,7 +70,7 @@ - + - - - - - - - - - - - + + diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-non-stored-docvalues.xml b/solr/core/src/test-files/solr/collection1/conf/schema-non-stored-docvalues.xml index f700e60a416..4e631becdf2 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema-non-stored-docvalues.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema-non-stored-docvalues.xml @@ -17,52 +17,15 @@ --> - - - - - - - - - - - - - - - + + + + - - - - - + diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-not-required-unique-key.xml b/solr/core/src/test-files/solr/collection1/conf/schema-not-required-unique-key.xml index d38952f3183..ba8bbd5e801 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema-not-required-unique-key.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema-not-required-unique-key.xml @@ -23,7 +23,6 @@ --> - diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-replication1.xml b/solr/core/src/test-files/solr/collection1/conf/schema-replication1.xml index eeb51bdccd7..1a4e32e2645 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema-replication1.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema-replication1.xml @@ -27,7 +27,6 @@ - diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-replication2.xml b/solr/core/src/test-files/solr/collection1/conf/schema-replication2.xml index ade7bf54517..8b831272719 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema-replication2.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema-replication2.xml @@ -27,7 +27,6 @@ - diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-required-fields.xml b/solr/core/src/test-files/solr/collection1/conf/schema-required-fields.xml index 551ac1d9357..66add46c7b7 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema-required-fields.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema-required-fields.xml @@ -27,34 +27,10 @@ - - - - - - - - - - - - - + + + + @@ -77,7 +53,7 @@ - + - - - - + diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-tokenizer-test.xml b/solr/core/src/test-files/solr/collection1/conf/schema-tokenizer-test.xml index 174787ef989..6c33504427f 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema-tokenizer-test.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema-tokenizer-test.xml @@ -27,27 +27,8 @@ more concise example. - - - - - - - - - + diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-version-dv.xml b/solr/core/src/test-files/solr/collection1/conf/schema-version-dv.xml index 593b2a27881..375311a9b69 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema-version-dv.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema-version-dv.xml @@ -16,15 +16,14 @@ limitations under the License. --> - + id - - + diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-version-indexed.xml b/solr/core/src/test-files/solr/collection1/conf/schema-version-indexed.xml index 06d6b656abe..4523d507ece 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema-version-indexed.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema-version-indexed.xml @@ -16,15 +16,14 @@ limitations under the License. --> - + id - - + From f3c851a015e5d8c775f0fb28ffbd8f5725f2f11b Mon Sep 17 00:00:00 2001 From: Chris Hostetter Date: Fri, 30 Jun 2017 11:23:28 -0700 Subject: [PATCH 02/10] SOLR-10864: Fixup: Restore the DV vs trie vs points intent of TestRandomDVFaceting --- .../org/apache/solr/TestRandomDVFaceting.java | 28 ++++++++++++++++--- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/solr/core/src/test/org/apache/solr/TestRandomDVFaceting.java b/solr/core/src/test/org/apache/solr/TestRandomDVFaceting.java index 46762ea006a..aaeab54d5fc 100644 --- a/solr/core/src/test/org/apache/solr/TestRandomDVFaceting.java +++ b/solr/core/src/test/org/apache/solr/TestRandomDVFaceting.java @@ -28,6 +28,8 @@ import org.apache.lucene.util.LuceneTestCase.Slow; import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.schema.SchemaField; +import org.apache.solr.schema.TrieIntField; +import org.apache.solr.schema.IntPointField; import org.junit.BeforeClass; import org.junit.Test; import org.slf4j.Logger; @@ -39,18 +41,36 @@ import org.slf4j.LoggerFactory; * to the indexed facet results as if it were just another faceting method. */ @Slow +@SolrTestCaseJ4.SuppressPointFields(bugUrl="Test explicitly compares Trie to Points, randomization defeats the point") public class TestRandomDVFaceting extends SolrTestCaseJ4 { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); @BeforeClass public static void beforeTests() throws Exception { - // we need DVs on point fields to compute stats & facets - // but test also has hard coded assumptions about these field types *NOT* having DV when *NOT* points - // so use docvalue if and only if we are using points... - System.setProperty(NUMERIC_DOCVALUES_SYSPROP, System.getProperty(NUMERIC_POINTS_SYSPROP)); + // This tests explicitly compares Trie DV with non-DV Trie with DV Points + // so we don't want randomized DocValues on all Trie fields + System.setProperty(NUMERIC_DOCVALUES_SYSPROP, "false"); initCore("solrconfig-basic.xml","schema-docValuesFaceting.xml"); + + assertEquals("DocValues: Schema assumptions are broken", + false, h.getCore().getLatestSchema().getField("foo_i").hasDocValues()); + assertEquals("DocValues: Schema assumptions are broken", + true, h.getCore().getLatestSchema().getField("foo_i_dv").hasDocValues()); + assertEquals("DocValues: Schema assumptions are broken", + true, h.getCore().getLatestSchema().getField("foo_i_p").hasDocValues()); + + assertEquals("Type: Schema assumptions are broken", + TrieIntField.class, + h.getCore().getLatestSchema().getField("foo_i").getType().getClass()); + assertEquals("Type: Schema assumptions are broken", + TrieIntField.class, + h.getCore().getLatestSchema().getField("foo_i_dv").getType().getClass()); + assertEquals("Type: Schema assumptions are broken", + IntPointField.class, + h.getCore().getLatestSchema().getField("foo_i_p").getType().getClass()); + } int indexSize; From 2e56e7eb28a99109aa0b7c35a9698250f531b984 Mon Sep 17 00:00:00 2001 From: Noble Paul Date: Sat, 1 Jul 2017 11:44:22 +0930 Subject: [PATCH 03/10] SOLR-10954: this was supposed to be in the solrj package --- .../src/java/org/apache/solr/common/cloud/ReplicaPosition.java | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename solr/{core => solrj}/src/java/org/apache/solr/common/cloud/ReplicaPosition.java (100%) diff --git a/solr/core/src/java/org/apache/solr/common/cloud/ReplicaPosition.java b/solr/solrj/src/java/org/apache/solr/common/cloud/ReplicaPosition.java similarity index 100% rename from solr/core/src/java/org/apache/solr/common/cloud/ReplicaPosition.java rename to solr/solrj/src/java/org/apache/solr/common/cloud/ReplicaPosition.java From 5b484806a07e6bb4888c4c704de4eee891d76910 Mon Sep 17 00:00:00 2001 From: Mike Drob Date: Fri, 30 Jun 2017 16:22:37 -0500 Subject: [PATCH 04/10] LUCENE-5822 fix smoke test for renamed README --- dev-tools/scripts/.smokeTestRelease.py.swp | Bin 0 -> 16384 bytes dev-tools/scripts/smokeTestRelease.py | 12 ++++++++---- 2 files changed, 8 insertions(+), 4 deletions(-) create mode 100644 dev-tools/scripts/.smokeTestRelease.py.swp diff --git a/dev-tools/scripts/.smokeTestRelease.py.swp b/dev-tools/scripts/.smokeTestRelease.py.swp new file mode 100644 index 0000000000000000000000000000000000000000..3c5acb19f4e134d155f52a3bc8c72ea4f0ea6c8d GIT binary patch literal 16384 zcmeHOON=8&8Sb!oEF@t`HV49~)~ks<#M9nQ#A}mDcC00M-F#DPPQ;1DSw9tR?U2+{&K#DPPE6omugb;t!oHUSbK@cq@*9*>i3 z!V&0}zHwLo_4w?Z3^A!7_d(6S{F2{NF$Vbln{Gp*!eg8Gi$XBvU*WctY z^%hP&u<3@<XbtMsKsT+Ofaa%>4e#9^76Z&&Xf3^8AV7*%|piXWB2=0rGMF zVdWpQ?Y=l8|GSl+wemmC$X~JYe7|DVPAKpHWh*~t+kbOL{2R=#QFhpye<|BF`MwDQi3{EC%-*vg-rk-uQ&Lo5H&jQr15F0B0Q zb=gTiKdY{Di^to7J%Qq({UaLe4q;)2Wr6cH#^P~z**on;L1&o^8)Z&;91~jz|+9D zfUg2y06q>3fevsAxF0wUlz>CPb-*(>I?iR_Dd6kCCxNrTN#Gvf4&Y|s0PxZ~9Ont`xC}f2JPND< zw8pmsZ|z_>f*y-x!X8cokuVriuQM??eDvL1IJZkBLtE3%_ z1ga(Yap_NG-tk3``EneElKT~^TxF>WTZsyyewF3D82uMlc&G)&M0ZLgsft(_L_>`t z%g$UL#{$!;B;rz1WxX^Ex3|{jX?*CJl`uXA2!k$Wr0_us3KwM%Ri;MOX?I(N)T$>H z%SzA%eX=0J(5qXG6~iA7!^$8Tg@y++h~j}3<}6X%6J72N-57I8I^vRt=`tLqTq&QA zR_e{Q#xhGJlTe$~Vi|iAU8n&XiZ~&r8Kc240YlclN{QN~8io&4ICp_aqNA$>A3Ag< zKwj1h+oKqDBQai+o~c6CVoEH~#%|VSWZT&5i%hwFkqiX)g{qY4bD<=VQ8}BLLU#j_ zE1_con2B(dv-frt`!wcXxqEy*atw@T2HR@se20==`5MTj}? zbx4j-qV~F!Q}ud1UZum=@6PsyfuJA$?lV7QhkN!%ij4B9%t0#3LvSqba?n{zZ6hRv!+j17EPNaZK}x znXL^2@rE#BR)XIuUNUhiA376;0X|*RqJ{{GvwaF=6&|c_u@R4g9<*RPLTQ-LzV1hS zN90!^4|=!ld1ER*lEE?96CE~W=B_#I#_<1XH--%( zVpS$J5xbLA*Nz`MW-`UyCM8UyQ{iB*pUivmI&V2F7+3J8BEy?_pRK>=(NI0RcJ9~( z3f7e}c{2iz`PF3;sk0ha^TaGKrI8*;m52yQgdB`gkGI<&2hnLHPdV0G61jDeQFf#V zUMWLLd6F%RS*<*esE4FB?`mYT7{5xyA(8?@|M_yYj0%*VvkpykX63T&ADyCVg7hAV!k{LFaogNQx|EZ_pS2)2WXU>Ol7znC+fm+6Ma8%+keGzwkH zXu@rT$T?Bc#yJB8jrJR=j?m20F!2K~$$O>Lo!xS%{1R3+Kb_7AGuLeDG&z&? zDH&VNTc`yxlBFsi)`ms=>KvRs@WVPt!bhEv>vE62;>swp(^>(w- zVjEj*X=8o4*=}yEqqM^6>u1=>=K6A#33w!2432=fhcK=f=)`S^PYU@+ZYt=e zPOAr}nyvsB=173`7HhUj?16f#*{Y(C?Fh+JBaq{E8216JyUTFHi-g0wxLXMMn8xIN zm?zSLB-b6QAi>nCOv_#}=CF^W=!Xt6kQ24qcs#~UGlyTeQuS*#dFk3(bE&c3YAoD^ z@5-skl(L!IB8BZT*`=I>=;6s2rOdtUDlksTyh062_EKs79ttL=DLs|YJwieOg{%R* z(S0syP4^jLz$4^jigM_tjb7@ll{~p3%1oMJqKG0>oYu{Ba$~5nhqAoQ%v7?56zq&i zaNh?(5~h!FDcTyF0>gc-$biknRL~zJ40odFXb{A-nv{*rYJzafSw=j?2MvaD`&-{= zgUM`y(<^M}Wr;D%MNC=UTXwbBiiE9XI#n?Lc{@$qmD5s>Bo>sO|F6fhdk>)J|NQ;$ z5Aoc80k{@;3eWoA1HJ($;0VBgU!l!&z-8bEz_)?NfIdL|37|L%1_}lW1_}lW1_}lW z1_}lW1_}lW2L8_&pl4?L;^w5-?Mgf!;*l}IW=t9<*eS>EH9cvY7a{gZHm}#rt7V&_ zBRn#j_af$<4R!!K9eVH8>CnrEa;HP`PN!^Mb4^|qwYHa*8m-n5c5Y5zI3Jl;Z23Nn zJ2}DYlFB?b-zP8I@aoH9%zh)uR9FW~N_;9m$}_Y5$L2nXQ@s2{&7REer^H3@=$TqY}~i zrpN0mLff2m)z=fvNu$@yi*|R-BgJ{fGYbBv?i|dhP3ff$(5N^B-0vNy4W6@(hxh+{{|#KW|#l~ literal 0 HcmV?d00001 diff --git a/dev-tools/scripts/smokeTestRelease.py b/dev-tools/scripts/smokeTestRelease.py index 84978bbede3..1083a09d425 100644 --- a/dev-tools/scripts/smokeTestRelease.py +++ b/dev-tools/scripts/smokeTestRelease.py @@ -644,10 +644,14 @@ def verifyUnpacked(java, project, artifact, unpackPath, gitRevision, version, te textFiles.append('BUILD') for fileName in textFiles: - fileName += '.txt' - if fileName not in l: - raise RuntimeError('file "%s" is missing from artifact %s' % (fileName, artifact)) - l.remove(fileName) + fileNameTxt = fileName + '.txt' + fileNameMd = fileName + '.md' + if fileNameTxt in l: + l.remove(fileNameTxt) + elif fileNameMd in l: + l.remove(fileNameMd) + else: + raise RuntimeError('file "%s".[txt|md] is missing from artifact %s' % (fileName, artifact)) if project == 'lucene': if LUCENE_NOTICE is None: From 8000b25cabef69bc31e64dee2c3ef619b77f84f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Sun, 2 Jul 2017 01:36:09 +0200 Subject: [PATCH 05/10] SOLR-6671: Fix tests on Windows --- .../solr/core/DirectoryFactoryTest.java | 21 ++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/solr/core/src/test/org/apache/solr/core/DirectoryFactoryTest.java b/solr/core/src/test/org/apache/solr/core/DirectoryFactoryTest.java index 35198b7786f..35bb09b3766 100755 --- a/solr/core/src/test/org/apache/solr/core/DirectoryFactoryTest.java +++ b/solr/core/src/test/org/apache/solr/core/DirectoryFactoryTest.java @@ -20,16 +20,19 @@ import java.io.IOException; import java.nio.file.Paths; import java.util.Properties; +import org.apache.commons.exec.OS; import org.apache.lucene.util.LuceneTestCase; import org.apache.solr.common.util.NamedList; import org.apache.solr.handler.admin.CoreAdminHandler; import org.apache.solr.handler.component.HttpShardHandlerFactory; +import org.apache.solr.util.MockCoreContainer; import org.junit.After; import org.junit.Before; import org.junit.Test; public class DirectoryFactoryTest extends LuceneTestCase { + private static boolean IS_WINDOWS = (OS.isFamilyDOS() || OS.isFamilyWin9x() || OS.isFamilyWindows()); public void testLockTypesUnchanged() throws Exception { assertEquals("simple", DirectoryFactory.LOCK_TYPE_SIMPLE); assertEquals("native", DirectoryFactory.LOCK_TYPE_NATIVE); @@ -55,20 +58,28 @@ public class DirectoryFactoryTest extends LuceneTestCase { rdf.init(new NamedList()); // No solr.data.home property set. Absolute instanceDir - assertEquals("/tmp/inst1/data", rdf.getDataHome(new CoreDescriptor("core_name", Paths.get("/tmp/inst1"), cp, zkAware))); + assertDataHome("/tmp/inst1/data", "/tmp/inst1", rdf, cc); // Simulate solr.data.home set in solrconfig.xml tag NamedList args = new NamedList(); args.add("solr.data.home", "/solrdata/"); rdf.init(args); - assertEquals("/solrdata/inst_dir/data", rdf.getDataHome(new CoreDescriptor("core_name", Paths.get("inst_dir"), cp, zkAware))); - + assertDataHome("/solrdata/inst_dir/data", "inst_dir", rdf, cc); + // solr.data.home set with System property, and relative path System.setProperty("solr.data.home", "solrdata"); rdf.init(new NamedList()); - assertEquals("/solr/home/solrdata/inst_dir/data", rdf.getDataHome(new CoreDescriptor("core_name", Paths.get("inst_dir"), cp, zkAware))); + assertDataHome("/solr/home/solrdata/inst_dir/data", "inst_dir", rdf, cc); // Test parsing last component of instanceDir, and using custom dataDir - assertEquals("/solr/home/solrdata/myinst/mydata", rdf.getDataHome(new CoreDescriptor("core_name", Paths.get("/path/to/myinst"), cp, zkAware, "dataDir", "mydata"))); + assertDataHome("/solr/home/solrdata/myinst/mydata", "/path/to/myinst", rdf, cc, "dataDir", "mydata"); + } + + private void assertDataHome(String expected, String instanceDir, RAMDirectoryFactory rdf, MockCoreContainer cc, String... properties) throws IOException { + String dataHome = rdf.getDataHome(new CoreDescriptor("core_name", Paths.get(instanceDir), cc.containerProperties, cc.isZooKeeperAware(), properties)); + if (IS_WINDOWS) { + dataHome = dataHome.replaceFirst("^C:", "").replaceAll("\\\\","/"); + } + assertEquals(expected, dataHome); } From 7b322bd67e5a3a9c7f9ecf165d89da60c3767fbd Mon Sep 17 00:00:00 2001 From: Uwe Schindler Date: Sun, 2 Jul 2017 11:44:10 +0200 Subject: [PATCH 06/10] SOLR-6671: Fix precommit and use the Lucene-Constant to detect Windows. Also allow other local drives! --- .../test/org/apache/solr/core/DirectoryFactoryTest.java | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/solr/core/src/test/org/apache/solr/core/DirectoryFactoryTest.java b/solr/core/src/test/org/apache/solr/core/DirectoryFactoryTest.java index 35bb09b3766..2c2ddd9016d 100755 --- a/solr/core/src/test/org/apache/solr/core/DirectoryFactoryTest.java +++ b/solr/core/src/test/org/apache/solr/core/DirectoryFactoryTest.java @@ -20,19 +20,17 @@ import java.io.IOException; import java.nio.file.Paths; import java.util.Properties; -import org.apache.commons.exec.OS; +import org.apache.lucene.util.Constants; import org.apache.lucene.util.LuceneTestCase; import org.apache.solr.common.util.NamedList; import org.apache.solr.handler.admin.CoreAdminHandler; import org.apache.solr.handler.component.HttpShardHandlerFactory; -import org.apache.solr.util.MockCoreContainer; import org.junit.After; import org.junit.Before; import org.junit.Test; public class DirectoryFactoryTest extends LuceneTestCase { - private static boolean IS_WINDOWS = (OS.isFamilyDOS() || OS.isFamilyWin9x() || OS.isFamilyWindows()); public void testLockTypesUnchanged() throws Exception { assertEquals("simple", DirectoryFactory.LOCK_TYPE_SIMPLE); assertEquals("native", DirectoryFactory.LOCK_TYPE_NATIVE); @@ -76,8 +74,9 @@ public class DirectoryFactoryTest extends LuceneTestCase { private void assertDataHome(String expected, String instanceDir, RAMDirectoryFactory rdf, MockCoreContainer cc, String... properties) throws IOException { String dataHome = rdf.getDataHome(new CoreDescriptor("core_name", Paths.get(instanceDir), cc.containerProperties, cc.isZooKeeperAware(), properties)); - if (IS_WINDOWS) { - dataHome = dataHome.replaceFirst("^C:", "").replaceAll("\\\\","/"); + if (Constants.WINDOWS) { + // TODO: find a less-hacky way to assert this! + dataHome = dataHome.replaceFirst("^[A-Z]:", "").replace("\\", "/"); } assertEquals(expected, dataHome); } From 80ae5e29d57ef8b9c3cf382cf2e1676126f5b13d Mon Sep 17 00:00:00 2001 From: Uwe Schindler Date: Sun, 2 Jul 2017 12:32:09 +0200 Subject: [PATCH 07/10] LUCENE-5822: Remove python binary temp artifact --- dev-tools/scripts/.smokeTestRelease.py.swp | Bin 16384 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 dev-tools/scripts/.smokeTestRelease.py.swp diff --git a/dev-tools/scripts/.smokeTestRelease.py.swp b/dev-tools/scripts/.smokeTestRelease.py.swp deleted file mode 100644 index 3c5acb19f4e134d155f52a3bc8c72ea4f0ea6c8d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 16384 zcmeHOON=8&8Sb!oEF@t`HV49~)~ks<#M9nQ#A}mDcC00M-F#DPPQ;1DSw9tR?U2+{&K#DPPE6omugb;t!oHUSbK@cq@*9*>i3 z!V&0}zHwLo_4w?Z3^A!7_d(6S{F2{NF$Vbln{Gp*!eg8Gi$XBvU*WctY z^%hP&u<3@<XbtMsKsT+Ofaa%>4e#9^76Z&&Xf3^8AV7*%|piXWB2=0rGMF zVdWpQ?Y=l8|GSl+wemmC$X~JYe7|DVPAKpHWh*~t+kbOL{2R=#QFhpye<|BF`MwDQi3{EC%-*vg-rk-uQ&Lo5H&jQr15F0B0Q zb=gTiKdY{Di^to7J%Qq({UaLe4q;)2Wr6cH#^P~z**on;L1&o^8)Z&;91~jz|+9D zfUg2y06q>3fevsAxF0wUlz>CPb-*(>I?iR_Dd6kCCxNrTN#Gvf4&Y|s0PxZ~9Ont`xC}f2JPND< zw8pmsZ|z_>f*y-x!X8cokuVriuQM??eDvL1IJZkBLtE3%_ z1ga(Yap_NG-tk3``EneElKT~^TxF>WTZsyyewF3D82uMlc&G)&M0ZLgsft(_L_>`t z%g$UL#{$!;B;rz1WxX^Ex3|{jX?*CJl`uXA2!k$Wr0_us3KwM%Ri;MOX?I(N)T$>H z%SzA%eX=0J(5qXG6~iA7!^$8Tg@y++h~j}3<}6X%6J72N-57I8I^vRt=`tLqTq&QA zR_e{Q#xhGJlTe$~Vi|iAU8n&XiZ~&r8Kc240YlclN{QN~8io&4ICp_aqNA$>A3Ag< zKwj1h+oKqDBQai+o~c6CVoEH~#%|VSWZT&5i%hwFkqiX)g{qY4bD<=VQ8}BLLU#j_ zE1_con2B(dv-frt`!wcXxqEy*atw@T2HR@se20==`5MTj}? zbx4j-qV~F!Q}ud1UZum=@6PsyfuJA$?lV7QhkN!%ij4B9%t0#3LvSqba?n{zZ6hRv!+j17EPNaZK}x znXL^2@rE#BR)XIuUNUhiA376;0X|*RqJ{{GvwaF=6&|c_u@R4g9<*RPLTQ-LzV1hS zN90!^4|=!ld1ER*lEE?96CE~W=B_#I#_<1XH--%( zVpS$J5xbLA*Nz`MW-`UyCM8UyQ{iB*pUivmI&V2F7+3J8BEy?_pRK>=(NI0RcJ9~( z3f7e}c{2iz`PF3;sk0ha^TaGKrI8*;m52yQgdB`gkGI<&2hnLHPdV0G61jDeQFf#V zUMWLLd6F%RS*<*esE4FB?`mYT7{5xyA(8?@|M_yYj0%*VvkpykX63T&ADyCVg7hAV!k{LFaogNQx|EZ_pS2)2WXU>Ol7znC+fm+6Ma8%+keGzwkH zXu@rT$T?Bc#yJB8jrJR=j?m20F!2K~$$O>Lo!xS%{1R3+Kb_7AGuLeDG&z&? zDH&VNTc`yxlBFsi)`ms=>KvRs@WVPt!bhEv>vE62;>swp(^>(w- zVjEj*X=8o4*=}yEqqM^6>u1=>=K6A#33w!2432=fhcK=f=)`S^PYU@+ZYt=e zPOAr}nyvsB=173`7HhUj?16f#*{Y(C?Fh+JBaq{E8216JyUTFHi-g0wxLXMMn8xIN zm?zSLB-b6QAi>nCOv_#}=CF^W=!Xt6kQ24qcs#~UGlyTeQuS*#dFk3(bE&c3YAoD^ z@5-skl(L!IB8BZT*`=I>=;6s2rOdtUDlksTyh062_EKs79ttL=DLs|YJwieOg{%R* z(S0syP4^jLz$4^jigM_tjb7@ll{~p3%1oMJqKG0>oYu{Ba$~5nhqAoQ%v7?56zq&i zaNh?(5~h!FDcTyF0>gc-$biknRL~zJ40odFXb{A-nv{*rYJzafSw=j?2MvaD`&-{= zgUM`y(<^M}Wr;D%MNC=UTXwbBiiE9XI#n?Lc{@$qmD5s>Bo>sO|F6fhdk>)J|NQ;$ z5Aoc80k{@;3eWoA1HJ($;0VBgU!l!&z-8bEz_)?NfIdL|37|L%1_}lW1_}lW1_}lW z1_}lW1_}lW2L8_&pl4?L;^w5-?Mgf!;*l}IW=t9<*eS>EH9cvY7a{gZHm}#rt7V&_ zBRn#j_af$<4R!!K9eVH8>CnrEa;HP`PN!^Mb4^|qwYHa*8m-n5c5Y5zI3Jl;Z23Nn zJ2}DYlFB?b-zP8I@aoH9%zh)uR9FW~N_;9m$}_Y5$L2nXQ@s2{&7REer^H3@=$TqY}~i zrpN0mLff2m)z=fvNu$@yi*|R-BgJ{fGYbBv?i|dhP3ff$(5N^B-0vNy4W6@(hxh+{{|#KW|#l~ From ee1edd9d467bbccfe7015ec5ebee93f8b5bd4b14 Mon Sep 17 00:00:00 2001 From: Mike McCandless Date: Sun, 2 Jul 2017 15:32:44 -0400 Subject: [PATCH 08/10] LUCENE-7894: make sure IW finishes segment flush so we don't see a temporary drop in IW.maxDoc() --- .../apache/lucene/index/DocumentsWriter.java | 20 +++--- .../index/DocumentsWriterFlushQueue.java | 4 ++ .../lucene/index/TestIndexManyDocuments.java | 71 +++++++++++++++++++ 3 files changed, 86 insertions(+), 9 deletions(-) create mode 100644 lucene/core/src/test/org/apache/lucene/index/TestIndexManyDocuments.java diff --git a/lucene/core/src/java/org/apache/lucene/index/DocumentsWriter.java b/lucene/core/src/java/org/apache/lucene/index/DocumentsWriter.java index 27e28c08734..0e2a06708b4 100644 --- a/lucene/core/src/java/org/apache/lucene/index/DocumentsWriter.java +++ b/lucene/core/src/java/org/apache/lucene/index/DocumentsWriter.java @@ -544,11 +544,11 @@ final class DocumentsWriter implements Closeable, Accountable { dwptSuccess = true; } finally { subtractFlushedNumDocs(flushingDocsInRam); - if (!flushingDWPT.pendingFilesToDelete().isEmpty()) { + if (flushingDWPT.pendingFilesToDelete().isEmpty() == false) { putEvent(new DeleteNewFilesEvent(flushingDWPT.pendingFilesToDelete())); hasEvents = true; } - if (!dwptSuccess) { + if (dwptSuccess == false) { putEvent(new FlushFailedEvent(flushingDWPT.getSegmentInfo())); hasEvents = true; } @@ -582,6 +582,10 @@ final class DocumentsWriter implements Closeable, Accountable { flushingDWPT = flushControl.nextPendingFlush(); } + if (hasEvents) { + writer.doAfterSegmentFlushed(false, false); + } + // If deletes alone are consuming > 1/2 our RAM // buffer, force them all to apply now. This is to // prevent too-frequent flushing of a long tail of @@ -605,7 +609,7 @@ final class DocumentsWriter implements Closeable, Accountable { void subtractFlushedNumDocs(int numFlushed) { int oldValue = numDocsInRAM.get(); - while (!numDocsInRAM.compareAndSet(oldValue, oldValue - numFlushed)) { + while (numDocsInRAM.compareAndSet(oldValue, oldValue - numFlushed) == false) { oldValue = numDocsInRAM.get(); } assert numDocsInRAM.get() >= 0; @@ -726,10 +730,9 @@ final class DocumentsWriter implements Closeable, Accountable { static final class ApplyDeletesEvent implements Event { static final Event INSTANCE = new ApplyDeletesEvent(); - private int instCount = 0; + private ApplyDeletesEvent() { - assert instCount == 0; - instCount++; + // only one instance } @Override @@ -740,10 +743,9 @@ final class DocumentsWriter implements Closeable, Accountable { static final class ForcedPurgeEvent implements Event { static final Event INSTANCE = new ForcedPurgeEvent(); - private int instCount = 0; + private ForcedPurgeEvent() { - assert instCount == 0; - instCount++; + // only one instance } @Override diff --git a/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterFlushQueue.java b/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterFlushQueue.java index df1b38c45fc..b051545841a 100644 --- a/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterFlushQueue.java +++ b/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterFlushQueue.java @@ -119,6 +119,10 @@ class DocumentsWriterFlushQueue { synchronized (this) { // finally remove the published ticket from the queue final FlushTicket poll = queue.poll(); + + // we hold the purgeLock so no other thread should have polled: + assert poll == head; + ticketCount.decrementAndGet(); assert poll == head; } diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexManyDocuments.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexManyDocuments.java new file mode 100644 index 00000000000..eb31e738856 --- /dev/null +++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexManyDocuments.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.lucene.index; + +import java.io.IOException; +import java.util.concurrent.atomic.AtomicInteger; + +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.store.Directory; +import org.apache.lucene.util.IOUtils; +import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.util.TestUtil; + +public class TestIndexManyDocuments extends LuceneTestCase { + + public void test() throws Exception { + Directory dir = newFSDirectory(createTempDir()); + IndexWriterConfig iwc = new IndexWriterConfig(); + iwc.setMaxBufferedDocs(TestUtil.nextInt(random(), 100, 2000)); + + int numDocs = atLeast(10000); + + final IndexWriter w = new IndexWriter(dir, iwc); + final AtomicInteger count = new AtomicInteger(); + Thread[] threads = new Thread[2]; + for(int i=0;i infos) throws IOException { + void writeDocValuesUpdatesForMerge(List infos) throws IOException { boolean any = false; for (SegmentCommitInfo info : infos) { ReadersAndUpdates rld = get(info, false); if (rld != null) { any |= rld.writeFieldUpdates(directory, bufferedUpdatesStream.getCompletedDelGen(), infoStream); + rld.setIsMerging(); } } if (any) { @@ -4216,7 +4217,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit, Accountable { // Must move the pending doc values updates to disk now, else the newly merged segment will not see them: // TODO: we could fix merging to pull the merged DV iterator so we don't have to move these updates to disk first, i.e. just carry them // in memory: - readerPool.writeDocValuesUpdates(merge.segments); + readerPool.writeDocValuesUpdatesForMerge(merge.segments); // Bind a new segment name here so even with // ConcurrentMergePolicy we keep deterministic segment diff --git a/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java b/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java index 630131ef5e7..dd49fcb9437 100644 --- a/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java +++ b/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java @@ -808,14 +808,16 @@ class ReadersAndUpdates { return true; } - /** Returns a reader for merge, with the latest doc values updates and deletions. */ - synchronized SegmentReader getReaderForMerge(IOContext context) throws IOException { - + synchronized public void setIsMerging() { // This ensures any newly resolved doc value updates while we are merging are // saved for re-applying after this segment is done merging: isMerging = true; assert mergingDVUpdates.isEmpty(); + } + + /** Returns a reader for merge, with the latest doc values updates and deletions. */ + synchronized SegmentReader getReaderForMerge(IOContext context) throws IOException { // We must carry over any still-pending DV updates because they were not // successfully written, e.g. because there was a hole in the delGens, From 20dcb56da85accabd8e32b41afaca71707797ade Mon Sep 17 00:00:00 2001 From: Uwe Schindler Date: Mon, 3 Jul 2017 08:15:54 +0200 Subject: [PATCH 10/10] SOLR-6671: More generic fix to assert Solr's dataHome --- .../test/org/apache/solr/core/DirectoryFactoryTest.java | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/solr/core/src/test/org/apache/solr/core/DirectoryFactoryTest.java b/solr/core/src/test/org/apache/solr/core/DirectoryFactoryTest.java index 2c2ddd9016d..aa00342116d 100755 --- a/solr/core/src/test/org/apache/solr/core/DirectoryFactoryTest.java +++ b/solr/core/src/test/org/apache/solr/core/DirectoryFactoryTest.java @@ -20,7 +20,6 @@ import java.io.IOException; import java.nio.file.Paths; import java.util.Properties; -import org.apache.lucene.util.Constants; import org.apache.lucene.util.LuceneTestCase; import org.apache.solr.common.util.NamedList; import org.apache.solr.handler.admin.CoreAdminHandler; @@ -74,11 +73,7 @@ public class DirectoryFactoryTest extends LuceneTestCase { private void assertDataHome(String expected, String instanceDir, RAMDirectoryFactory rdf, MockCoreContainer cc, String... properties) throws IOException { String dataHome = rdf.getDataHome(new CoreDescriptor("core_name", Paths.get(instanceDir), cc.containerProperties, cc.isZooKeeperAware(), properties)); - if (Constants.WINDOWS) { - // TODO: find a less-hacky way to assert this! - dataHome = dataHome.replaceFirst("^[A-Z]:", "").replace("\\", "/"); - } - assertEquals(expected, dataHome); + assertEquals(Paths.get(expected).toAbsolutePath(), Paths.get(dataHome).toAbsolutePath()); }