From aad84395c94c1cb60e888a52aa60caa2c5453156 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Tue, 24 Nov 2015 19:13:49 +0100 Subject: [PATCH] Add a test that upgrades succeed even if a mapping contains fields that come from a plugin. --- dev-tools/create_bwc_index.py | 10 ++ .../create_bwc_index_with_plugin_mappings.py | 124 ++++++++++++++++++ .../Murmur3FieldMapperUpgradeTests.java | 77 +++++++++++ .../bwc/index-mapper-murmur3-2.0.0.zip | Bin 0 -> 8226 bytes .../size/SizeFieldMapperUpgradeTests.java | 88 +++++++++++++ .../indices/bwc/index-mapper-size-2.0.0.zip | Bin 0 -> 7973 bytes 6 files changed, 299 insertions(+) create mode 100644 dev-tools/create_bwc_index_with_plugin_mappings.py create mode 100644 plugins/mapper-murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapperUpgradeTests.java create mode 100644 plugins/mapper-murmur3/src/test/resources/indices/bwc/index-mapper-murmur3-2.0.0.zip create mode 100644 plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/SizeFieldMapperUpgradeTests.java create mode 100644 plugins/mapper-size/src/test/resources/indices/bwc/index-mapper-size-2.0.0.zip diff --git a/dev-tools/create_bwc_index.py b/dev-tools/create_bwc_index.py index 5d663ca69f3..83a35941577 100644 --- a/dev-tools/create_bwc_index.py +++ b/dev-tools/create_bwc_index.py @@ -149,6 +149,16 @@ def start_node(version, release_dir, data_dir, repo_dir, tcp_port=DEFAULT_TRANSP cmd.append('-f') # version before 1.0 start in background automatically return subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) +def install_plugin(version, release_dir, plugin_name): + run_plugin(version, release_dir, 'install', [plugin_name]) + +def remove_plugin(version, release_dir, plugin_name): + run_plugin(version, release_dir, 'remove', [plugin_name]) + +def run_plugin(version, release_dir, plugin_cmd, args): + cmd = [os.path.join(release_dir, 'bin/plugin'), plugin_cmd] + args + subprocess.check_call(cmd) + def create_client(http_port=DEFAULT_HTTP_TCP_PORT, timeout=30): logging.info('Waiting for node to startup') for _ in range(0, timeout): diff --git a/dev-tools/create_bwc_index_with_plugin_mappings.py b/dev-tools/create_bwc_index_with_plugin_mappings.py new file mode 100644 index 00000000000..c30de412d1d --- /dev/null +++ b/dev-tools/create_bwc_index_with_plugin_mappings.py @@ -0,0 +1,124 @@ +import create_bwc_index +import logging +import os +import random +import shutil +import subprocess +import sys +import tempfile + +def fetch_version(version): + logging.info('fetching ES version %s' % version) + if subprocess.call([sys.executable, os.path.join(os.path.split(sys.argv[0])[0], 'get-bwc-version.py'), version]) != 0: + raise RuntimeError('failed to download ES version %s' % version) + +def create_index(plugin, mapping, docs): + ''' + Creates a static back compat index (.zip) with mappings using fields defined in plugins. + ''' + + logging.basicConfig(format='[%(levelname)s] [%(asctime)s] %(message)s', level=logging.INFO, + datefmt='%Y-%m-%d %I:%M:%S %p') + logging.getLogger('elasticsearch').setLevel(logging.ERROR) + logging.getLogger('urllib3').setLevel(logging.WARN) + + tmp_dir = tempfile.mkdtemp() + plugin_installed = False + node = None + try: + data_dir = os.path.join(tmp_dir, 'data') + repo_dir = os.path.join(tmp_dir, 'repo') + logging.info('Temp data dir: %s' % data_dir) + logging.info('Temp repo dir: %s' % repo_dir) + + version = '2.0.0' + classifier = '%s-%s' %(plugin, version) + index_name = 'index-%s' % classifier + + # Download old ES releases if necessary: + release_dir = os.path.join('backwards', 'elasticsearch-%s' % version) + if not os.path.exists(release_dir): + fetch_version(version) + + create_bwc_index.install_plugin(version, release_dir, plugin) + plugin_installed = True + node = create_bwc_index.start_node(version, release_dir, data_dir, repo_dir, cluster_name=index_name) + client = create_bwc_index.create_client() + put_plugin_mappings(client, index_name, mapping, docs) + create_bwc_index.shutdown_node(node) + + print('%s server output:\n%s' % (version, node.stdout.read().decode('utf-8'))) + node = None + create_bwc_index.compress_index(classifier, tmp_dir, 'plugins/%s/src/test/resources/indices/bwc' %plugin) + finally: + if node is not None: + create_bwc_index.shutdown_node(node) + if plugin_installed: + create_bwc_index.remove_plugin(version, release_dir, plugin) + shutil.rmtree(tmp_dir) + +def put_plugin_mappings(client, index_name, mapping, docs): + client.indices.delete(index=index_name, ignore=404) + logging.info('Create single shard test index') + + client.indices.create(index=index_name, body={ + 'settings': { + 'number_of_shards': 1, + 'number_of_replicas': 0 + }, + 'mappings': { + 'type': mapping + } + }) + health = client.cluster.health(wait_for_status='green', wait_for_relocating_shards=0) + assert health['timed_out'] == False, 'cluster health timed out %s' % health + + logging.info('Indexing documents') + for i in range(len(docs)): + client.index(index=index_name, doc_type="type", id=str(i), body=docs[i]) + logging.info('Flushing index') + client.indices.flush(index=index_name) + + logging.info('Running basic checks') + count = client.count(index=index_name)['count'] + assert count == len(docs), "expected %d docs, got %d" %(len(docs), count) + +def main(): + docs = [ + { + "foo": "abc" + }, + { + "foo": "abcdef" + }, + { + "foo": "a" + } + ] + + murmur3_mapping = { + 'properties': { + 'foo': { + 'type': 'string', + 'fields': { + 'hash': { + 'type': 'murmur3' + } + } + } + } + } + + create_index("mapper-murmur3", murmur3_mapping, docs) + + size_mapping = { + '_size': { + 'enabled': True + } + } + + create_index("mapper-size", size_mapping, docs) + +if __name__ == '__main__': + main() + diff --git a/plugins/mapper-murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapperUpgradeTests.java b/plugins/mapper-murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapperUpgradeTests.java new file mode 100644 index 00000000000..8ede994e2f4 --- /dev/null +++ b/plugins/mapper-murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapperUpgradeTests.java @@ -0,0 +1,77 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.mapper.murmur3; + +import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.util.TestUtil; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.env.NodeEnvironment; +import org.elasticsearch.plugin.mapper.MapperMurmur3Plugin; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.search.aggregations.AggregationBuilders; +import org.elasticsearch.search.aggregations.metrics.cardinality.Cardinality; +import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.test.hamcrest.ElasticsearchAssertions; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Collection; +import java.util.Collections; + +@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0) +@LuceneTestCase.SuppressFileSystems("ExtrasFS") +public class Murmur3FieldMapperUpgradeTests extends ESIntegTestCase { + + @Override + protected Collection> nodePlugins() { + return Collections.singleton(MapperMurmur3Plugin.class); + } + + public void testUpgradeOldMapping() throws IOException { + final String indexName = "index-mapper-murmur3-2.0.0"; + Path unzipDir = createTempDir(); + Path unzipDataDir = unzipDir.resolve("data"); + Path backwardsIndex = getBwcIndicesPath().resolve(indexName + ".zip"); + try (InputStream stream = Files.newInputStream(backwardsIndex)) { + TestUtil.unzip(stream, unzipDir); + } + assertTrue(Files.exists(unzipDataDir)); + + final String node = internalCluster().startNode(); + Path[] nodePaths = internalCluster().getInstance(NodeEnvironment.class, node).nodeDataPaths(); + assertEquals(1, nodePaths.length); + Path dataPath = nodePaths[0].resolve(NodeEnvironment.INDICES_FOLDER); + assertFalse(Files.exists(dataPath)); + Path src = unzipDataDir.resolve(indexName + "/nodes/0/indices"); + Files.move(src, dataPath); + + ensureYellow(); + final SearchResponse countResponse = client().prepareSearch(indexName).setSize(0).get(); + ElasticsearchAssertions.assertHitCount(countResponse, 3L); + + final SearchResponse cardinalityResponse = client().prepareSearch(indexName).addAggregation( + AggregationBuilders.cardinality("card").field("foo.hash")).get(); + Cardinality cardinality = cardinalityResponse.getAggregations().get("card"); + assertEquals(3L, cardinality.getValue()); + } + +} diff --git a/plugins/mapper-murmur3/src/test/resources/indices/bwc/index-mapper-murmur3-2.0.0.zip b/plugins/mapper-murmur3/src/test/resources/indices/bwc/index-mapper-murmur3-2.0.0.zip new file mode 100644 index 0000000000000000000000000000000000000000..0b69aac180bc6a62f65f735812ba05ed5b59cf01 GIT binary patch literal 8226 zcmcIp2|UzkAODXrZW80_m5|z@9W!H^n8;P$TvK7(SB`O|QI3%9phK=+$jqepu!RW`-aMj z**f`R>H!9*0{%XcJ}5_`H~_UePjuQ2uBaGE03zHE1%Szi9rJ!&efsOcWPef&a<|W( zJ*05t?x=9^jz#L}qIJ=novr!ZdF>*0CWrZxA{MznX3oJ8i{5|g1ltxm!CVVyJ}iLr z1^Z2(g>8ww=Jmo{OEzC&X(5O4eO&vc1>qeL=@UufIF|<&I2gxb0Gp6rL@3!D0ssfV z(eR2?JQPJW~)SZS3^QVt7dms zi>A8X6I=`SA$GLQDcg;hPbrMP04x6mU#pJ${+0owSPP4~HQKIqW!tS6)Yb;MbJS7z zIk{Dv<>e*)B(ah<@>t2!1F{OSv}!B5#hbC0%XAYtj!h%RqhM|;|;r`6DD^v#9VheloM`gqTzvXR4gUnrrUU+TGp5}FkcdUJ}n z(^4NjGTet85gHcAsXOde$g>@?<1^*$`SEq28`z5^Fg(-1z@8cj;^S3-Oxi=0#HK$F zf)WCk;?@9Xf`6=y8PS`x)iGc@(jaV`&2Bq(rp;8)cQT<2_EiG=KfVG#iLT$SJ-m+L zw-bQw77^ab%l{5bjdHRpF6ev?W$a)q|5NYpM3Amq;PMmgs-f6X)2H`B1v$c5dLaOHxLXtj@idqH0WQ zq{q@rL?+Gy#m`Mv0ROq8Zyr0_@a6V6FJr#kaRBB=M z8P~NiKOAm<*w9e_w4PPp(pvX4or3H!Md6WI3YH3i<_65vw|l~JUMSH@u9He$7Q=7$ zDfO+*?(ch9S{#nA7D;Rn6}^7rhU&vJ9}^%FBWj+@#pgJuQ0RhL6&SX=KnOvB6jAx_ zq@p_8pes&$n<}6!Kb7SR&!JI1?AKB~a8UY5Ot3UY@S>4Hbhm`tDI4c>P~K$R=-GXX z;RcT{PdJtksuqZ6Sbv9?-Ec(n?;SgUyTeO5;=I$^Bx15OTO?1GahMB z;T%LG?3e}frnU9YU405!j%_$b)Xh!9(rE;szGmXxhrd1vKN)Qs+?#b3VWRlBUMW%j zsMP6(qdjHErFD+wsWz7zjaBE#x?d@b&k)sdQISp=N=mpp6&BPK*b7zLZ$&g&z!fSgiR^{vouw4CwYPnNUcC+V3L$}|;3LNaZ&6#_~0394XK9G0b7*hUksy*h@@E$Z4S zb4&(O1jCD>9e*Z- zhWaIXsAZurmio>c=~rLvZHm4*(DRpP1gk1OPe#QpVat1>TuT3eWU^GlYOO4kIK)>2 z(*3R8@s%x`Mql`xJkrs*LhHC($NRe;LXnWF5NSz<E-!^Za24~597@aekLVh z-{YQC} z&}}RAjpD_4(fFD=SR(ZIsL{h882-`0zqlVL=x8&9wOOa7Va`(G#ZCa##vebq3)@_d z2{wzpnRlydL(OB7)1Ri%$o^tlFC_7q&sEX75LtAY_7C*4d11DF#577)ULP==e(&wY z>({zuD^Qo#wX^m$o+31h^x9SK+pvGl;A>`TrAkIHDZK>qi@7u;1AR4yFk;t~{74b^ z4Q17D#U3Oby|CgMFtoqo0a=&T;E*$dfpQZOZDO@Aua#TB*IP zk2RaS8}T&e8gR0m9A7WLd8yc_ODni-oMbk3>Ns7b+UuZp{K_@-Y8c+sA?48cr|}Q$ zabbs_ik?vI0Oum)%W4;fkF&AGr5S&*^(*%JmnH)dQiFck7z|Jb>15wzfpIhT^OVpA z;|OhhY^Bb%mC5v0JT)B93ZH=2a#^7)!pgJ&x3I0blRSdU*4UzLu}5~$8asV>@Hf{Q zn#-Cyc*Qg63LgA%PKHK-ADI&7__u1qWtI6AR(SyY8XD}m09k{BA-^>60Q9-&@FI#c z8?fEOE!RzF3IJ5aKM@G7*=hzagq&QS8$bju<}M2_DRW;J(?l*vIM@P&EZK8t0HVyK#ro<~R zW-^ysCYlXZ8JvNZu%e(*(+EGm;D|=NaB}i@uDcz$lI6#x#yMFQ7x49rEW#}s%_h)B ze2EF<0Gll#f!xg-jx!Lsm7CcRgF%9q)W;l%Q##J#&~Zx_v(br22@2h$kg+&;+>*d- zcm!#|!2<;XUhux$*E~Px1-sP9Ef)y9*`p5#k^T0nL7K0)T> nodePlugins() { + return Collections.singleton(MapperSizePlugin.class); + } + + public void testUpgradeOldMapping() throws IOException { + final String indexName = "index-mapper-size-2.0.0"; + Path unzipDir = createTempDir(); + Path unzipDataDir = unzipDir.resolve("data"); + Path backwardsIndex = getBwcIndicesPath().resolve(indexName + ".zip"); + try (InputStream stream = Files.newInputStream(backwardsIndex)) { + TestUtil.unzip(stream, unzipDir); + } + assertTrue(Files.exists(unzipDataDir)); + + final String node = internalCluster().startNode(); + Path[] nodePaths = internalCluster().getInstance(NodeEnvironment.class, node).nodeDataPaths(); + assertEquals(1, nodePaths.length); + Path dataPath = nodePaths[0].resolve(NodeEnvironment.INDICES_FOLDER); + assertFalse(Files.exists(dataPath)); + Path src = unzipDataDir.resolve(indexName + "/nodes/0/indices"); + Files.move(src, dataPath); + + ensureYellow(); + final SearchResponse countResponse = client().prepareSearch(indexName).setSize(0).get(); + ElasticsearchAssertions.assertHitCount(countResponse, 3L); + + final SearchResponse sizeResponse = client().prepareSearch(indexName) + .addField("_source") + .addField("_size") + .get(); + ElasticsearchAssertions.assertHitCount(sizeResponse, 3L); + for (SearchHit hit : sizeResponse.getHits().getHits()) { + String source = hit.getSourceAsString(); + assertNotNull(source); + Map fields = hit.getFields(); + assertTrue(fields.containsKey("_size")); + Number size = fields.get("_size").getValue(); + assertNotNull(size); + assertEquals(source.length(), size.longValue()); + } + } + +} diff --git a/plugins/mapper-size/src/test/resources/indices/bwc/index-mapper-size-2.0.0.zip b/plugins/mapper-size/src/test/resources/indices/bwc/index-mapper-size-2.0.0.zip new file mode 100644 index 0000000000000000000000000000000000000000..0a74f835c3eb43ff8a9f65f724dcd917b908a853 GIT binary patch literal 7973 zcmcIo2|U#48~-t`P>qa~rDU8HW(=`KId-z{!8j7*7&8&ZG1Dg5B&~=_j=_*)YTA&^ zy;-4^YRQozrVq-MR=biDv?|r_{_j%sud7m%F48kW2uwEKT z!N$`+%zp3#(txKs#a-PFy9R(@va$B|;5rg40`Ty?gaE+w2j=tRD2~Tv<{tMZc@lzE z{oDfr2tlf3?+}8jh8jW*F|)1pYPW?iw=K!vlR#eK=2{$>6;`(SEOxu)uyby&y3*}I z%DnM&Ua-ut)}qXFnnADD%#smWwU_4HGVe;JxKju-p}7c zy$!io7q}ioFT&N2kb}OPI)2R9={D`sn262T(->PbQ;o4lF<}#Q$2(9#l}V40a4T@#(+5>nG0hushoF1b4JLS;tB?*-y= z)=sq44MRhN&LrG02`Y_~+7LJFPK@inc#*c3pUyk9D{3-Fg|uHl-_3ixLZ*Lx93*Ul zFiZ&?b9a=uT55A2PI;VLpf*YNNr{cvD`KH0ESynvFYmz)oeGMuL%Y)JERy===cM}< zsr$mV2dMsyE5}Di7M>V3MXtjrPfj&AyeU>iDxnH>Nr9Hb#7_O%QEjY z$kxH(S}xIf?@r$9mqhd>9TINl0~U`yhI>v^g4{`DU;jfh<%89^mh4`vJnW}pckTeG zXD!#SfMgC5*#9UXvgtaq2(W$azLG+7{}0091=mJ9J4f^`Ej$WqV|7%=3#H-hsO=tT zz+!X01ioMATgqzZtlv0)MIY4gJ^@QPfOi8Regb@Myh}vef_wO;JJ_Z=G(=yU%4mJ(9Z{H5Ne6!EH;v=&w+O`X487SU$ z&#M;T8P;zQsxcoGi1o@X3;K|#`11<}O={u{Z`}DQFThtcA;84i0~cs{A|!M-#@O1! zgm_}hWifTQ1>k9m=R0-NJ{Xth0z|N2GiVQQ?^m&pa=3 zn;PLFVE5bwE*oz1;tJLt@8u{_8@8pT(~=J#hR#sao0?2*2H6SbWJef=cjNa^jFX3= zP(g?R$>NWmFOuv74k=I-hLRXB82yrK#w8UL2d`|`8I+XhmsFs7z8Ddc40>NFR4F~c0Y`wH+VM1H*~i4#@BZ8;*7+_MerghGYA}lG9s1*Ln`}Au=cWc z>1%pmsQNpVF7d93v|ef|Ex~4(>YAA5NKa17jIST)my+JFRr>kRP>??UZxI3ID2MTK zNsh2|@9I2X3Z|+L7#7>F5EeXdgj46S6Xr}n6`$AX45>%jpSL-lGGL--3 zAHRzk--8)<+gPcaqDcJlT9#G}&dHP?ix9cGp;7UgS;~9Efd>zrod7W+GP1rJve}8( zy?DDJe}(=NBMT3gwGy$sxeXD*MTLekp*_=?%`Y05anX>2!7#|JEvn$qG9dJ|7$h&n zU@6wjOWa^n#{Jjw-U(%Y-q~yGvt3yz-rf``E1?=rPl{rM6uzRFQ;>D{x-3jeJw~&7 zvjw*boyEmG&w0t%(i$QjBA%PvEgUl3nAYU`!y}6{KdV&Tf0&a~Q)z!?P9e)OMtt|V znz!i6L|6lnVG zqx-2?2lP{I0q|ob=I+p2n0vKmBXs|{ zA9+zfL&al>FVnAFV_vsHzNKHJ7kM;FT)P4*Epbg{MCSZ?EWPE(sSz~MSBxgxJ~7eP~RRzL8Y!Q(aOh>wYeTKUnTH0t3bHGSQ0Wt+&@Afp$r zl^Sw^9okwZs#=v(e(ti9WGdv?hCSaX;{pi9pIRG_nmK&P@oPIbXcA^T*i*|;!L^)h z8x;s@dYT<~3+Bl*>T&&_&Gleq-aJDjt#!-o_BRni#~Nk){(RSue$5;5LHu2mPkw5TjZAGqN zcUrPpU2au@kN*wxVx5b9y6XP^M+u2^ui7mPUgmj{TxQ`O&!e6vkj$H8#>sm8x*`vW z8yI<$wcnUgaHp*8^ZPa*{BSoAMT1*kd3zk^H=Ocj)HP=tR7r9}&r)~fH67e^o z&ZaL{3zfD@pL`Kco-h5=;+cqAiOY@m$`|^i+wmx%+{0Fl5{gCUe z`e=|hg`no^kN25Ay^BnBiv!xZ(qhax3$QGGVW6My#MI0;-+6&0{|LCWS83&tufK=8 zuj(qU`rxpNc*>ep9gInbP3R<350l;-Wks#@-iC==MqPE^GVUtmphp>uRF&*syiHUT z6|-^M1fs=~&|>#CZYETnACOVVDd+!I-!>*}@}J31(}ftn`D!H&K6AyGk(f9&BDIw| z#u9br!^9GeSnK}?K9ztwB>?OyT(maGFwOGi3wbWTqpq7;Q5? zA=JUY&q`sdH;HC5S-x76i$VJ@-kY;Q(E;bVRe8S{l+R^lDRSnE3cElym&O245?;k+ zL<{AzrLfJMR?b{8W@GCJAD7trZ;5Nk2Id|2z^pQO?CQ|M;GN*-8bPy|7HUPS49dF8?xGGcUiH(Z&;zb3+4#W<`-|} U{9t1M07>xu9hiC#z?}i$UwIFvIsgCw literal 0 HcmV?d00001