diff --git a/CHANGES.txt b/CHANGES.txt index 2c2ff808bf1..c69f90014e9 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -63,6 +63,9 @@ New Features 8. SOLR-680: Add StatsComponent. This gets simple statists on matched numeric fields, including: min, max, mean, median, stddev. (koji, ryan) + 9. SOLR-561: Added Replication implemented in Java as a request handler. Supports index replication + as well as configuration replication and exposes detailed statistics and progress information + on the Admin page. Works on all platforms. (Noble Paul, yonik, Akshay Ukey, shalin) Optimizations ---------------------- diff --git a/example/solr/conf/solrconfig.xml b/example/solr/conf/solrconfig.xml index eefcc626915..59d0779a337 100755 --- a/example/solr/conf/solrconfig.xml +++ b/example/solr/conf/solrconfig.xml @@ -397,6 +397,24 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + text + id + + + + + + + + + + + + + + + + is there an echo? + + + diff --git a/src/test/test-files/solr/conf/schema-replication2.xml b/src/test/test-files/solr/conf/schema-replication2.xml new file mode 100644 index 00000000000..964866b362e --- /dev/null +++ b/src/test/test-files/solr/conf/schema-replication2.xml @@ -0,0 +1,463 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + text + id + + + + + + + + + + + + + + + + is there an echo? + + + diff --git a/src/test/test-files/solr/conf/solrconfig-master.xml b/src/test/test-files/solr/conf/solrconfig-master.xml new file mode 100644 index 00000000000..c27bf15f755 --- /dev/null +++ b/src/test/test-files/solr/conf/solrconfig-master.xml @@ -0,0 +1,425 @@ + + + + + + + + + + + + ${solr.data.dir:./solr/data} + + + + + false + 10 + + + + 32 + 2147483647 + 10000 + 1000 + 10000 + + + false + + + org.apache.lucene.index.LogByteSizeMergePolicy + + + org.apache.lucene.index.ConcurrentMergeScheduler + + 1000 + 10000 + + single + + + + + false + 10 + 32 + 2147483647 + 10000 + + true + + + + + + + + + + + + + + + + + 1024 + + + + + + + + + + + true + + + + + true + + 10 + + + + + + + + + + + + + + + + + + + + + + + + + true + + + + 0.01 + + text^0.5 features_t^1.0 subject^1.4 title_stemmed^2.0 + + + text^0.2 features_t^1.1 subject^1.4 title_stemmed^2.0 title^1.5 + + + ord(weight)^0.5 recip(rord(iind),1,1000,1000)^0.3 + + + 3<-1 5<-2 6<90% + + 100 + + + + + commit + schema.xml,stopwords.txt + + + + + + *:* + 0.01 + + text^0.5 features_t^1.0 subject^1.4 title_stemmed^2.0 + + + text^0.2 features_t^1.1 subject^1.4 title_stemmed^2.0 title^1.5 + + + ord(weight)^0.5 recip(rord(iind),1,1000,1000)^0.3 + + + 3<-1 5<-2 6<90% + + 100 + + + + 1000 + 1.4142135 + 12 + foo + + + sqrt 2 + log 10 + + + + + + + + 4 + true + text,name,subject,title,whitetok + + + + + + + 4 + true + text,name,subject,title,whitetok + + + + + + false + + + + + + + + explicit + + + + + + + + lowerfilt + + + default + lowerfilt + spellchecker1 + true + + + + jarowinkler + lowerfilt + + org.apache.lucene.search.spell.JaroWinklerDistance + spellchecker2 + + + + solr.FileBasedSpellChecker + external + spellings.txt + UTF-8 + spellchecker3 + + + + + + + + + false + + false + + 1 + + + spellcheck + + + + + + + + 100 + + + + + + 70 + + + + + + + ]]> + ]]> + + + + + + + + + + max-age=30, public + + + + + solr + solrconfig.xml scheam.xml admin-extra.html + + + + prefix-${solr.test.sys.prop2}-suffix + + + + + + diff --git a/src/test/test-files/solr/conf/solrconfig-slave.xml b/src/test/test-files/solr/conf/solrconfig-slave.xml new file mode 100644 index 00000000000..bc20deb5913 --- /dev/null +++ b/src/test/test-files/solr/conf/solrconfig-slave.xml @@ -0,0 +1,425 @@ + + + + + + + + + + + + ${solr.data.dir:./solr/data} + + + + + false + 10 + + + + 32 + 2147483647 + 10000 + 1000 + 10000 + + + false + + + org.apache.lucene.index.LogByteSizeMergePolicy + + + org.apache.lucene.index.ConcurrentMergeScheduler + + 1000 + 10000 + + single + + + + + false + 10 + 32 + 2147483647 + 10000 + + true + + + + + + + + + + + + + + + + + 1024 + + + + + + + + + + + true + + + + + true + + 10 + + + + + + + + + + + + + + + + + + + + + + + + + true + + + + 0.01 + + text^0.5 features_t^1.0 subject^1.4 title_stemmed^2.0 + + + text^0.2 features_t^1.1 subject^1.4 title_stemmed^2.0 title^1.5 + + + ord(weight)^0.5 recip(rord(iind),1,1000,1000)^0.3 + + + 3<-1 5<-2 6<90% + + 100 + + + + *:* + 0.01 + + text^0.5 features_t^1.0 subject^1.4 title_stemmed^2.0 + + + text^0.2 features_t^1.1 subject^1.4 title_stemmed^2.0 title^1.5 + + + ord(weight)^0.5 recip(rord(iind),1,1000,1000)^0.3 + + + 3<-1 5<-2 6<90% + + 100 + + + + 1000 + 1.4142135 + 12 + foo + + + sqrt 2 + log 10 + + + + + + + + 4 + true + text,name,subject,title,whitetok + + + + + + + 4 + true + text,name,subject,title,whitetok + + + + + + false + + + + + + + + explicit + + + + + + + + lowerfilt + + + default + lowerfilt + spellchecker1 + true + + + + jarowinkler + lowerfilt + + org.apache.lucene.search.spell.JaroWinklerDistance + spellchecker2 + + + + solr.FileBasedSpellChecker + external + spellings.txt + UTF-8 + spellchecker3 + + + + + + + + http://localhost:9999/solr/replication + 00:00:01 + + + + + + + + false + + false + + 1 + + + spellcheck + + + + + + + + 100 + + + + + + 70 + + + + + + + ]]> + ]]> + + + + + + + + + + max-age=30, public + + + + + solr + solrconfig.xml scheam.xml admin-extra.html + + + + prefix-${solr.test.sys.prop2}-suffix + + + + + + diff --git a/src/webapp/web/admin/index.jsp b/src/webapp/web/admin/index.jsp index f7fd5ad7001..14af7941252 100644 --- a/src/webapp/web/admin/index.jsp +++ b/src/webapp/web/admin/index.jsp @@ -23,11 +23,12 @@ <%@ page import="java.util.Date" %> <%@ page import="java.util.List" %> <%@ page import="java.util.Collection" %> +<%@ page import="org.apache.solr.request.SolrRequestHandler"%> <%-- jsp:include page="header.jsp"/ --%> <%-- do a verbatim include so we can use the local vars --%> <%@include file="header.jsp" %> - +<%SolrRequestHandler replicationhandler = core.getRequestHandler("/replication");%>
@@ -43,7 +44,7 @@ [Config] <% } %> [Analysis] - [Schema Browser] + [Schema Browser] <%if(replicationhandler != null ){%>[Replication]<%}%>
[Statistics] [Info] diff --git a/src/webapp/web/admin/replication/header.jsp b/src/webapp/web/admin/replication/header.jsp new file mode 100644 index 00000000000..16751171684 --- /dev/null +++ b/src/webapp/web/admin/replication/header.jsp @@ -0,0 +1,81 @@ +<%@ page contentType="text/html; charset=utf-8" pageEncoding="UTF-8"%> +<%-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--%> + +<%@ page import="org.apache.solr.common.util.NamedList, + org.apache.solr.common.util.SimpleOrderedMap, + org.apache.solr.request.LocalSolrQueryRequest, + org.apache.solr.request.SolrQueryResponse, + org.apache.solr.request.SolrRequestHandler"%> + + + + +<% +request.setCharacterEncoding("UTF-8"); +%> + +<%@include file="../_info.jsp" %> + + + + + + + +Solr replication admin page + + +<%! +public NamedList executeCommand(String command, SolrCore core, SolrRequestHandler rh){ + NamedList namedlist = new SimpleOrderedMap(); + namedlist.add("command", command); + LocalSolrQueryRequest solrqreq = new LocalSolrQueryRequest(core, namedlist); + SolrQueryResponse rsp = new SolrQueryResponse(); + core.execute(rh, solrqreq, rsp); + namedlist = rsp.getValues(); + return namedlist; +} +%> + +<% + +final SolrRequestHandler rh = core.getRequestHandler("/replication"); +NamedList namedlist = executeCommand("details",core,rh); +NamedList detailsMap = (NamedList)namedlist.get("details"); + +if("false".equals((String)detailsMap.get("isMaster"))){ +%> + +<%}%> + + + + +Solr +

Solr replication (<%= collectionName %>) +<% +if("true".equals((String)detailsMap.get("isMaster"))) + out.println(" Master"); + else + out.println(" Slave"); +%>

+ +<%= hostname %>:<%= port %>
+cwd=<%= cwd %> SolrHome=<%= solrHome %> diff --git a/src/webapp/web/admin/replication/index.jsp b/src/webapp/web/admin/replication/index.jsp new file mode 100644 index 00000000000..a2e025c5a92 --- /dev/null +++ b/src/webapp/web/admin/replication/index.jsp @@ -0,0 +1,379 @@ +<%@ page contentType="text/html; charset=utf-8" pageEncoding="UTF-8" %> +<%-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--%> +<%@ page import="java.util.Collection" %> +<%@ page import="java.util.Date" %> + + +<%-- do a verbatim include so we can use the local vars --%> +<%@include file="header.jsp" + %> + +
+
+ +<% + + final SolrCore solrcore = core; + +%> +<% + if ("false".equals(detailsMap.get("isMaster"))) + if (detailsMap != null) {%> + + + + + + + <% + NamedList nl = (NamedList) detailsMap.get("masterDetails"); + if (nl != null) { + long masterVersion = (Long) nl.get("indexversion"); + long masterGeneration = (Long) nl.get("generation"); + long replicatableMasterVer = 0, replicatableMasterGen = 0; + if (nl.get("replicatableindexversion") != null) + replicatableMasterVer = (Long) nl.get("replicatableindexversion"); + if (nl.get("replicatablegeneration") != null) + replicatableMasterGen = (Long) nl.get("replicatablegeneration"); + %> + + + + + + + + +<%}%> + + + + + +<%}%> + + + + + + + + + + + + + + + + +<% + if ("true".equals(detailsMap.get("isMaster"))) + if (detailsMap != null) { +%> + + + + + + + + + + +<%}%> + +<% + if ("false".equals(detailsMap.get("isMaster"))) + if (detailsMap != null) {%> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +<% + if ("true".equals(detailsMap.get("isReplicating"))) { +%> + + + + + + + + + + + + + + + + + + + + + + + +<%}%> + + + + + + + + + + + + +<%}%> + +<%-- List the cores (that arent this one) so we can switch --%> +<% org.apache.solr.core.CoreContainer cores = (org.apache.solr.core.CoreContainer) request.getAttribute("org.apache.solr.CoreContainer"); + if (cores != null) { + Collection names = cores.getCoreNames(); + if (names.size() > 1) {%> + + + + +<% + } + }%> + + +
+ Master + + <% + out.println((String) detailsMap.get("masterUrl")); + %> +
+ Latest Index Version:<%=masterVersion%>, Generation: <%=masterGeneration%> +
Replicatable Index Version:<%=replicatableMasterVer%>, Generation: <%=replicatableMasterGen%> +
+ Poll Interval + + <% + out.println((String) detailsMap.get("pollInterval")); + %> +
+ Local Index + + <% + if (detailsMap != null) + out.println("Index Version: " + detailsMap.get("indexversion") + ", Generation: " + detailsMap.get("generation")); + %> +
+ + <% if (null != core.getIndexDir()) { + File dir = new File(core.getIndexDir()); + out.println("Location: " + dir.getCanonicalPath()); + }%> +
<% if (detailsMap != null) + out.println("Size: " + detailsMap.get("indexSize")); + %> +
+ <%out.println("Config Files To Replicate: " + detailsMap.get("confFiles"));%> +
+ <%out.println("Trigger Replication On: " + detailsMap.get("replicateAfter")); %> +
+ + <% + out.println("Times Replicated Since Startup: " + detailsMap.get("timesIndexReplicated")); + %> +
+ + <% + out.println("Previous Replication Done At: " + detailsMap.get("indexReplicatedAt")); + %> +
+ + <% + out.println("Config Files Replicated At: " + detailsMap.get("confFilesReplicatedAt")); + %> +
+ + <% + out.println("Config Files Replicated: " + detailsMap.get("confFilesReplicated")); + %> +
+ + <% + out.println("Times Config Files Replicated Since Startup: " + detailsMap.get("timesConfigReplicated")); + %> +
+ + <% + if (detailsMap.get("nextExecutionAt") != null) + if (detailsMap.get("nextExecutionAt") != "") + out.println("Next Replication Cycle At: " + detailsMap.get("nextExecutionAt")); + else if ("true".equals(detailsMap.get("isPollingDisabled"))) + out.println("Next Replication Cycle At: Polling disabled."); + else { + NamedList nl1 = (NamedList) detailsMap.get("masterDetails"); + out.println("Next Replication Cycle At: After " + nl1.get("replicateAfter") + " on master."); + } + %> +
Current Replication Status + + + <%out.println("Start Time: " + detailsMap.get("replicationStartTime"));%> +
+ <% + out.println("Files Downloaded: " + detailsMap.get("numFilesDownloaded") + " / " + detailsMap.get("numFilesToDownload"));%> +
+ <% + out.println("Downloaded: " + detailsMap.get("bytesDownloaded") + " / " + detailsMap.get("bytesToDownload") + " [" + detailsMap.get("totalPercent") + "%]");%> +
+ <% + out.println("Downloading File: " + detailsMap.get("currentFile") + ", Downloaded: " + detailsMap.get("currentFileSizeDownloaded") + " / " + detailsMap.get("currentFileSize") + " [" + detailsMap.get("currentFileSizePercent") + "%]");%> +
+ <% + out.println("Time Elapsed: " + detailsMap.get("timeElapsed") + ", Estimated Time Remaining: " + detailsMap.get("timeRemaining") + ", Speed: " + detailsMap.get("downloadSpeed") + "/s");%> +
Controls + <% + String pollVal = request.getParameter("poll"); + if (pollVal != null) + if (pollVal.equals("disable")) + executeCommand("disablepoll", core, rh); + else if (pollVal.equals("enable")) + executeCommand("enablepoll", core, rh); + if ("false".equals(detailsMap.get("isPollingDisabled"))) { + %> + +
+ + +
+ + <%}%> + <% + if ("true".equals(detailsMap.get("isPollingDisabled"))) { + %> + +
+ + +
+ <% + } + %> + +
+
+ + +
+ <% + if ("true".equals(detailsMap.get("isReplicating"))) { + %> + +
+ + +
+ + <%} else {%> + + <% + } + String replicateParam = request.getParameter("replicate"); + String abortParam = request.getParameter("abort"); + if (replicateParam != null) + if (replicateParam.equals("now")) { + new Thread() { + public void run() { + executeCommand("snappull", solrcore, rh); + } + }.start(); + } + if (abortParam != null) + if (abortParam.equals("stop")) { + new Thread() { + public void run() { + executeCommand("abortsnappull", solrcore, rh); + } + }.start(); + } + + %> +
Cores:
<% + for (String name : names) { + %>[<%=name%> + ]<% + }%>
+

+ +

+ + + + + + + + + + +
+ + Current Time: <%= new Date() %> +
+ + Server Start At: <%= new Date(core.getStartTime()) %> +
+ +
+Return to Admin Page + + diff --git a/src/webapp/web/admin/solr-admin.css b/src/webapp/web/admin/solr-admin.css index 3af3db38d39..25f06ff718a 100644 --- a/src/webapp/web/admin/solr-admin.css +++ b/src/webapp/web/admin/solr-admin.css @@ -51,6 +51,15 @@ input.stdbutton:hover { border: groove #0000ff; } +input.stdbuttondis{ + font-family: ITC Officina Sans Book, Helvetica, Arial, sans-serif; + font-style: bold; + font-size: 11; + text-transform: capitalize; + color: #8B8B83; + background-color: #dddddd; + border: groove #8B8B83; +} body { background-color: #bbbbbb; @@ -203,4 +212,4 @@ table.histogram td, table.histogram th { } div.analyzer { margin-left:20px; -} \ No newline at end of file +}