SOLR-6841: Visualize lucene segment information in Admin UI

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1665105 13f79535-47bb-0310-9956-ffa450edef68

solr/CHANGES.txt

@@ -147,6 +147,9 @@ New Features
 * SOLR-7189: Allow DIH to extract content from embedded documents via Tika.
   (Tim Allison via shalin)
 
+* SOLR-6841: Visualize lucene segment information in Admin UI.
+  (Alexey Kozhemiakin, Michal Bienkowski, hossman, Shawn Heisey, Varun Thacker via shalin)
+
 Bug Fixes
 ----------------------
 

solr/core/src/java/org/apache/solr/core/ImplicitPlugins.java

@@ -34,6 +34,7 @@ import org.apache.solr.handler.admin.LoggingHandler;
 import org.apache.solr.handler.admin.LukeRequestHandler;
 import org.apache.solr.handler.admin.PluginInfoHandler;
 import org.apache.solr.handler.admin.PropertiesRequestHandler;
+import org.apache.solr.handler.admin.SegmentsInfoRequestHandler;
 import org.apache.solr.handler.admin.ShowFileRequestHandler;
 import org.apache.solr.handler.admin.SolrInfoMBeanHandler;
 import org.apache.solr.handler.admin.SystemInfoHandler;
@@ -80,6 +81,7 @@ public class ImplicitPlugins {
     PluginInfo ping = getReqHandlerInfo("/admin/ping", PingRequestHandler.class, null);
     ping.initArgs.add(INVARIANTS, new NamedList<>(makeMap("echoParams", "all", "q", "solrpingquery")));
     implicits.add(ping);
+    implicits.add(getReqHandlerInfo("/admin/segments", SegmentsInfoRequestHandler.class, null));
     return implicits;
   }
 

solr/core/src/java/org/apache/solr/handler/admin/SegmentsInfoRequestHandler.java

@@ -0,0 +1,120 @@
+package org.apache.solr.handler.admin;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
+
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.MergePolicy;
+import org.apache.lucene.index.MergePolicy.MergeSpecification;
+import org.apache.lucene.index.MergePolicy.OneMerge;
+import org.apache.lucene.index.MergeTrigger;
+import org.apache.lucene.index.SegmentCommitInfo;
+import org.apache.lucene.index.SegmentInfos;
+import org.apache.solr.common.util.SimpleOrderedMap;
+import org.apache.solr.handler.RequestHandlerBase;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.response.SolrQueryResponse;
+import org.apache.solr.search.SolrIndexSearcher;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * This handler exposes information about last commit generation segments
+ */
+public class SegmentsInfoRequestHandler extends RequestHandlerBase {
+  private static Logger log = LoggerFactory.getLogger(SegmentsInfoRequestHandler.class);
+
+  @Override
+  public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp)
+      throws Exception {
+    log.info("Handling data for segmets info query searcher");
+    rsp.add("segments", getSegmentsInfo(req, rsp));
+    rsp.setHttpCaching(false);
+  }
+
+  private SimpleOrderedMap<Object> getSegmentsInfo(SolrQueryRequest req, SolrQueryResponse rsp)
+      throws Exception {
+    SolrIndexSearcher searcher = req.getSearcher();
+
+    SegmentInfos infos =
+        SegmentInfos.readLatestCommit(searcher.getIndexReader().directory());
+
+    List<String> mergeCandidates = getMergeCandidatesNames(req, infos);
+
+    SimpleOrderedMap<Object> segmentInfos = new SimpleOrderedMap<>();
+    SimpleOrderedMap<Object> segmentInfo = null;
+    for (SegmentCommitInfo segmentCommitInfo : infos) {
+      segmentInfo = getSegmentInfo(segmentCommitInfo);
+      if (mergeCandidates.contains(segmentCommitInfo.info.name)) {
+        segmentInfo.add("mergeCandidate", true);
+      }
+      segmentInfos.add((String) segmentInfo.get("name"), segmentInfo);
+    }
+
+    return segmentInfos;
+  }
+
+  private SimpleOrderedMap<Object> getSegmentInfo(
+      SegmentCommitInfo segmentCommitInfo) throws IOException {
+    SimpleOrderedMap<Object> segmentInfoMap = new SimpleOrderedMap<>();
+
+    segmentInfoMap.add("name", segmentCommitInfo.info.name);
+    segmentInfoMap.add("delCount", segmentCommitInfo.getDelCount());
+    segmentInfoMap.add("sizeInBytes", segmentCommitInfo.sizeInBytes());
+    segmentInfoMap.add("size", segmentCommitInfo.info.maxDoc());
+    Long timestamp = Long.parseLong(segmentCommitInfo.info.getDiagnostics()
+        .get("timestamp"));
+    segmentInfoMap.add("age", new Date(timestamp));
+    segmentInfoMap.add("source",
+        segmentCommitInfo.info.getDiagnostics().get("source"));
+
+    return segmentInfoMap;
+  }
+
+  private List<String> getMergeCandidatesNames(SolrQueryRequest req, SegmentInfos infos) throws IOException {
+    List<String> result = new ArrayList<String>();
+    IndexWriter indexWriter = getIndexWriter(req);
+    //get chosen merge policy
+    MergePolicy mp = indexWriter.getConfig().getMergePolicy();
+    //Find merges
+    MergeSpecification findMerges = mp.findMerges(MergeTrigger.EXPLICIT, infos, indexWriter);
+    if (findMerges != null && findMerges.merges != null && findMerges.merges.size() > 0) {
+      for (OneMerge merge : findMerges.merges) {
+        //TODO: add merge grouping
+        for (SegmentCommitInfo mergeSegmentInfo : merge.segments) {
+          result.add(mergeSegmentInfo.info.name);
+        }
+      }
+    }
+
+    return result;
+  }
+
+  private IndexWriter getIndexWriter(SolrQueryRequest req) throws IOException {
+    return req.getCore().getSolrCoreState().getIndexWriter(req.getCore()).get();
+  }
+
+  @Override
+  public String getDescription() {
+    return "Lucene segments info.";
+  }
+}

solr/core/src/test/org/apache/solr/handler/admin/SegmentsInfoRequestHandlerTest.java

@@ -0,0 +1,66 @@
+package org.apache.solr.handler.admin;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.solr.util.AbstractSolrTestCase;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+/**
+ * Tests for SegmentsInfoRequestHandler. Plugin entry, returning data of created segment.
+ */
+public class SegmentsInfoRequestHandlerTest extends AbstractSolrTestCase {
+  private static final int DOC_COUNT = 5;
+  
+  private static final int DEL_COUNT = 1;
+  
+  @BeforeClass
+  public static void beforeClass() throws Exception {
+    System.setProperty("enable.update.log", "false");
+    initCore("solrconfig.xml", "schema12.xml");
+  }
+
+  @Before
+  public void before() throws Exception {
+    for (int i = 0; i < DOC_COUNT; i++) {
+      assertU(adoc("id","SOLR100" + i, "name","Apache Solr:" + i));
+    }
+    for (int i = 0; i < DEL_COUNT; i++) {
+      assertU(delI("SOLR100" + i));
+    }
+    assertU(commit());
+  }
+
+  @Test
+  public void testSegmentInfos() {   
+    assertQ("No segments mentioned in result",
+        req("qt","/admin/segments"),
+          "1=count(//lst[@name='segments']/lst)");
+  }
+  
+  @Test
+  public void testSegmentInfosData() {   
+    assertQ("No segments mentioned in result",
+        req("qt","/admin/segments"),
+          //#Document
+          DOC_COUNT+"=//lst[@name='segments']/lst[1]/int[@name='size']",
+          //#Deletes
+          DEL_COUNT+"=//lst[@name='segments']/lst[1]/int[@name='delCount']");
+  }
+}

solr/webapp/web/admin.html

@@ -42,6 +42,7 @@ limitations under the License.
   <link rel="stylesheet" type="text/css" href="css/styles/replication.css?_=${version}">
   <link rel="stylesheet" type="text/css" href="css/styles/schema-browser.css?_=${version}">
   <link rel="stylesheet" type="text/css" href="css/styles/threads.css?_=${version}">
+  <link rel="stylesheet" type="text/css" href="css/styles/segments.css?_=${version}">
   <link rel="stylesheet" type="text/css" href="css/chosen.css?_=${version}">
 
   <meta http-equiv="x-ua-compatible" content="IE=9">

solr/webapp/web/css/styles/menu.css

@@ -287,6 +287,7 @@ limitations under the License.
 #core-menu .logging a { background-image: url( ../../img/ico/inbox-document-text.png ); }
 #core-menu .plugins a { background-image: url( ../../img/ico/block.png ); }
 #core-menu .dataimport a { background-image: url( ../../img/ico/document-import.png ); }
+#core-menu .segments a { background-image: url( ../../img/ico/construction.png ); }
 
 
 #content #navigation

solr/webapp/web/css/styles/segments.css

@@ -0,0 +1,145 @@
+/*
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+*/
+
+#content #segments .loader
+{
+  background-position: 0 50%;
+  padding-left: 21px;
+}
+
+#content #segments .reload
+{
+  background-image: url( ../../img/ico/arrow-circle.png );
+  background-position: 50% 50%;
+  display: block;
+  height: 30px;
+  position: absolute;
+  right: 10px;
+  top: 10px;
+  width: 30px; 
+}
+
+#content #segments .reload.loader
+{
+  padding-left: 0;
+}
+
+#content #segments .reload span
+{
+  display: none;
+}
+
+#content #segments #result
+{   
+  width: 77%;
+}
+
+#content #segments #result #response
+{
+	margin-left: 25px;
+}
+
+#content #segments .segments-holder ul {
+    margin-left: 25px;
+}
+#content #segments .segments-holder li {
+    margin-bottom: 2px;
+    position: relative;
+    width: 100%;
+}
+
+#content #segments .segments-holder li .toolitp {
+	display: none;
+    background: #C8C8C8;
+    position: absolute;
+    z-index: 1000;
+    width:220px;
+    height:120px;
+    margin-left: 100%;
+    opacity: .8;
+    padding: 5px;
+    border: 1px solid;
+    border-radius: 5px;
+}
+
+#content #segments .segments-holder li .toolitp .label {
+	float: left;
+	width: 20%;	
+	opacity: 1;
+}
+
+#content #segments .segments-holder li:hover .toolitp {
+	display:block;	
+}
+
+#content #segments .segments-holder li dl, 
+#content #segments .segments-holder li dt {
+    padding-bottom: 1px;
+    padding-top: 1px;
+}
+#content #segments .segments-holder li dl {
+    min-width: 1px;
+}
+#content #segments .segments-holder li dt {
+    color: #a0a0a0;
+    left: -45px;
+    overflow: hidden;
+    position: absolute;
+    top: 0;
+}
+#content #segments .segments-holder li dt div {
+    display: block;
+    padding-right: 4px;
+    text-align: right;
+}
+#content #segments .segments-holder li dd {
+    clear: left;
+    float: left;
+    margin-left: 2px;
+    white-space: nowrap;
+    width: 100%;
+}
+
+#content #segments .segments-holder li dd div.deleted {
+    background-color: #808080;    
+    padding-left: 5px;    
+}
+
+#content #segments .segments-holder li dd div.live {
+	background-color: #DDDDDD;
+	float: left;	
+}
+
+#content #segments .segments-holder li dd div.start {
+	float: left;
+	width: 20%;
+}
+
+#content #segments .segments-holder li dd div.end {
+	text-align: right;
+}
+
+.merge-candidate {
+	background-color: #FFC9F9 !important;
+}
+
+#content #segments .segments-holder li dd div.w5 {
+	width: 20%;
+	float: left;
+}

solr/webapp/web/js/main.js

@@ -49,7 +49,8 @@ require
     'lib/order!scripts/query',
     'lib/order!scripts/replication',
     'lib/order!scripts/schema-browser',
-    'lib/order!scripts/threads'
+    'lib/order!scripts/threads',
+    'lib/order!scripts/segments'
   ],
   function( $ )
   {

solr/webapp/web/js/scripts/app.js

@@ -392,7 +392,8 @@ var solr_admin = function( app_config )
                     '<li class="plugins"><a href="#/' + core_name + '/plugins"><span>Plugins / Stats</span></a></li>' + "\n" +
                     '<li class="query"><a href="#/' + core_name + '/query"><span>Query</span></a></li>' + "\n" +
                     '<li class="replication"><a href="#/' + core_name + '/replication"><span>Replication</span></a></li>' + "\n" +
-                    '<li class="schema-browser"><a href="#/' + core_name + '/schema-browser"><span>Schema Browser</span></a></li>'
+                    '<li class="schema-browser"><a href="#/' + core_name + '/schema-browser"><span>Schema Browser</span></a></li>' +
+                    '<li class="segments"><a href="#/' + core_name + '/segments"><span>Segments info</span></a></li>'
                   )
                   .show();
 

solr/webapp/web/js/scripts/segments.js

@@ -0,0 +1,206 @@
+/*
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements.  See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+*/
+
+
+var get_tooltip = function( segment_response ) {
+    var tooltip =
+        '<div>Segment <b>' + segment_response.name + '</b>:</div>' +
+        '<div class="label">#docs:</div><div>' + number_format(segment_response.size) +'</div>' +
+        '<div class="label">#dels:</div><div>' + number_format(segment_response.delCount) + '</div>' +
+        '<div class="label">size:</div><div>' + number_format(segment_response.sizeInBytes) + ' bytes </div>' +
+        '<div class="label">age:</div><div>' + segment_response.age + '</div>' +
+        '<div class="label">source:</div><div>' + segment_response.source + '</div>';
+    return tooltip;
+};
+
+var get_entry = function( segment_response, segment_bytes_max ) {
+    //calcualte dimensions of graph
+    var dims = calculate_dimensions(segment_response.sizeInBytes, 
+            segment_bytes_max, segment_response.size, segment_response.delCount)
+    //create entry for segment with given dimensions
+    var entry = get_entry_item(segment_response.name, dims, 
+            get_tooltip(segment_response), (segment_response.mergeCandidate)?true:false);
+
+    return entry;
+};
+
+var get_entry_item = function(name, dims, tooltip, isMergeCandidate) {
+    var entry = '<li>' + "\n" +
+    '  <dl class="clearfix" style="width: ' + dims['size'] + '%;">' + "\n" +
+    '    <dt><div>' + name + '</div></dt>' + "\n" +
+    '    <dd>';
+    entry += '<div class="live' + ((isMergeCandidate)?' merge-candidate':'') + 
+    		 '" style="width: ' + dims['alive_doc_size'] + '%;">&nbsp;</div>';
+    entry += '<div class="toolitp">' + tooltip +'</div>';
+      
+    if (dims['deleted_doc_size'] > 0.001) {
+     entry += '<div class="deleted" style="width:' + dims['deleted_doc_size']  
+     		+ '%;margin-left:' + dims['alive_doc_size'] + '%;">&nbsp;</div>';
+    }
+    entry += '</dd></dl></li>';
+    return entry;
+};
+
+var get_footer = function(deletions_count, documents_count) {
+    return '<li><dl><dt></dt><dd>Deletions: ' + 
+        (documents_count == 0 ? 0 : round_2(deletions_count/documents_count * 100)) +
+            '% </dd></dl></li>';
+};
+
+var calculate_dimensions = function(segment_size_in_bytes, segment_size_in_bytes_max, doc_count, delete_doc_count) {
+    var segment_size_in_bytes_log = Math.log(segment_size_in_bytes);
+    var segment_size_in_bytes_max_log = Math.log(segment_size_in_bytes_max);
+
+    var dims = {};
+    //Normalize to 100% size of bar
+    dims['size'] = Math.floor((segment_size_in_bytes_log / segment_size_in_bytes_max_log ) * 100);
+    //Deleted doc part size
+    dims['deleted_doc_size'] = Math.floor((delete_doc_count/(delete_doc_count + doc_count)) * dims['size']);
+    //Alive doc part size
+    dims['alive_doc_size'] = dims['size'] - dims['deleted_doc_size'];
+
+    return dims;
+};
+
+var calculate_max_size_on_disk = function(segment_entries) {
+    var max = 0;
+    $.each(segment_entries, function(idx, obj) {
+        if (obj.sizeInBytes > max) {
+            max = obj.sizeInBytes;
+        }
+    });
+    return max;
+};
+
+var round_2 = function(num) {
+    return Math.round(num*100)/100;
+};
+
+var number_format = function(x) {
+    return x.toString().replace(/\B(?=(\d{3})+(?!\d))/g, " ");
+};
+
+var prepare_x_axis = function(segment_bytes_max) {
+    var factor = 1024*1024; //for MB
+    
+    var segment_bytes_max_log = Math.log(segment_bytes_max);
+    
+    var series_entry = '<li>' + "\n" +
+    '  <dl class="clearfix" style="width:100%;">' + "\n" +
+    '    <dt><div>Size</div></dt>' + "\n" +
+    '    <dd>' + 
+    '        <div class="start">0</div>';
+    var step = 0;
+    for (var j = 0; j < 3; j+=1) {
+            step += segment_bytes_max_log/4;
+            var step_value = number_format(Math.floor((Math.pow(Math.E, step))/factor))
+            series_entry += '<div class="w5">' + ((step_value > 0.001)?step_value : '&nbsp;')  + '</div>'
+    }
+    series_entry += '<div class="end">' + number_format(Math.floor(segment_bytes_max/factor)) + ' MB </div>' +
+    '    </dd>' +
+    '  </dl>' +
+    '</li>';
+    return series_entry;
+};
+
+// #/:core/admin/segments
+sammy.get
+(
+  new RegExp( app.core_regex_base + '\\/(segments)$' ),
+  function( context )
+  {
+    var core_basepath = this.active_core.attr( 'data-basepath' );
+    var content_element = $( '#content' );
+        
+    $.get
+    (
+      'tpl/segments.html',
+      function( template )
+      {
+        content_element.html( template );
+            
+        var segments_element = $('#segments', content_element);
+        var segments_reload = $( '#segments a.reload' );
+        var url_element = $('#url', segments_element);
+        var result_element = $('#result', segments_element);
+        var response_element = $('#response', result_element);
+        var segments_holder_element = $('.segments-holder', result_element);
+
+        segments_reload
+            .die( 'click' )
+            .live
+            (
+            'click',
+            function( event )
+            {
+                $.ajax
+                (
+                  {
+                    url : core_basepath +  '/admin/segments?wt=json',
+                    dataType : 'json',
+                    context: this,
+                    beforeSend : function( arr, form, options )
+                    {
+                      loader.show( this );    
+                    },
+                    success : function( response, text_status, xhr )
+                    {
+                        var segments_response = response['segments'],
+                            segments_entries = [],
+                            segment_bytes_max = calculate_max_size_on_disk( segments_response );
+
+                        //scale
+                        segments_entries.push( prepare_x_axis( segment_bytes_max ) );
+                        
+                        var documents_count = 0, deletions_count = 0;
+                        
+                        //elements
+                        $.each( segments_response, function( key, segment_response ) {
+                            segments_entries.push( get_entry( segment_response, segment_bytes_max ) );
+                            
+                            documents_count += segment_response.size;
+                           deletions_count += segment_response.delCount;
+                        });
+     
+                        //footer
+                        segments_entries.push( get_footer( deletions_count, documents_count ) );
+                        
+                        $( 'ul', segments_holder_element ).html( segments_entries.join("\n" ) );
+                    },
+                    error : function( xhr, text_status, error_thrown )
+                    {
+                      $( this )
+                        .attr( 'title', '/admin/segments is not configured (' + xhr.status + ': ' + error_thrown + ')' );
+
+                      $( this ).parents( 'li' )
+                        .addClass( 'error' );
+                    },
+                    complete : function( xhr, text_status )
+                    {
+                      loader.hide( this );
+                    }
+                  }
+                );
+              return false;
+            }
+          );
+        //initially submit
+        segments_reload.click();
+      }
+    );
+  }
+);

solr/webapp/web/tpl/segments.html

@@ -0,0 +1,49 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+<div id="segments">	
+	<div class="clearfix">
+		
+		<div class="block fieldlist" id="statistics">
+			
+			<h2><span>Segments</span></h2>
+			<a class="reload"><span>reload</span></a>
+			
+			<div class="message-container">
+	        	<div class="message"></div>
+	      	</div>
+	
+	      	<div class="content">	      		
+				
+				  <div id="result">
+				 
+				    <div id="response">
+				
+				      <div class="segments-holder">
+			
+				         <ul></ul>
+				
+				       </div>
+				    
+				    </div>
+				    
+				</div>
+	      	</div>
+		</div>
+		
+	</div>	
+
+</div>