[Tessellator] Improve the checks that validate the diagonal between two polygon nodes (#12353)

lucene/CHANGES.txt

@@ -182,6 +182,9 @@ Bug Fixes
 
 * GITHUB#12291: Skip blank lines from stopwords list. (Jerry Chin)
 
+* GITHUB#12352: [Tessellator] Improve the checks that validate the diagonal between two polygon nodes so
+  the resulting polygons are valid counter clockwise polygons. (Ignacio Vera)
+
 Other
 ---------------------
 (No changes)

lucene/core/src/java/org/apache/lucene/geo/Tessellator.java

@@ -1090,17 +1090,22 @@ public final class Tessellator {
    * Determines whether a diagonal between two polygon nodes lies within a polygon interior. (This
    * determines the validity of the ray.) *
    */
-  private static final boolean isValidDiagonal(final Node a, final Node b) {
-    if (isVertexEquals(a, b)) {
-      // If points are equal then use it if they are valid polygons
-      return isCWPolygon(a, b);
+  private static boolean isValidDiagonal(final Node a, final Node b) {
+    if (a.next.idx == b.idx
+        || a.previous.idx == b.idx
+        // check next edges are locally visible
+        || isLocallyInside(a.previous, b) == false
+        || isLocallyInside(b.next, a) == false
+        // check polygons are CCW in both sides
+        || isCWPolygon(a, b) == false
+        || isCWPolygon(b, a) == false) {
+      return false;
     }
-    return a.next.idx != b.idx
-        && a.previous.idx != b.idx
-        && isLocallyInside(a, b)
+    if (isVertexEquals(a, b)) {
+      return true;
+    }
+    return isLocallyInside(a, b)
         && isLocallyInside(b, a)
-        && isLocallyInside(a.previous, b)
-        && isLocallyInside(b.next, a)
         && middleInsert(a, a.getX(), a.getY(), b.getX(), b.getY())
         // make sure we don't introduce collinear lines
         && area(a.previous.getX(), a.previous.getY(), a.getX(), a.getY(), b.getX(), b.getY()) != 0
@@ -1114,7 +1119,7 @@ public final class Tessellator {
   /** Determine whether the polygon defined between node start and node end is CW */
   private static boolean isCWPolygon(final Node start, final Node end) {
     // The polygon must be CW
-    return (signedArea(start, end) < 0) ? true : false;
+    return signedArea(start, end) < 0;
   }
 
   /** Determine the signed area between node start and node end */

lucene/core/src/test/org/apache/lucene/geo/TestTessellator.java

@@ -915,6 +915,28 @@ public class TestTessellator extends LuceneTestCase {
     }
   }
 
+  public void testComplexPolygon55() throws Exception {
+    String geoJson = GeoTestUtil.readShape("github-12352-1.geojson.gz");
+    Polygon[] polygons = Polygon.fromGeoJSON(geoJson);
+    for (Polygon polygon : polygons) {
+      List<Tessellator.Triangle> tessellation =
+          Tessellator.tessellate(polygon, random().nextBoolean());
+      assertEquals(area(polygon), area(tessellation), 0.0);
+      // don't check edges as it takes several minutes
+    }
+  }
+
+  public void testComplexPolygon56() throws Exception {
+    String geoJson = GeoTestUtil.readShape("github-12352-2.geojson.gz");
+    Polygon[] polygons = Polygon.fromGeoJSON(geoJson);
+    for (Polygon polygon : polygons) {
+      List<Tessellator.Triangle> tessellation =
+          Tessellator.tessellate(polygon, random().nextBoolean());
+      assertEquals(area(polygon), area(tessellation), 0.0);
+      // don't check edges as it takes several minutes
+    }
+  }
+
   private static class TestCountingMonitor implements Tessellator.Monitor {
     private int count = 0;
     private int splitsStarted = 0;