From b0116c225e9ceeee98f7b8aa4638f6fa71149bb8 Mon Sep 17 00:00:00 2001
From: Michael Olayemi <michaelolayemi28@gmail.com>
Date: Sat, 22 Apr 2023 06:32:53 +0100
Subject: [PATCH] BAEL-5766 Overview of NLP Libraries in Java (#13798)

* BAEL-5766 Overview of NLP Libraries in Java

* BAEL-5766 Overview of NLP Libraries in Java

* Overview of NLP Libraries in Java

* Overview of NLP Libraries in Java
---
 libraries-ai/README.md                        |  1 +
 libraries-ai/pom.xml                          | 33 +++++++++++++++
 .../nlp/CoreNLPTokenizerUnitTest.java         | 41 +++++++++++++++++++
 .../baeldung/nlp/OpenNLPLanguageDetector.java | 33 +++++++++++++++
 pom.xml                                       |  1 +
 5 files changed, 109 insertions(+)
 create mode 100644 libraries-ai/README.md
 create mode 100644 libraries-ai/pom.xml
 create mode 100644 libraries-ai/src/test/java/com/baeldung/nlp/CoreNLPTokenizerUnitTest.java
 create mode 100644 libraries-ai/src/test/java/com/baeldung/nlp/OpenNLPLanguageDetector.java

diff --git a/libraries-ai/README.md b/libraries-ai/README.md
new file mode 100644
index 0000000000..8b13789179
--- /dev/null
+++ b/libraries-ai/README.md
@@ -0,0 +1 @@
+
diff --git a/libraries-ai/pom.xml b/libraries-ai/pom.xml
new file mode 100644
index 0000000000..2c1490440f
--- /dev/null
+++ b/libraries-ai/pom.xml
@@ -0,0 +1,33 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+    xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <artifactId>libraries-ai</artifactId>
+    <name>libraries-ai</name>
+
+    <parent>
+        <groupId>com.baeldung</groupId>
+        <artifactId>parent-modules</artifactId>
+        <version>1.0.0-SNAPSHOT</version>
+    </parent>
+
+    <dependencies>
+        <dependency>
+            <groupId>edu.stanford.nlp</groupId>
+            <artifactId>stanford-corenlp</artifactId>
+            <version>${stanford-corenlp.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.opennlp</groupId>
+            <artifactId>opennlp-tools</artifactId>
+            <version>${opennlp-tools.version}</version>
+        </dependency>  
+    </dependencies>
+
+    <properties>
+        <stanford-corenlp.version>4.5.3</stanford-corenlp.version>
+        <opennlp-tools.version>2.1.1</opennlp-tools.version>
+    </properties>
+
+</project>
\ No newline at end of file
diff --git a/libraries-ai/src/test/java/com/baeldung/nlp/CoreNLPTokenizerUnitTest.java b/libraries-ai/src/test/java/com/baeldung/nlp/CoreNLPTokenizerUnitTest.java
new file mode 100644
index 0000000000..11c5cd0be8
--- /dev/null
+++ b/libraries-ai/src/test/java/com/baeldung/nlp/CoreNLPTokenizerUnitTest.java
@@ -0,0 +1,41 @@
+package com.baeldung.nlp;
+
+import edu.stanford.nlp.ling.CoreAnnotations;
+import edu.stanford.nlp.ling.CoreLabel;
+import edu.stanford.nlp.pipeline.Annotation;
+import edu.stanford.nlp.pipeline.StanfordCoreNLP;
+import edu.stanford.nlp.util.CoreMap;
+import org.junit.Test;
+
+import java.util.List;
+import java.util.Properties;
+
+import static org.junit.Assert.assertEquals;
+
+public class CoreNLPTokenizerUnitTest {
+    @Test
+    public void givenSampleText_whenTokenize_thenExpectedTokensReturned() {
+
+        Properties props = new Properties();
+        props.setProperty("annotators", "tokenize");
+
+        StanfordCoreNLP pipeline = new StanfordCoreNLP(props);
+        String text = "The german shepard display an act of kindness";
+
+        Annotation document = new Annotation(text);
+        pipeline.annotate(document);
+
+        List<CoreMap> sentences = document.get(CoreAnnotations.SentencesAnnotation.class);
+        StringBuilder tokens = new StringBuilder();
+
+        for (CoreMap sentence : sentences) {
+            for (CoreLabel token : sentence.get(CoreAnnotations.TokensAnnotation.class)) {
+                String word = token.get(CoreAnnotations.TextAnnotation.class);
+                tokens.append(word)
+                  .append(" ");
+            }
+        }
+        assertEquals("The german shepard display an act of kindness", tokens.toString()
+          .trim());
+    }
+}
diff --git a/libraries-ai/src/test/java/com/baeldung/nlp/OpenNLPLanguageDetector.java b/libraries-ai/src/test/java/com/baeldung/nlp/OpenNLPLanguageDetector.java
new file mode 100644
index 0000000000..00792b4875
--- /dev/null
+++ b/libraries-ai/src/test/java/com/baeldung/nlp/OpenNLPLanguageDetector.java
@@ -0,0 +1,33 @@
+package com.baeldung.nlp;
+
+import opennlp.tools.langdetect.Language;
+import opennlp.tools.langdetect.LanguageDetectorME;
+import opennlp.tools.langdetect.LanguageDetectorModel;
+import org.junit.jupiter.api.Test;
+
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+class OpenNLPLanguageDetector {
+
+    @Test
+    public void givenTextInEnglish_whenDetectLanguage_thenReturnsEnglishLanguageCode() {
+
+        String text = "the dream my father told me";
+        LanguageDetectorModel model;
+
+        try (InputStream modelIn = new FileInputStream("langdetect-183.bin")) {
+            model = new LanguageDetectorModel(modelIn);
+        } catch (IOException e) {
+            return;
+        }
+
+        LanguageDetectorME detector = new LanguageDetectorME(model);
+        Language language = detector.predictLanguage(text);
+
+        assertEquals("eng", language.getLang());
+    }
+}
\ No newline at end of file
diff --git a/pom.xml b/pom.xml
index 0b5b73ddfd..9cab85f40b 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1148,6 +1148,7 @@
                 <module>libraries-http</module>
                 <module>libraries-http-2</module>
                 <module>libraries-io</module>
+                <module>libraries-ai</module>
                 <module>libraries-primitive</module>
                 <module>libraries-rpc</module>
                 <module>libraries-server</module>