-
-
-
diff --git a/solr/contrib/dataimporthandler-extras/src/test-files/dihextras/test_jpeg.jpg b/solr/contrib/dataimporthandler-extras/src/test-files/dihextras/test_jpeg.jpg
deleted file mode 100644
index 10d1ebb2d32..00000000000
Binary files a/solr/contrib/dataimporthandler-extras/src/test-files/dihextras/test_jpeg.jpg and /dev/null differ
diff --git a/solr/contrib/dataimporthandler-extras/src/test-files/dihextras/test_recursive_embedded.docx b/solr/contrib/dataimporthandler-extras/src/test-files/dihextras/test_recursive_embedded.docx
deleted file mode 100644
index cd562cbb82d..00000000000
Binary files a/solr/contrib/dataimporthandler-extras/src/test-files/dihextras/test_recursive_embedded.docx and /dev/null differ
diff --git a/solr/contrib/dataimporthandler-extras/src/test-files/dihextras/test_vsdx.vsdx b/solr/contrib/dataimporthandler-extras/src/test-files/dihextras/test_vsdx.vsdx
deleted file mode 100644
index 659ecdd5853..00000000000
Binary files a/solr/contrib/dataimporthandler-extras/src/test-files/dihextras/test_vsdx.vsdx and /dev/null differ
diff --git a/solr/contrib/dataimporthandler-extras/src/test/org/apache/solr/handler/dataimport/TestMailEntityProcessor.java b/solr/contrib/dataimporthandler-extras/src/test/org/apache/solr/handler/dataimport/TestMailEntityProcessor.java
deleted file mode 100644
index 027a8d73279..00000000000
--- a/solr/contrib/dataimporthandler-extras/src/test/org/apache/solr/handler/dataimport/TestMailEntityProcessor.java
+++ /dev/null
@@ -1,199 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import org.apache.solr.common.SolrInputDocument;
-import org.junit.Ignore;
-import org.junit.Test;
-
-import java.text.ParseException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-// Test mailbox is like this: foldername(mailcount)
-// top1(2) -> child11(6)
-// -> child12(0)
-// top2(2) -> child21(1)
-// -> grandchild211(2)
-// -> grandchild212(1)
-// -> child22(2)
-
-/**
- * Test for MailEntityProcessor. The tests are marked as ignored because we'd need a mail server (real or mocked) for
- * these to work.
- *
- * TODO: Find a way to make the tests actually test code
- *
- *
- * @see org.apache.solr.handler.dataimport.MailEntityProcessor
- * @since solr 1.4
- */
-@Ignore("Needs a Mock Mail Server to work")
-public class TestMailEntityProcessor extends AbstractDataImportHandlerTestCase {
-
- // Credentials
- private static final String user = "user";
- private static final String password = "password";
- private static final String host = "host";
- private static final String protocol = "imaps";
-
- private static Map paramMap = new HashMap<>();
-
- @Test
- @Ignore("Needs a Mock Mail Server to work")
- public void testConnection() {
- // also tests recurse = false and default settings
- paramMap.put("folders", "top2");
- paramMap.put("recurse", "false");
- paramMap.put("processAttachement", "false");
- DataImporter di = new DataImporter();
- di.loadAndInit(getConfigFromMap(paramMap));
- @SuppressWarnings({"unchecked"})
- RequestInfo rp = new RequestInfo(null, createMap("command", "full-import"), null);
- SolrWriterImpl swi = new SolrWriterImpl();
- di.runCmd(rp, swi);
- assertEquals("top1 did not return 2 messages", swi.docs.size(), 2);
- }
-
- @Test
- @Ignore("Needs a Mock Mail Server to work")
- public void testRecursion() {
- paramMap.put("folders", "top2");
- paramMap.put("recurse", "true");
- paramMap.put("processAttachement", "false");
- DataImporter di = new DataImporter();
- di.loadAndInit(getConfigFromMap(paramMap));
- @SuppressWarnings({"unchecked"})
- RequestInfo rp = new RequestInfo(null, createMap("command", "full-import"), null);
- SolrWriterImpl swi = new SolrWriterImpl();
- di.runCmd(rp, swi);
- assertEquals("top2 and its children did not return 8 messages", swi.docs.size(), 8);
- }
-
- @Test
- @Ignore("Needs a Mock Mail Server to work")
- public void testExclude() {
- paramMap.put("folders", "top2");
- paramMap.put("recurse", "true");
- paramMap.put("processAttachement", "false");
- paramMap.put("exclude", ".*grandchild.*");
- DataImporter di = new DataImporter();
- di.loadAndInit(getConfigFromMap(paramMap));
- @SuppressWarnings({"unchecked"})
- RequestInfo rp = new RequestInfo(null, createMap("command", "full-import"), null);
- SolrWriterImpl swi = new SolrWriterImpl();
- di.runCmd(rp, swi);
- assertEquals("top2 and its direct children did not return 5 messages", swi.docs.size(), 5);
- }
-
- @Test
- @Ignore("Needs a Mock Mail Server to work")
- public void testInclude() {
- paramMap.put("folders", "top2");
- paramMap.put("recurse", "true");
- paramMap.put("processAttachement", "false");
- paramMap.put("include", ".*grandchild.*");
- DataImporter di = new DataImporter();
- di.loadAndInit(getConfigFromMap(paramMap));
- @SuppressWarnings({"unchecked"})
- RequestInfo rp = new RequestInfo(null, createMap("command", "full-import"), null);
- SolrWriterImpl swi = new SolrWriterImpl();
- di.runCmd(rp, swi);
- assertEquals("top2 and its direct children did not return 3 messages", swi.docs.size(), 3);
- }
-
- @Test
- @Ignore("Needs a Mock Mail Server to work")
- public void testIncludeAndExclude() {
- paramMap.put("folders", "top1,top2");
- paramMap.put("recurse", "true");
- paramMap.put("processAttachement", "false");
- paramMap.put("exclude", ".*top1.*");
- paramMap.put("include", ".*grandchild.*");
- DataImporter di = new DataImporter();
- di.loadAndInit(getConfigFromMap(paramMap));
- @SuppressWarnings({"unchecked"})
- RequestInfo rp = new RequestInfo(null, createMap("command", "full-import"), null);
- SolrWriterImpl swi = new SolrWriterImpl();
- di.runCmd(rp, swi);
- assertEquals("top2 and its direct children did not return 3 messages", swi.docs.size(), 3);
- }
-
- @Test
- @Ignore("Needs a Mock Mail Server to work")
- @SuppressWarnings({"unchecked"})
- public void testFetchTimeSince() throws ParseException {
- paramMap.put("folders", "top1/child11");
- paramMap.put("recurse", "true");
- paramMap.put("processAttachement", "false");
- paramMap.put("fetchMailsSince", "2008-12-26 00:00:00");
- DataImporter di = new DataImporter();
- di.loadAndInit(getConfigFromMap(paramMap));
- RequestInfo rp = new RequestInfo(null, createMap("command", "full-import"), null);
- SolrWriterImpl swi = new SolrWriterImpl();
- di.runCmd(rp, swi);
- assertEquals("top2 and its direct children did not return 3 messages", swi.docs.size(), 3);
- }
-
- private String getConfigFromMap(Map params) {
- String conf =
- "" +
- "" +
- "" +
- "" +
- "";
- params.put("user", user);
- params.put("password", password);
- params.put("host", host);
- params.put("protocol", protocol);
- StringBuilder attribs = new StringBuilder("");
- for (String key : params.keySet())
- attribs.append(" ").append(key).append("=" + "\"").append(params.get(key)).append("\"");
- attribs.append(" ");
- return conf.replace("someconfig", attribs.toString());
- }
-
- static class SolrWriterImpl extends SolrWriter {
- List docs = new ArrayList<>();
- Boolean deleteAllCalled;
- Boolean commitCalled;
-
- public SolrWriterImpl() {
- super(null, null);
- }
-
- @Override
- public boolean upload(SolrInputDocument doc) {
- return docs.add(doc);
- }
-
-
- @Override
- public void doDeleteAll() {
- deleteAllCalled = Boolean.TRUE;
- }
-
- @Override
- public void commit(boolean b) {
- commitCalled = Boolean.TRUE;
- }
- }
-}
diff --git a/solr/contrib/dataimporthandler-extras/src/test/org/apache/solr/handler/dataimport/TestTikaEntityProcessor.java b/solr/contrib/dataimporthandler-extras/src/test/org/apache/solr/handler/dataimport/TestTikaEntityProcessor.java
deleted file mode 100644
index 05acfca90c3..00000000000
--- a/solr/contrib/dataimporthandler-extras/src/test/org/apache/solr/handler/dataimport/TestTikaEntityProcessor.java
+++ /dev/null
@@ -1,221 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import java.util.Locale;
-
-/**Testcase for TikaEntityProcessor
- *
- * @since solr 3.1
- */
-public class TestTikaEntityProcessor extends AbstractDataImportHandlerTestCase {
- private String conf =
- "" +
- " " +
- " " +
- " " +
- " " +
- " " +
- " " +
- " " +
- " " +
- "";
-
- private String skipOnErrConf =
- "" +
- " " +
- " " +
- " " +
- "" +
- " " +
- " " +
- " " +
- "" +
- " " +
- "";
-
- private String spatialConf =
- "" +
- " " +
- " " +
- " " +
- " " +
- " " +
- " " +
- "";
-
- private String vsdxConf =
- "" +
- " " +
- " " +
- " " +
- " " +
- " " +
- " " +
- "";
-
- private String[] tests = {
- "//*[@numFound='1']"
- ,"//str[@name='author'][.='Grant Ingersoll']"
- ,"//str[@name='title'][.='solr-word']"
- ,"//str[@name='text']"
- };
-
- private String[] testsHTMLDefault = {
- "//*[@numFound='1']"
- , "//str[@name='text'][contains(.,'Basic div')]"
- , "//str[@name='text'][contains(.,'
')]"
- , "//str[@name='text'][not(contains(.,'
'))]" //default mapper lower-cases elements as it maps
- , "//str[@name='text'][not(contains(.,'
')]"
- , "//str[@name='text'][contains(.,'class=\"classAttribute\"')]" //attributes are lower-cased
- };
-
- private String[] testsSpatial = {
- "//*[@numFound='1']"
- };
-
- private String[] testsEmbedded = {
- "//*[@numFound='1']",
- "//str[@name='text'][contains(.,'When in the Course')]"
- };
-
- private String[] testsIgnoreEmbedded = {
- "//*[@numFound='1']",
- "//str[@name='text'][not(contains(.,'When in the Course'))]"
- };
-
- private String[] testsVSDX = {
- "//*[@numFound='1']",
- "//str[@name='text'][contains(.,'Arrears')]"
- };
-
- @BeforeClass
- public static void beforeClass() throws Exception {
- assumeFalse("This test fails on UNIX with Turkish default locale (https://issues.apache.org/jira/browse/SOLR-6387)",
- new Locale("tr").getLanguage().equals(Locale.getDefault().getLanguage()));
- initCore("dataimport-solrconfig.xml", "dataimport-schema-no-unique-key.xml", getFile("dihextras/solr").getAbsolutePath());
- }
-
- @Test
- public void testIndexingWithTikaEntityProcessor() throws Exception {
- runFullImport(conf);
- assertQ(req("*:*"), tests );
- }
-
- @Test
- public void testSkip() throws Exception {
- runFullImport(skipOnErrConf);
- assertQ(req("*:*"), "//*[@numFound='1']");
- }
-
- @Test
- public void testVSDX() throws Exception {
- //this ensures that we've included the curvesapi dependency
- //and that the ConnectsType class is bundled with poi-ooxml-schemas.
- runFullImport(vsdxConf);
- assertQ(req("*:*"), testsVSDX);
- }
-
- @Test
- public void testTikaHTMLMapperEmpty() throws Exception {
- runFullImport(getConfigHTML(null));
- assertQ(req("*:*"), testsHTMLDefault);
- }
-
- @Test
- public void testTikaHTMLMapperDefault() throws Exception {
- runFullImport(getConfigHTML("default"));
- assertQ(req("*:*"), testsHTMLDefault);
- }
-
- @Test
- public void testTikaHTMLMapperIdentity() throws Exception {
- runFullImport(getConfigHTML("identity"));
- assertQ(req("*:*"), testsHTMLIdentity);
- }
-
- @Test
- public void testTikaGeoMetadata() throws Exception {
- runFullImport(spatialConf);
- String pt = "38.97,-77.018";
- Double distance = 5.0d;
- assertQ(req("q", "*:* OR foo_i:" + random().nextInt(100), "fq",
- "{!geofilt sfield=\"home\"}\"",
- "pt", pt, "d", String.valueOf(distance)), testsSpatial);
- }
-
- private String getConfigHTML(String htmlMapper) {
- return
- "" +
- " " +
- " " +
- " " +
- " " +
- " " +
- " " +
- "";
-
- }
-
- @Test
- public void testEmbeddedDocsLegacy() throws Exception {
- //test legacy behavior: ignore embedded docs
- runFullImport(conf);
- assertQ(req("*:*"), testsIgnoreEmbedded);
- }
-
- @Test
- public void testEmbeddedDocsTrue() throws Exception {
- runFullImport(getConfigEmbedded(true));
- assertQ(req("*:*"), testsEmbedded);
- }
-
- @Test
- public void testEmbeddedDocsFalse() throws Exception {
- runFullImport(getConfigEmbedded(false));
- assertQ(req("*:*"), testsIgnoreEmbedded);
- }
-
- private String getConfigEmbedded(boolean extractEmbedded) {
- return
- "" +
- " " +
- " " +
- " " +
- " " +
- " " +
- " " +
- " " +
- " " +
- "";
- }
-}
diff --git a/solr/contrib/dataimporthandler/README.md b/solr/contrib/dataimporthandler/README.md
deleted file mode 100644
index 8dc9391494e..00000000000
--- a/solr/contrib/dataimporthandler/README.md
+++ /dev/null
@@ -1,26 +0,0 @@
-Apache Solr - DataImportHandler
-================================
-
-Introduction
-------------
-DataImportHandler is a data import tool for Solr which makes importing data from Databases, XML files and
-HTTP data sources quick and easy.
-
-Important Note
---------------
-Although Solr strives to be agnostic of the Locale where the server is
-running, some code paths in DataImportHandler are known to depend on the
-System default Locale, Timezone, or Charset. It is recommended that when
-running Solr you set the following system properties:
- -Duser.language=xx -Duser.country=YY -Duser.timezone=ZZZ
-
-where xx, YY, and ZZZ are consistent with any database server's configuration.
-
-Deprecation notice
-------------------
-This contrib module is deprecated as of v8.6, scheduled for removal in Solr 9.0.
-The reason is that DIH is no longer being maintained in a manner we feel is necessary in order to keep it
-healthy and secure. Also it was not designed to work with SolrCloud and does not meet current performance requirements.
-
-The project hopes that the community will take over maintenance of DIH as a 3rd party package (See SOLR-14066 for more details). Please reach out to us at the dev@ mailing list if you want to help.
-
diff --git a/solr/contrib/dataimporthandler/build.gradle b/solr/contrib/dataimporthandler/build.gradle
deleted file mode 100644
index 9286d4317b6..00000000000
--- a/solr/contrib/dataimporthandler/build.gradle
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-apply plugin: 'java-library'
-
-description = 'Data Import Handler'
-
-dependencies {
- implementation project(':solr:core')
-
- testImplementation project(':solr:test-framework')
-
- testImplementation('org.mockito:mockito-core', {
- exclude group: "net.bytebuddy", module: "byte-buddy-agent"
- })
- testImplementation ('org.hsqldb:hsqldb')
- testImplementation ('org.apache.derby:derby')
- testImplementation ('org.objenesis:objenesis')
-}
diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/BinContentStreamDataSource.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/BinContentStreamDataSource.java
deleted file mode 100644
index f4b1d7a24bf..00000000000
--- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/BinContentStreamDataSource.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import org.apache.solr.common.util.ContentStream;
-import static org.apache.solr.handler.dataimport.DataImportHandlerException.SEVERE;
-
-import java.io.InputStream;
-import java.io.IOException;
-import java.util.Properties;
-/**
- *
A data source implementation which can be used to read binary stream from content streams.
- * This API is experimental and may change in the future.
- *
- * @since solr 3.1
- */
-
-public class BinContentStreamDataSource extends DataSource {
- private ContextImpl context;
- private ContentStream contentStream;
- private InputStream in;
-
-
- @Override
- public void init(Context context, Properties initProps) {
- this.context = (ContextImpl) context;
- }
-
- @Override
- public InputStream getData(String query) {
- contentStream = context.getDocBuilder().getReqParams().getContentStream();
- if (contentStream == null)
- throw new DataImportHandlerException(SEVERE, "No stream available. The request has no body");
- try {
- return in = contentStream.getStream();
- } catch (IOException e) {
- DataImportHandlerException.wrapAndThrow(SEVERE, e);
- return null;
- }
- }
-
- @Override
- public void close() {
- if (contentStream != null) {
- try {
- if (in == null) in = contentStream.getStream();
- in.close();
- } catch (IOException e) {
- /*no op*/
- }
- }
- }
-}
diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/BinFileDataSource.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/BinFileDataSource.java
deleted file mode 100644
index dc7a0f552e3..00000000000
--- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/BinFileDataSource.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import static org.apache.solr.handler.dataimport.DataImportHandlerException.wrapAndThrow;
-import static org.apache.solr.handler.dataimport.DataImportHandlerException.SEVERE;
-
-import java.io.InputStream;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.util.Properties;
-/**
- *
- * This API is experimental and may change in the future.
- *
- * @since solr 3.1
- */
-
-public class BinFileDataSource extends DataSource{
- protected String basePath;
- @Override
- public void init(Context context, Properties initProps) {
- basePath = initProps.getProperty(FileDataSource.BASE_PATH);
- }
-
- @Override
- public InputStream getData(String query) {
- File f = FileDataSource.getFile(basePath,query);
- try {
- return new FileInputStream(f);
- } catch (FileNotFoundException e) {
- wrapAndThrow(SEVERE,e,"Unable to open file "+f.getAbsolutePath());
- return null;
- }
- }
-
- @Override
- public void close() {
-
- }
-}
diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/BinURLDataSource.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/BinURLDataSource.java
deleted file mode 100644
index 03a30ab07a9..00000000000
--- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/BinURLDataSource.java
+++ /dev/null
@@ -1,104 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import static org.apache.solr.handler.dataimport.DataImportHandlerException.*;
-import static org.apache.solr.handler.dataimport.URLDataSource.*;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.InputStream;
-import java.lang.invoke.MethodHandles;
-import java.net.URL;
-import java.net.URLConnection;
-import java.util.Properties;
-/**
- *
A data source implementation which can be used to read binary streams using HTTP.
- * This API is experimental and may change in the future.
- *
- * @since solr 3.1
- */
-public class BinURLDataSource extends DataSource{
- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-
- private String baseUrl;
- private int connectionTimeout = CONNECTION_TIMEOUT;
-
- private int readTimeout = READ_TIMEOUT;
-
- private Context context;
-
- private Properties initProps;
-
- public BinURLDataSource() { }
-
- @Override
- public void init(Context context, Properties initProps) {
- this.context = context;
- this.initProps = initProps;
-
- baseUrl = getInitPropWithReplacements(BASE_URL);
- String cTimeout = getInitPropWithReplacements(CONNECTION_TIMEOUT_FIELD_NAME);
- String rTimeout = getInitPropWithReplacements(READ_TIMEOUT_FIELD_NAME);
- if (cTimeout != null) {
- try {
- connectionTimeout = Integer.parseInt(cTimeout);
- } catch (NumberFormatException e) {
- log.warn("Invalid connection timeout: {}", cTimeout);
- }
- }
- if (rTimeout != null) {
- try {
- readTimeout = Integer.parseInt(rTimeout);
- } catch (NumberFormatException e) {
- log.warn("Invalid read timeout: {}", rTimeout);
- }
- }
- }
-
- @Override
- public InputStream getData(String query) {
- URL url = null;
- try {
- if (URIMETHOD.matcher(query).find()) url = new URL(query);
- else url = new URL(baseUrl + query);
- log.debug("Accessing URL: {}", url);
- URLConnection conn = url.openConnection();
- conn.setConnectTimeout(connectionTimeout);
- conn.setReadTimeout(readTimeout);
- return conn.getInputStream();
- } catch (Exception e) {
- log.error("Exception thrown while getting data", e);
- wrapAndThrow (SEVERE, e, "Exception in invoking url " + url);
- return null;//unreachable
- }
- }
-
- @Override
- public void close() { }
-
- private String getInitPropWithReplacements(String propertyName) {
- final String expr = initProps.getProperty(propertyName);
- if (expr == null) {
- return null;
- }
- return context.replaceTokens(expr);
- }
-}
diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/CachePropertyUtil.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/CachePropertyUtil.java
deleted file mode 100644
index 544761f8d88..00000000000
--- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/CachePropertyUtil.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-public class CachePropertyUtil {
- public static String getAttributeValueAsString(Context context, String attr) {
- Object o = context.getSessionAttribute(attr, Context.SCOPE_ENTITY);
- if (o == null) {
- o = context.getResolvedEntityAttribute(attr);
- }
- if (o == null && context.getRequestParameters() != null) {
- o = context.getRequestParameters().get(attr);
- }
- if (o == null) {
- return null;
- }
- return o.toString();
- }
-
- public static Object getAttributeValue(Context context, String attr) {
- Object o = context.getSessionAttribute(attr, Context.SCOPE_ENTITY);
- if (o == null) {
- o = context.getResolvedEntityAttribute(attr);
- }
- if (o == null && context.getRequestParameters() != null) {
- o = context.getRequestParameters().get(attr);
- }
- if (o == null) {
- return null;
- }
- return o;
- }
-
-}
diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/ClobTransformer.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/ClobTransformer.java
deleted file mode 100644
index 2e9d93a0c1a..00000000000
--- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/ClobTransformer.java
+++ /dev/null
@@ -1,85 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import static org.apache.solr.handler.dataimport.HTMLStripTransformer.TRUE;
-
-import java.io.IOException;
-import java.io.Reader;
-import java.sql.Clob;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-
-/**
- * {@link Transformer} instance which converts a {@link Clob} to a {@link String}.
- *
- * This API is experimental and subject to change
- *
- * @since solr 1.4
- */
-public class ClobTransformer extends Transformer {
- @Override
- public Object transformRow(Map aRow, Context context) {
- for (Map map : context.getAllEntityFields()) {
- if (!TRUE.equals(map.get(CLOB))) continue;
- String column = map.get(DataImporter.COLUMN);
- String srcCol = map.get(RegexTransformer.SRC_COL_NAME);
- if (srcCol == null)
- srcCol = column;
- Object o = aRow.get(srcCol);
- if (o instanceof List) {
- @SuppressWarnings({"unchecked"})
- List inputs = (List) o;
- List results = new ArrayList<>();
- for (Object input : inputs) {
- if (input instanceof Clob) {
- Clob clob = (Clob) input;
- results.add(readFromClob(clob));
- }
- }
- aRow.put(column, results);
- } else {
- if (o instanceof Clob) {
- Clob clob = (Clob) o;
- aRow.put(column, readFromClob(clob));
- }
- }
- }
- return aRow;
- }
-
- private String readFromClob(Clob clob) {
- Reader reader = FieldReaderDataSource.readCharStream(clob);
- StringBuilder sb = new StringBuilder();
- char[] buf = new char[1024];
- int len;
- try {
- while ((len = reader.read(buf)) != -1) {
- sb.append(buf, 0, len);
- }
- } catch (IOException e) {
- DataImportHandlerException.wrapAndThrow(DataImportHandlerException.SEVERE, e);
- }
- return sb.toString();
- }
-
- public static final String CLOB = "clob";
-}
diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/ConfigParseUtil.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/ConfigParseUtil.java
deleted file mode 100644
index 179df231526..00000000000
--- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/ConfigParseUtil.java
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-
-import org.w3c.dom.Element;
-import org.w3c.dom.NamedNodeMap;
-import org.w3c.dom.Node;
-import org.w3c.dom.NodeList;
-
-public class ConfigParseUtil {
- public static String getStringAttribute(Element e, String name, String def) {
- String r = e.getAttribute(name);
- if (r == null || "".equals(r.trim()))
- r = def;
- return r;
- }
-
- public static HashMap getAllAttributes(Element e) {
- HashMap m = new HashMap<>();
- NamedNodeMap nnm = e.getAttributes();
- for (int i = 0; i < nnm.getLength(); i++) {
- m.put(nnm.item(i).getNodeName(), nnm.item(i).getNodeValue());
- }
- return m;
- }
-
- public static String getText(Node elem, StringBuilder buffer) {
- if (elem.getNodeType() != Node.CDATA_SECTION_NODE) {
- NodeList childs = elem.getChildNodes();
- for (int i = 0; i < childs.getLength(); i++) {
- Node child = childs.item(i);
- short childType = child.getNodeType();
- if (childType != Node.COMMENT_NODE
- && childType != Node.PROCESSING_INSTRUCTION_NODE) {
- getText(child, buffer);
- }
- }
- } else {
- buffer.append(elem.getNodeValue());
- }
-
- return buffer.toString();
- }
-
- public static List getChildNodes(Element e, String byName) {
- List result = new ArrayList<>();
- NodeList l = e.getChildNodes();
- for (int i = 0; i < l.getLength(); i++) {
- if (e.equals(l.item(i).getParentNode())
- && byName.equals(l.item(i).getNodeName()))
- result.add((Element) l.item(i));
- }
- return result;
- }
-}
diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/ContentStreamDataSource.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/ContentStreamDataSource.java
deleted file mode 100644
index 4482160c22b..00000000000
--- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/ContentStreamDataSource.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import org.apache.solr.common.util.ContentStream;
-import static org.apache.solr.handler.dataimport.DataImportHandlerException.SEVERE;
-
-import java.io.IOException;
-import java.io.Reader;
-import java.util.Properties;
-
-/**
- * A DataSource implementation which reads from the ContentStream of a POST request
- *
- * This API is experimental and may change in the future.
- *
- * @since solr 1.4
- */
-public class ContentStreamDataSource extends DataSource {
- private ContextImpl context;
- private ContentStream contentStream;
- private Reader reader;
-
- @Override
- public void init(Context context, Properties initProps) {
- this.context = (ContextImpl) context;
- }
-
- @Override
- public Reader getData(String query) {
- contentStream = context.getDocBuilder().getReqParams().getContentStream();
- if (contentStream == null)
- throw new DataImportHandlerException(SEVERE, "No stream available. The request has no body");
- try {
- return reader = contentStream.getReader();
- } catch (IOException e) {
- DataImportHandlerException.wrapAndThrow(SEVERE, e);
- return null;
- }
- }
-
- @Override
- public void close() {
- if (contentStream != null) {
- try {
- if (reader == null) reader = contentStream.getReader();
- reader.close();
- } catch (IOException e) {
- }
- }
- }
-}
diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/Context.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/Context.java
deleted file mode 100644
index 70dbbcb6ec6..00000000000
--- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/Context.java
+++ /dev/null
@@ -1,221 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import org.apache.solr.core.SolrCore;
-
-import java.util.List;
-import java.util.Map;
-
-/**
- *
- * This abstract class gives access to all available objects. So any
- * component implemented by a user can have the full power of DataImportHandler
- *
- * This API is experimental and subject to change
- *
- * @since solr 1.3
- */
-public abstract class Context {
- public static final String FULL_DUMP = "FULL_DUMP", DELTA_DUMP = "DELTA_DUMP", FIND_DELTA = "FIND_DELTA";
-
- /**
- * An object stored in entity scope is valid only for the current entity for the current document only.
- */
- public static final String SCOPE_ENTITY = "entity";
-
- /**
- * An object stored in global scope is available for the current import only but across entities and documents.
- */
- public static final String SCOPE_GLOBAL = "global";
-
- /**
- * An object stored in document scope is available for the current document only but across entities.
- */
- public static final String SCOPE_DOC = "document";
-
- /**
- * An object stored in 'solrcore' scope is available across imports, entities and documents throughout the life of
- * a solr core. A solr core unload or reload will destroy this data.
- */
- public static final String SCOPE_SOLR_CORE = "solrcore";
-
- /**
- * Get the value of any attribute put into this entity
- *
- * @param name name of the attribute eg: 'name'
- * @return value of named attribute in entity
- */
- public abstract String getEntityAttribute(String name);
-
- /**
- * Get the value of any attribute put into this entity after resolving all variables found in the attribute value
- * @param name name of the attribute
- * @return value of the named attribute after resolving all variables
- */
- public abstract String getResolvedEntityAttribute(String name);
-
- /**
- * Returns all the fields put into an entity. each item (which is a map ) in
- * the list corresponds to one field. each if the map contains the attribute
- * names and values in a field
- *
- * @return all fields in an entity
- */
- public abstract List> getAllEntityFields();
-
- /**
- * Returns the VariableResolver used in this entity which can be used to
- * resolve the tokens in ${<namespce.name>}
- *
- * @return a VariableResolver instance
- * @see org.apache.solr.handler.dataimport.VariableResolver
- */
-
- public abstract VariableResolver getVariableResolver();
-
- /**
- * Gets the datasource instance defined for this entity. Do not close() this instance.
- * Transformers should use the getDataSource(String name) method.
- *
- * @return a new DataSource instance as configured for the current entity
- * @see org.apache.solr.handler.dataimport.DataSource
- * @see #getDataSource(String)
- */
- @SuppressWarnings({"rawtypes"})
- public abstract DataSource getDataSource();
-
- /**
- * Gets a new DataSource instance with a name. Ensure that you close() this after use
- * because this is created just for this method call.
- *
- * @param name Name of the dataSource as defined in the dataSource tag
- * @return a new DataSource instance
- * @see org.apache.solr.handler.dataimport.DataSource
- */
- @SuppressWarnings({"rawtypes"})
- public abstract DataSource getDataSource(String name);
-
- /**
- * Returns the instance of EntityProcessor used for this entity
- *
- * @return instance of EntityProcessor used for the current entity
- * @see org.apache.solr.handler.dataimport.EntityProcessor
- */
- public abstract EntityProcessor getEntityProcessor();
-
- /**
- * Store values in a certain name and scope (entity, document,global)
- *
- * @param name the key
- * @param val the value
- * @param scope the scope in which the given key, value pair is to be stored
- */
- public abstract void setSessionAttribute(String name, Object val, String scope);
-
- /**
- * get a value by name in the given scope (entity, document,global)
- *
- * @param name the key
- * @param scope the scope from which the value is to be retrieved
- * @return the object stored in the given scope with the given key
- */
- public abstract Object getSessionAttribute(String name, String scope);
-
- /**
- * Get the context instance for the parent entity. works only in the full dump
- * If the current entity is rootmost a null is returned
- *
- * @return parent entity's Context
- */
- public abstract Context getParentContext();
-
- /**
- * The request parameters passed over HTTP for this command the values in the
- * map are either String(for single valued parameters) or List<String> (for
- * multi-valued parameters)
- *
- * @return the request parameters passed in the URL to initiate this process
- */
- public abstract Map getRequestParameters();
-
- /**
- * Returns if the current entity is the root entity
- *
- * @return true if current entity is the root entity, false otherwise
- */
- public abstract boolean isRootEntity();
-
- /**
- * Returns the current process FULL_DUMP, DELTA_DUMP, FIND_DELTA
- *
- * @return the type of the current running process
- */
- public abstract String currentProcess();
-
- /**
- * Exposing the actual SolrCore to the components
- *
- * @return the core
- */
- public abstract SolrCore getSolrCore();
-
- /**
- * Makes available some basic running statistics such as "docCount",
- * "deletedDocCount", "rowCount", "queryCount" and "skipDocCount"
- *
- * @return a Map containing running statistics of the current import
- */
- public abstract Map getStats();
-
- /**
- * Returns the text specified in the script tag in the data-config.xml
- */
- public abstract String getScript();
-
- /**
- * Returns the language of the script as specified in the script tag in data-config.xml
- */
- public abstract String getScriptLanguage();
-
- /**delete a document by id
- */
- public abstract void deleteDoc(String id);
-
- /**delete documents by query
- */
- public abstract void deleteDocByQuery(String query);
-
- /**Use this directly to resolve variable
- * @param var the variable name
- * @return the resolved value
- */
- public abstract Object resolve(String var);
-
- /** Resolve variables in a template
- *
- * @return The string w/ variables resolved
- */
- public abstract String replaceTokens(String template);
-
-}
diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/ContextImpl.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/ContextImpl.java
deleted file mode 100644
index 3d9f3868508..00000000000
--- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/ContextImpl.java
+++ /dev/null
@@ -1,264 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-import org.apache.solr.core.SolrCore;
-import org.apache.solr.handler.dataimport.config.Script;
-
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-/**
- *
- * An implementation for the Context
- *
- * This API is experimental and subject to change
- *
- * @since solr 1.3
- */
-public class ContextImpl extends Context {
- protected EntityProcessorWrapper epw;
-
- private ContextImpl parent;
-
- private VariableResolver resolver;
-
- @SuppressWarnings({"rawtypes"})
- private DataSource ds;
-
- private String currProcess;
-
- private Map requestParams;
-
- private DataImporter dataImporter;
-
- private Map entitySession, globalSession;
-
- private Exception lastException = null;
-
- DocBuilder.DocWrapper doc;
-
- DocBuilder docBuilder;
-
-
-
- public ContextImpl(EntityProcessorWrapper epw, VariableResolver resolver,
- @SuppressWarnings({"rawtypes"})DataSource ds, String currProcess,
- Map global, ContextImpl parentContext, DocBuilder docBuilder) {
- this.epw = epw;
- this.docBuilder = docBuilder;
- this.resolver = resolver;
- this.ds = ds;
- this.currProcess = currProcess;
- if (docBuilder != null) {
- this.requestParams = docBuilder.getReqParams().getRawParams();
- dataImporter = docBuilder.dataImporter;
- }
- globalSession = global;
- parent = parentContext;
- }
-
- @Override
- public String getEntityAttribute(String name) {
- return epw==null || epw.getEntity() == null ? null : epw.getEntity().getAllAttributes().get(name);
- }
-
- @Override
- public String getResolvedEntityAttribute(String name) {
- return epw==null || epw.getEntity() == null ? null : resolver.replaceTokens(epw.getEntity().getAllAttributes().get(name));
- }
-
- @Override
- public List> getAllEntityFields() {
- return epw==null || epw.getEntity() == null ? Collections.emptyList() : epw.getEntity().getAllFieldsList();
- }
-
- @Override
- public VariableResolver getVariableResolver() {
- return resolver;
- }
-
- @Override
- @SuppressWarnings({"rawtypes"})
- public DataSource getDataSource() {
- if (ds != null) return ds;
- if(epw==null) { return null; }
- if (epw!=null && epw.getDatasource() == null) {
- epw.setDatasource(dataImporter.getDataSourceInstance(epw.getEntity(), epw.getEntity().getDataSourceName(), this));
- }
- if (epw!=null && epw.getDatasource() != null && docBuilder != null && docBuilder.verboseDebug &&
- Context.FULL_DUMP.equals(currentProcess())) {
- //debug is not yet implemented properly for deltas
- epw.setDatasource(docBuilder.getDebugLogger().wrapDs(epw.getDatasource()));
- }
- return epw.getDatasource();
- }
-
- @Override
- @SuppressWarnings({"rawtypes"})
- public DataSource getDataSource(String name) {
- return dataImporter.getDataSourceInstance(epw==null ? null : epw.getEntity(), name, this);
- }
-
- @Override
- public boolean isRootEntity() {
- return epw==null ? false : epw.getEntity().isDocRoot();
- }
-
- @Override
- public String currentProcess() {
- return currProcess;
- }
-
- @Override
- public Map getRequestParameters() {
- return requestParams;
- }
-
- @Override
- public EntityProcessor getEntityProcessor() {
- return epw;
- }
-
- @Override
- public void setSessionAttribute(String name, Object val, String scope) {
- if(name == null) {
- return;
- }
- if (Context.SCOPE_ENTITY.equals(scope)) {
- if (entitySession == null) {
- entitySession = new HashMap<>();
- }
- entitySession.put(name, val);
- } else if (Context.SCOPE_GLOBAL.equals(scope)) {
- if (globalSession != null) {
- globalSession.put(name, val);
- }
- } else if (Context.SCOPE_DOC.equals(scope)) {
- DocBuilder.DocWrapper doc = getDocument();
- if (doc != null) {
- doc.setSessionAttribute(name, val);
- }
- } else if (SCOPE_SOLR_CORE.equals(scope)){
- if(dataImporter != null) {
- dataImporter.putToCoreScopeSession(name, val);
- }
- }
- }
-
- @Override
- public Object getSessionAttribute(String name, String scope) {
- if (Context.SCOPE_ENTITY.equals(scope)) {
- if (entitySession == null)
- return null;
- return entitySession.get(name);
- } else if (Context.SCOPE_GLOBAL.equals(scope)) {
- if (globalSession != null) {
- return globalSession.get(name);
- }
- } else if (Context.SCOPE_DOC.equals(scope)) {
- DocBuilder.DocWrapper doc = getDocument();
- return doc == null ? null: doc.getSessionAttribute(name);
- } else if (SCOPE_SOLR_CORE.equals(scope)){
- return dataImporter == null ? null : dataImporter.getFromCoreScopeSession(name);
- }
- return null;
- }
-
- @Override
- public Context getParentContext() {
- return parent;
- }
-
- private DocBuilder.DocWrapper getDocument() {
- ContextImpl c = this;
- while (true) {
- if (c.doc != null)
- return c.doc;
- if (c.parent != null)
- c = c.parent;
- else
- return null;
- }
- }
-
- void setDoc(DocBuilder.DocWrapper docWrapper) {
- this.doc = docWrapper;
- }
-
-
- @Override
- public SolrCore getSolrCore() {
- return dataImporter == null ? null : dataImporter.getCore();
- }
-
-
- @Override
- public Map getStats() {
- return docBuilder != null ? docBuilder.importStatistics.getStatsSnapshot() : Collections.emptyMap();
- }
-
- @Override
- public String getScript() {
- if (dataImporter != null) {
- Script script = dataImporter.getConfig().getScript();
- return script == null ? null : script.getText();
- }
- return null;
- }
-
- @Override
- public String getScriptLanguage() {
- if (dataImporter != null) {
- Script script = dataImporter.getConfig().getScript();
- return script == null ? null : script.getLanguage();
- }
- return null;
- }
-
- @Override
- public void deleteDoc(String id) {
- if(docBuilder != null){
- docBuilder.writer.deleteDoc(id);
- }
- }
-
- @Override
- public void deleteDocByQuery(String query) {
- if(docBuilder != null){
- docBuilder.writer.deleteByQuery(query);
- }
- }
-
- DocBuilder getDocBuilder(){
- return docBuilder;
- }
- @Override
- public Object resolve(String var) {
- return resolver.resolve(var);
- }
-
- @Override
- public String replaceTokens(String template) {
- return resolver.replaceTokens(template);
- }
-
- public Exception getLastException() { return lastException; }
-
- public void setLastException(Exception lastException) {this.lastException = lastException; }
-}
diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DIHCache.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DIHCache.java
deleted file mode 100644
index a67b3e42963..00000000000
--- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DIHCache.java
+++ /dev/null
@@ -1,103 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import java.util.Iterator;
-import java.util.Map;
-
-/**
- *
- * A cache that allows a DIH entity's data to persist locally prior being joined
- * to other data and/or indexed.
- *
- * Opens the cache using the specified properties. The {@link Context}
- * includes any parameters needed by the cache impl. This must be called
- * before any read/write operations are permitted.
- */
- void open(Context context);
-
- /**
- *
- * Releases resources used by this cache, if possible. The cache is flushed
- * but not destroyed.
- *
- */
- void close();
-
- /**
- *
- * Persists any pending data to the cache
- *
- */
- void flush();
-
- /**
- *
- * Closes the cache, if open. Then removes all data, possibly removing the
- * cache entirely from persistent storage.
- *
- */
- public void destroy();
-
- /**
- *
- * Adds a document. If a document already exists with the same key, both
- * documents will exist in the cache, as the cache allows duplicate keys. To
- * update a key's documents, first call delete(Object key).
- *
- */
- void add(Map rec);
-
- /**
- *
- * Returns an iterator, allowing callers to iterate through the entire cache
- * in key, then insertion, order.
- *
- * Delete all documents associated with the given key
- *
- */
- void delete(Object key);
-
- /**
- *
- * Delete all data from the cache,leaving the empty cache intact.
- *
- */
- void deleteAll();
-
-}
diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DIHCacheSupport.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DIHCacheSupport.java
deleted file mode 100644
index 2f3d95743fd..00000000000
--- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DIHCacheSupport.java
+++ /dev/null
@@ -1,279 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import static org.apache.solr.handler.dataimport.DataImportHandlerException.wrapAndThrow;
-
-import java.lang.invoke.MethodHandles;
-import java.lang.reflect.Constructor;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.Map;
-
-import org.apache.solr.common.SolrException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class DIHCacheSupport {
- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
- private String cacheForeignKey;
- private String cacheImplName;
- private Map queryVsCache = new HashMap<>();
- private Map>> queryVsCacheIterator;
- private Iterator> dataSourceRowCache;
- private boolean cacheDoKeyLookup;
-
- public DIHCacheSupport(Context context, String cacheImplName) {
- this.cacheImplName = cacheImplName;
-
- Relation r = new Relation(context);
- cacheDoKeyLookup = r.doKeyLookup;
- String cacheKey = r.primaryKey;
- cacheForeignKey = r.foreignKey;
-
- context.setSessionAttribute(DIHCacheSupport.CACHE_PRIMARY_KEY, cacheKey,
- Context.SCOPE_ENTITY);
- context.setSessionAttribute(DIHCacheSupport.CACHE_FOREIGN_KEY, cacheForeignKey,
- Context.SCOPE_ENTITY);
- context.setSessionAttribute(DIHCacheSupport.CACHE_DELETE_PRIOR_DATA,
- "true", Context.SCOPE_ENTITY);
- context.setSessionAttribute(DIHCacheSupport.CACHE_READ_ONLY, "false",
- Context.SCOPE_ENTITY);
- }
-
- static class Relation{
- protected final boolean doKeyLookup;
- protected final String foreignKey;
- protected final String primaryKey;
-
- public Relation(Context context) {
- String where = context.getEntityAttribute("where");
- String cacheKey = context.getEntityAttribute(DIHCacheSupport.CACHE_PRIMARY_KEY);
- String lookupKey = context.getEntityAttribute(DIHCacheSupport.CACHE_FOREIGN_KEY);
- if (cacheKey != null && lookupKey == null) {
- throw new DataImportHandlerException(DataImportHandlerException.SEVERE,
- "'cacheKey' is specified for the entity "
- + context.getEntityAttribute("name")
- + " but 'cacheLookup' is missing");
-
- }
- if (where == null && cacheKey == null) {
- doKeyLookup = false;
- primaryKey = null;
- foreignKey = null;
- } else {
- if (where != null) {
- String[] splits = where.split("=");
- primaryKey = splits[0];
- foreignKey = splits[1].trim();
- } else {
- primaryKey = cacheKey;
- foreignKey = lookupKey;
- }
- doKeyLookup = true;
- }
- }
-
- @Override
- public String toString() {
- return "Relation "
- + primaryKey + "="+foreignKey ;
- }
-
-
- }
-
- private DIHCache instantiateCache(Context context) {
- DIHCache cache = null;
- try {
- @SuppressWarnings("unchecked")
- Class cacheClass = DocBuilder.loadClass(cacheImplName, context
- .getSolrCore());
- Constructor constr = cacheClass.getConstructor();
- cache = constr.newInstance();
- cache.open(context);
- } catch (Exception e) {
- throw new DataImportHandlerException(DataImportHandlerException.SEVERE,
- "Unable to load Cache implementation:" + cacheImplName, e);
- }
- return cache;
- }
-
- public void initNewParent(Context context) {
- dataSourceRowCache = null;
- queryVsCacheIterator = new HashMap<>();
- for (Map.Entry entry : queryVsCache.entrySet()) {
- queryVsCacheIterator.put(entry.getKey(), entry.getValue().iterator());
- }
- }
-
- public void destroyAll() {
- if (queryVsCache != null) {
- for (DIHCache cache : queryVsCache.values()) {
- cache.destroy();
- }
- }
- queryVsCache = null;
- dataSourceRowCache = null;
- cacheForeignKey = null;
- }
-
- /**
- *
- * Get all the rows from the datasource for the given query and cache them
- *
- */
- public void populateCache(String query,
- Iterator> rowIterator) {
- Map aRow = null;
- DIHCache cache = queryVsCache.get(query);
- while ((aRow = getNextFromCache(query, rowIterator)) != null) {
- cache.add(aRow);
- }
- }
-
- private Map getNextFromCache(String query,
- Iterator> rowIterator) {
- try {
- if (rowIterator == null) return null;
- if (rowIterator.hasNext()) return rowIterator.next();
- return null;
- } catch (Exception e) {
- SolrException.log(log, "getNextFromCache() failed for query '" + query
- + "'", e);
- wrapAndThrow(DataImportHandlerException.WARN, e);
- return null;
- }
- }
-
- public Map getCacheData(Context context, String query,
- Iterator> rowIterator) {
- if (cacheDoKeyLookup) {
- return getIdCacheData(context, query, rowIterator);
- } else {
- return getSimpleCacheData(context, query, rowIterator);
- }
- }
-
- /**
- * If the where clause is present the cache is sql Vs Map of key Vs List of
- * Rows.
- *
- * @param query
- * the query string for which cached data is to be returned
- *
- * @return the cached row corresponding to the given query after all variables
- * have been resolved
- */
- protected Map getIdCacheData(Context context, String query,
- Iterator> rowIterator) {
- Object key = context.resolve(cacheForeignKey);
- if (key == null) {
- throw new DataImportHandlerException(DataImportHandlerException.WARN,
- "The cache lookup value : " + cacheForeignKey
- + " is resolved to be null in the entity :"
- + context.getEntityAttribute("name"));
-
- }
- if (dataSourceRowCache == null) {
- DIHCache cache = queryVsCache.get(query);
-
- if (cache == null) {
- cache = instantiateCache(context);
- queryVsCache.put(query, cache);
- populateCache(query, rowIterator);
- }
- dataSourceRowCache = cache.iterator(key);
- }
- return getFromRowCacheTransformed();
- }
-
- /**
- * If where clause is not present the cache is a Map of query vs List of Rows.
- *
- * @param query
- * string for which cached row is to be returned
- *
- * @return the cached row corresponding to the given query
- */
- protected Map getSimpleCacheData(Context context,
- String query, Iterator> rowIterator) {
- if (dataSourceRowCache == null) {
- DIHCache cache = queryVsCache.get(query);
- if (cache == null) {
- cache = instantiateCache(context);
- queryVsCache.put(query, cache);
- populateCache(query, rowIterator);
- queryVsCacheIterator.put(query, cache.iterator());
- }
- Iterator> cacheIter = queryVsCacheIterator.get(query);
- dataSourceRowCache = cacheIter;
- }
-
- return getFromRowCacheTransformed();
- }
-
- protected Map getFromRowCacheTransformed() {
- if (dataSourceRowCache == null || !dataSourceRowCache.hasNext()) {
- dataSourceRowCache = null;
- return null;
- }
- Map r = dataSourceRowCache.next();
- return r;
- }
-
- /**
- *
- * Specify the class for the cache implementation
- *
- */
- public static final String CACHE_IMPL = "cacheImpl";
-
- /**
- *
- * If the cache supports persistent data, set to "true" to delete any prior
- * persisted data before running the entity.
- *
- */
-
- public static final String CACHE_DELETE_PRIOR_DATA = "cacheDeletePriorData";
- /**
- *
- * Specify the Foreign Key from the parent entity to join on. Use if the cache
- * is on a child entity.
- *
- */
- public static final String CACHE_FOREIGN_KEY = "cacheLookup";
-
- /**
- *
- * Specify the Primary Key field from this Entity to map the input records
- * with
- *
- */
- public static final String CACHE_PRIMARY_KEY = "cacheKey";
- /**
- *
- * If true, a pre-existing cache is re-opened for read-only access.
- *
- */
- public static final String CACHE_READ_ONLY = "cacheReadOnly";
-
-
-
-
-}
diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DIHLogLevels.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DIHLogLevels.java
deleted file mode 100644
index 24732d1454f..00000000000
--- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DIHLogLevels.java
+++ /dev/null
@@ -1,21 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-public enum DIHLogLevels {
- START_ENTITY, END_ENTITY, TRANSFORMED_ROW, ENTITY_META, PRE_TRANSFORMER_ROW, START_DOC, END_DOC, ENTITY_OUT, ROW_END, TRANSFORMER_EXCEPTION, ENTITY_EXCEPTION, DISABLE_LOGGING, ENABLE_LOGGING, NONE
-}
diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DIHProperties.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DIHProperties.java
deleted file mode 100644
index f51ef0713b6..00000000000
--- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DIHProperties.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import java.util.Date;
-import java.util.Map;
-
-/**
- * Implementations write out properties about the last data import
- * for use by the next import. ex: to persist the last import timestamp
- * so that future delta imports can know what needs to be updated.
- *
- * @lucene.experimental
- */
-public abstract class DIHProperties {
-
- public abstract void init(DataImporter dataImporter, Map initParams);
-
- public abstract boolean isWritable();
-
- public abstract void persist(Map props);
-
- public abstract Map readIndexerProperties();
-
- public abstract String convertDateToString(Date d);
-
- public Date getCurrentTimestamp() {
- return new Date();
- }
-
-}
diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DIHWriter.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DIHWriter.java
deleted file mode 100644
index bdb988d4836..00000000000
--- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DIHWriter.java
+++ /dev/null
@@ -1,99 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.solr.common.SolrInputDocument;
-
-/**
- * @lucene.experimental
- *
- */
-public interface DIHWriter {
-
- /**
- *
- * If this writer supports transactions or commit points, then commit any changes,
- * optionally optimizing the data for read/write performance
- *
- * Specify the keys to be modified by a delta update (required by writers that can store duplicate keys)
- *
- */
- public void setDeltaKeys(Set> deltaKeys) ;
-
-}
diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DIHWriterBase.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DIHWriterBase.java
deleted file mode 100644
index 43e92c37f79..00000000000
--- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DIHWriterBase.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler.dataimport;
-
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-
-public abstract class DIHWriterBase implements DIHWriter {
- protected String keyFieldName;
- protected Set