Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

LUCENE-4975: Add Replication module to Lucene 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1481804 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Shai Erera 2013-05-13 11:57:22 +00:00
parent c8178bd351
commit 6b75799b42
61 changed files with 5777 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

75
dev-tools/maven/lucene/replicator/pom.xml.template Normal file
View File

@ -0,0 +1,75 @@
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<!--
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-parent</artifactId>
<version>@version@</version>
<relativePath>../pom.xml</relativePath>
</parent>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-replicator</artifactId>
<packaging>jar</packaging>
<name>Lucene Replicator</name>
<description>Lucene Replicator Module</description>
<properties>
<module-directory>lucene/replicator</module-directory>
<relative-top-level>../../..</relative-top-level>
<module-path>${relative-top-level}/${module-directory}</module-path>
</properties>
<scm>
<connection>scm:svn:${vc-anonymous-base-url}/${module-directory}</connection>
<developerConnection>scm:svn:${vc-dev-base-url}/${module-directory}</developerConnection>
<url>${vc-browse-base-url}/${module-directory}</url>
</scm>
<dependencies>
<dependency>
<!-- lucene-test-framework dependency must be declared before lucene-core -->
<groupId>${project.groupId}</groupId>
<artifactId>lucene-test-framework</artifactId>
<version>${project.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>lucene-core</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>lucene-facet</artifactId>
<version>${project.version}</version>
</dependency>
</dependencies>
<build>
<sourceDirectory>${module-path}/src/java</sourceDirectory>
<testSourceDirectory>${module-path}/src/test</testSourceDirectory>
<testResources>
<testResource>
<directory>${project.build.testSourceDirectory}</directory>
<excludes>
<exclude>**/*.java</exclude>
</excludes>
</testResource>
</testResources>
</build>
</project>

3
lucene/CHANGES.txt
View File

@ -181,6 +181,9 @@ New Features
* LUCENE-4979: LiveFieldFields can work with any ReferenceManager, not
just ReferenceManager<IndexSearcher> (Mike McCandless).
* LUCENE-4975: Added a new Replicator module which can replicate index
revisions between server and client. (Shai Erera, Mike McCandless)
Build
* LUCENE-4987: Upgrade randomized testing to version 2.0.10:

8
lucene/build.xml
View File

@ -160,7 +160,13 @@
</target>
<target name="check-licenses" depends="compile-tools,resolve,load-custom-tasks" description="Validate license stuff.">
<license-check-macro dir="${basedir}" licensedir="${common.dir}/licenses" />
<license-check-macro dir="${basedir}" licensedir="${common.dir}/licenses">
<additional-filters>
<replaceregex pattern="jetty([^/]+)$" replace="jetty" flags="gi" />
<replaceregex pattern="slf4j-([^/]+)$" replace="slf4j" flags="gi" />
<replaceregex pattern="(bcmail|bcprov)-([^/]+)$" replace="\1" flags="gi" />
</additional-filters>
</license-check-macro>
</target>
<target name="check-forbidden-apis" depends="compile-tools,compile-test,install-forbidden-apis,-forbidden-apis-classpath,-check-forbidden-jdk-apis,-check-forbidden-test-apis,-check-system-out" description="Check forbidden API calls in compiled class files"/>

2
lucene/core/src/java/org/apache/lucene/index/DirectoryReader.java
View File

@ -327,7 +327,7 @@ public abstract class DirectoryReader extends BaseCompositeReader<AtomicReader>
// corrupt first commit, but it's too deadly to make
// this logic "smarter" and risk accidentally returning
// false due to various cases like file description
// exhaustion, access denited, etc., because in that
// exhaustion, access denied, etc., because in that
// case IndexWriter may delete the entire index. It's
// safer to err towards "index exists" than try to be
// smart about detecting not-yet-fully-committed or

9
lucene/core/src/java/org/apache/lucene/index/IndexFileNames.java
View File

@ -199,7 +199,10 @@ public final class IndexFileNames {
return filename;
}
// All files created by codecs much match this pattern (we
// check this in SegmentInfo.java):
static final Pattern CODEC_FILE_PATTERN = Pattern.compile("_[a-z0-9]+(_.*)?\\..*");
/**
* All files created by codecs much match this pattern (checked in
* SegmentInfo).
*/
public static final Pattern CODEC_FILE_PATTERN = Pattern.compile("_[a-z0-9]+(_.*)?\\..*");
}

55
lucene/core/src/java/org/apache/lucene/index/SegmentInfos.java
View File

@ -242,6 +242,39 @@ public final class SegmentInfos implements Cloneable, Iterable<SegmentInfoPerCom
}
}
/**
* A utility for writing the {@link IndexFileNames#SEGMENTS_GEN} file to a
* {@link Directory}.
*
* <p>
* <b>NOTE:</b> this is an internal utility which is kept public so that it's
* accessible by code from other packages. You should avoid calling this
* method unless you're absolutely sure what you're doing!
*
* @lucene.internal
*/
public static void writeSegmentsGen(Directory dir, long generation) {
try {
IndexOutput genOutput = dir.createOutput(IndexFileNames.SEGMENTS_GEN, IOContext.READONCE);
try {
genOutput.writeInt(FORMAT_SEGMENTS_GEN_CURRENT);
genOutput.writeLong(generation);
genOutput.writeLong(generation);
} finally {
genOutput.close();
dir.sync(Collections.singleton(IndexFileNames.SEGMENTS_GEN));
}
} catch (Throwable t) {
// It's OK if we fail to write this file since it's
// used only as one of the retry fallbacks.
try {
dir.deleteFile(IndexFileNames.SEGMENTS_GEN);
} catch (Throwable t2) {
// Ignore; this file is only used in a retry
// fallback on init.
}
}
}
/**
* Get the next segments_N filename that will be written.
@ -848,27 +881,7 @@ public final class SegmentInfos implements Cloneable, Iterable<SegmentInfoPerCom
}
lastGeneration = generation;
try {
IndexOutput genOutput = dir.createOutput(IndexFileNames.SEGMENTS_GEN, IOContext.READONCE);
try {
genOutput.writeInt(FORMAT_SEGMENTS_GEN_CURRENT);
genOutput.writeLong(generation);
genOutput.writeLong(generation);
} finally {
genOutput.close();
dir.sync(Collections.singleton(IndexFileNames.SEGMENTS_GEN));
}
} catch (Throwable t) {
// It's OK if we fail to write this file since it's
// used only as one of the retry fallbacks.
try {
dir.deleteFile(IndexFileNames.SEGMENTS_GEN);
} catch (Throwable t2) {
// Ignore; this file is only used in a retry
// fallback on init.
}
}
writeSegmentsGen(dir, generation);
}
/** Writes & syncs to the Directory dir, taking care to

1
lucene/licenses/httpclient-4.2.3.jar.sha1 Normal file
View File

@ -0,0 +1 @@
37ced84d839a02fb856255eca85f0a4be95aa634

182
lucene/licenses/httpclient-LICENSE-ASL.txt Normal file
View File

@ -0,0 +1,182 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
This project contains annotations derived from JCIP-ANNOTATIONS
Copyright (c) 2005 Brian Goetz and Tim Peierls.
See http://www.jcip.net and the Creative Commons Attribution License
(http://creativecommons.org/licenses/by/2.5)

8
lucene/licenses/httpclient-NOTICE.txt Normal file
View File

@ -0,0 +1,8 @@
Apache HttpComponents Client
Copyright 1999-2011 The Apache Software Foundation
This product includes software developed by
The Apache Software Foundation (http://www.apache.org/).
This project contains annotations derived from JCIP-ANNOTATIONS
Copyright (c) 2005 Brian Goetz and Tim Peierls. See http://www.jcip.net

1
lucene/licenses/httpcore-4.2.2.jar.sha1 Normal file
View File

@ -0,0 +1 @@
b76bee23cd3f3ee9b98bc7c2c14670e821ddbbfd

182
lucene/licenses/httpcore-LICENSE-ASL.txt Normal file
View File

@ -0,0 +1,182 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
This project contains annotations derived from JCIP-ANNOTATIONS
Copyright (c) 2005 Brian Goetz and Tim Peierls.
See http://www.jcip.net and the Creative Commons Attribution License
(http://creativecommons.org/licenses/by/2.5)

8
lucene/licenses/httpcore-NOTICE.txt Normal file
View File

@ -0,0 +1,8 @@
Apache HttpComponents Client
Copyright 1999-2011 The Apache Software Foundation
This product includes software developed by
The Apache Software Foundation (http://www.apache.org/).
This project contains annotations derived from JCIP-ANNOTATIONS
Copyright (c) 2005 Brian Goetz and Tim Peierls. See http://www.jcip.net

263
lucene/licenses/javax.servlet-api-LICENSE-CDDL.txt Normal file
View File

@ -0,0 +1,263 @@
COMMON DEVELOPMENT AND DISTRIBUTION LICENSE (CDDL) Version 1.0
1. Definitions.
1.1. Contributor. means each individual or entity that creates or contributes to the creation of Modifications.
1.2. Contributor Version. means the combination of the Original Software, prior Modifications used by a Contributor (if any), and the Modifications made by that particular Contributor.
1.3. Covered Software. means (a) the Original Software, or (b) Modifications, or (c) the combination of files containing Original Software with files containing Modifications, in each case including portions thereof.
1.4. Executable. means the Covered Software in any form other than Source Code.
1.5. Initial Developer. means the individual or entity that first makes Original Software available under this License.
1.6. Larger Work. means a work which combines Covered Software or portions thereof with code not governed by the terms of this License.
1.7. License. means this document.
1.8. Licensable. means having the right to grant, to the maximum extent possible, whether at the time of the initial grant or subsequently acquired, any and all of the rights conveyed herein.
1.9. Modifications. means the Source Code and Executable form of any of the following:
A. Any file that results from an addition to, deletion from or modification of the contents of a file containing Original Software or previous Modifications;
B. Any new file that contains any part of the Original Software or previous Modification; or
C. Any new file that is contributed or otherwise made available under the terms of this License.
1.10. Original Software. means the Source Code and Executable form of computer software code that is originally released under this License.
1.11. Patent Claims. means any patent claim(s), now owned or hereafter acquired, including without limitation, method, process, and apparatus claims, in any patent Licensable by grantor.
1.12. Source Code. means (a) the common form of computer software code in which modifications are made and (b) associated documentation included in or with such code.
1.13. You. (or .Your.) means an individual or a legal entity exercising rights under, and complying with all of the terms of, this License. For legal entities, .You. includes any entity which controls, is controlled by, or is under common control with You. For purposes of this definition, .control. means (a) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (b) ownership of more than fifty percent (50%) of the outstanding shares or beneficial ownership of such entity.
2. License Grants.
2.1. The Initial Developer Grant.
Conditioned upon Your compliance with Section 3.1 below and subject to third party intellectual property claims, the Initial Developer hereby grants You a world-wide, royalty-free, non-exclusive license:
(a) under intellectual property rights (other than patent or trademark) Licensable by Initial Developer, to use, reproduce, modify, display, perform, sublicense and distribute the Original Software (or portions thereof), with or without Modifications, and/or as part of a Larger Work; and
(b) under Patent Claims infringed by the making, using or selling of Original Software, to make, have made, use, practice, sell, and offer for sale, and/or otherwise dispose of the Original Software (or portions thereof).
(c) The licenses granted in Sections 2.1(a) and (b) are effective on the date Initial Developer first distributes or otherwise makes the Original Software available to a third party under the terms of this License.
(d) Notwithstanding Section 2.1(b) above, no patent license is granted: (1) for code that You delete from the Original Software, or (2) for infringements caused by: (i) the modification of the Original Software, or (ii) the combination of the Original Software with other software or devices.
2.2. Contributor Grant.
Conditioned upon Your compliance with Section 3.1 below and subject to third party intellectual property claims, each Contributor hereby grants You a world-wide, royalty-free, non-exclusive license:
(a) under intellectual property rights (other than patent or trademark) Licensable by Contributor to use, reproduce, modify, display, perform, sublicense and distribute the Modifications created by such Contributor (or portions thereof), either on an unmodified basis, with other Modifications, as Covered Software and/or as part of a Larger Work; and
(b) under Patent Claims infringed by the making, using, or selling of Modifications made by that Contributor either alone and/or in combination with its Contributor Version (or portions of such combination), to make, use, sell, offer for sale, have made, and/or otherwise dispose of: (1) Modifications made by that Contributor (or portions thereof); and (2) the combination of Modifications made by that Contributor with its Contributor Version (or portions of such combination).
(c) The licenses granted in Sections 2.2(a) and 2.2(b) are effective on the date Contributor first distributes or otherwise makes the Modifications available to a third party.
(d) Notwithstanding Section 2.2(b) above, no patent license is granted: (1) for any code that Contributor has deleted from the Contributor Version; (2) for infringements caused by: (i) third party modifications of Contributor Version, or (ii) the combination of Modifications made by that Contributor with other software (except as part of the Contributor Version) or other devices; or (3) under Patent Claims infringed by Covered Software in the absence of Modifications made by that Contributor.
3. Distribution Obligations.
3.1. Availability of Source Code.
Any Covered Software that You distribute or otherwise make available in Executable form must also be made available in Source Code form and that Source Code form must be distributed only under the terms of this License. You must include a copy of this License with every copy of the Source Code form of the Covered Software You distribute or otherwise make available. You must inform recipients of any such Covered Software in Executable form as to how they can obtain such Covered Software in Source Code form in a reasonable manner on or through a medium customarily used for software exchange.
3.2. Modifications.
The Modifications that You create or to which You contribute are governed by the terms of this License. You represent that You believe Your Modifications are Your original creation(s) and/or You have sufficient rights to grant the rights conveyed by this License.
3.3. Required Notices.
You must include a notice in each of Your Modifications that identifies You as the Contributor of the Modification. You may not remove or alter any copyright, patent or trademark notices contained within the Covered Software, or any notices of licensing or any descriptive text giving attribution to any Contributor or the Initial Developer.
3.4. Application of Additional Terms.
You may not offer or impose any terms on any Covered Software in Source Code form that alters or restricts the applicable version of this License or the recipients. rights hereunder. You may choose to offer, and to charge a fee for, warranty, support, indemnity or liability obligations to one or more recipients of Covered Software. However, you may do so only on Your own behalf, and not on behalf of the Initial Developer or any Contributor. You must make it absolutely clear that any such warranty, support, indemnity or liability obligation is offered by You alone, and You hereby agree to indemnify the Initial Developer and every Contributor for any liability incurred by the Initial Developer or such Contributor as a result of warranty, support, indemnity or liability terms You offer.
3.5. Distribution of Executable Versions.
You may distribute the Executable form of the Covered Software under the terms of this License or under the terms of a license of Your choice, which may contain terms different from this License, provided that You are in compliance with the terms of this License and that the license for the Executable form does not attempt to limit or alter the recipient.s rights in the Source Code form from the rights set forth in this License. If You distribute the Covered Software in Executable form under a different license, You must make it absolutely clear that any terms which differ from this License are offered by You alone, not by the Initial Developer or Contributor. You hereby agree to indemnify the Initial Developer and every Contributor for any liability incurred by the Initial Developer or such Contributor as a result of any such terms You offer.
3.6. Larger Works.
You may create a Larger Work by combining Covered Software with other code not governed by the terms of this License and distribute the Larger Work as a single product. In such a case, You must make sure the requirements of this License are fulfilled for the Covered Software.
4. Versions of the License.
4.1. New Versions.
Sun Microsystems, Inc. is the initial license steward and may publish revised and/or new versions of this License from time to time. Each version will be given a distinguishing version number. Except as provided in Section 4.3, no one other than the license steward has the right to modify this License.
4.2. Effect of New Versions.
You may always continue to use, distribute or otherwise make the Covered Software available under the terms of the version of the License under which You originally received the Covered Software. If the Initial Developer includes a notice in the Original Software prohibiting it from being distributed or otherwise made available under any subsequent version of the License, You must distribute and make the Covered Software available under the terms of the version of the License under which You originally received the Covered Software. Otherwise, You may also choose to use, distribute or otherwise make the Covered Software available under the terms of any subsequent version of the License published by the license steward.
4.3. Modified Versions.
When You are an Initial Developer and You want to create a new license for Your Original Software, You may create and use a modified version of this License if You: (a) rename the license and remove any references to the name of the license steward (except to note that the license differs from this License); and (b) otherwise make it clear that the license contains terms which differ from this License.
5. DISCLAIMER OF WARRANTY.
COVERED SOFTWARE IS PROVIDED UNDER THIS LICENSE ON AN .AS IS. BASIS, WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, WITHOUT LIMITATION, WARRANTIES THAT THE COVERED SOFTWARE IS FREE OF DEFECTS, MERCHANTABLE, FIT FOR A PARTICULAR PURPOSE OR NON-INFRINGING. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE COVERED SOFTWARE IS WITH YOU. SHOULD ANY COVERED SOFTWARE PROVE DEFECTIVE IN ANY RESPECT, YOU (NOT THE INITIAL DEVELOPER OR ANY OTHER CONTRIBUTOR) ASSUME THE COST OF ANY NECESSARY SERVICING, REPAIR OR CORRECTION. THIS DISCLAIMER OF WARRANTY CONSTITUTES AN ESSENTIAL PART OF THIS LICENSE. NO USE OF ANY COVERED SOFTWARE IS AUTHORIZED HEREUNDER EXCEPT UNDER THIS DISCLAIMER.
6. TERMINATION.
6.1. This License and the rights granted hereunder will terminate automatically if You fail to comply with terms herein and fail to cure such breach within 30 days of becoming aware of the breach. Provisions which, by their nature, must remain in effect beyond the termination of this License shall survive.
6.2. If You assert a patent infringement claim (excluding declaratory judgment actions) against Initial Developer or a Contributor (the Initial Developer or Contributor against whom You assert such claim is referred to as .Participant.) alleging that the Participant Software (meaning the Contributor Version where the Participant is a Contributor or the Original Software where the Participant is the Initial Developer) directly or indirectly infringes any patent, then any and all rights granted directly or indirectly to You by such Participant, the Initial Developer (if the Initial Developer is not the Participant) and all Contributors under Sections 2.1 and/or 2.2 of this License shall, upon 60 days notice from Participant terminate prospectively and automatically at the expiration of such 60 day notice period, unless if within such 60 day period You withdraw Your claim with respect to the Participant Software against such Participant either unilaterally or pursuant to a written agreement with Participant.
6.3. In the event of termination under Sections 6.1 or 6.2 above, all end user licenses that have been validly granted by You or any distributor hereunder prior to termination (excluding licenses granted to You by any distributor) shall survive termination.
7. LIMITATION OF LIABILITY.
UNDER NO CIRCUMSTANCES AND UNDER NO LEGAL THEORY, WHETHER TORT (INCLUDING NEGLIGENCE), CONTRACT, OR OTHERWISE, SHALL YOU, THE INITIAL DEVELOPER, ANY OTHER CONTRIBUTOR, OR ANY DISTRIBUTOR OF COVERED SOFTWARE, OR ANY SUPPLIER OF ANY OF SUCH PARTIES, BE LIABLE TO ANY PERSON FOR ANY INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES OF ANY CHARACTER INCLUDING, WITHOUT LIMITATION, DAMAGES FOR LOST PROFITS, LOSS OF GOODWILL, WORK STOPPAGE, COMPUTER FAILURE OR MALFUNCTION, OR ANY AND ALL OTHER COMMERCIAL DAMAGES OR LOSSES, EVEN IF SUCH PARTY SHALL HAVE BEEN INFORMED OF THE POSSIBILITY OF SUCH DAMAGES. THIS LIMITATION OF LIABILITY SHALL NOT APPLY TO LIABILITY FOR DEATH OR PERSONAL INJURY RESULTING FROM SUCH PARTY.S NEGLIGENCE TO THE EXTENT APPLICABLE LAW PROHIBITS SUCH LIMITATION. SOME JURISDICTIONS DO NOT ALLOW THE EXCLUSION OR LIMITATION OF INCIDENTAL OR CONSEQUENTIAL DAMAGES, SO THIS EXCLUSION AND LIMITATION MAY NOT APPLY TO YOU.
8. U.S. GOVERNMENT END USERS.
The Covered Software is a .commercial item,. as that term is defined in 48 C.F.R. 2.101 (Oct. 1995), consisting of .commercial computer software. (as that term is defined at 48 C.F.R. ? 252.227-7014(a)(1)) and .commercial computer software documentation. as such terms are used in 48 C.F.R. 12.212 (Sept. 1995). Consistent with 48 C.F.R. 12.212 and 48 C.F.R. 227.7202-1 through 227.7202-4 (June 1995), all U.S. Government End Users acquire Covered Software with only those rights set forth herein. This U.S. Government Rights clause is in lieu of, and supersedes, any other FAR, DFAR, or other clause or provision that addresses Government rights in computer software under this License.
9. MISCELLANEOUS.
This License represents the complete agreement concerning subject matter hereof. If any provision of this License is held to be unenforceable, such provision shall be reformed only to the extent necessary to make it enforceable. This License shall be governed by the law of the jurisdiction specified in a notice contained within the Original Software (except to the extent applicable law, if any, provides otherwise), excluding such jurisdiction.s conflict-of-law provisions. Any litigation relating to this License shall be subject to the jurisdiction of the courts located in the jurisdiction and venue specified in a notice contained within the Original Software, with the losing party responsible for costs, including, without limitation, court costs and reasonable attorneys. fees and expenses. The application of the United Nations Convention on Contracts for the International Sale of Goods is expressly excluded. Any law or regulation which provides that the language of a contract shall be construed against the drafter shall not apply to this License. You agree that You alone are responsible for compliance with the United States export administration regulations (and the export control laws and regulation of any other countries) when You use, distribute or otherwise make available any Covered Software.
10. RESPONSIBILITY FOR CLAIMS.
As between Initial Developer and the Contributors, each party is responsible for claims and damages arising, directly or indirectly, out of its utilization of rights under this License and You agree to work with Initial Developer and Contributors to distribute such responsibility on an equitable basis. Nothing herein is intended or shall be deemed to constitute any admission of liability.
NOTICE PURSUANT TO SECTION 9 OF THE COMMON DEVELOPMENT AND DISTRIBUTION LICENSE (CDDL)
The code released under the CDDL shall be governed by the laws of the State of California (excluding conflict-of-law provisions). Any litigation relating to this License shall be subject to the jurisdiction of the Federal Courts of the Northern District of California and the state courts of the State of California, with venue lying in Santa Clara County, California.
The GNU General Public License (GPL) Version 2, June 1991
Copyright (C) 1989, 1991 Free Software Foundation, Inc. 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed.
Preamble
The licenses for most software are designed to take away your freedom to share and change it. By contrast, the GNU General Public License is intended to guarantee your freedom to share and change free software--to make sure the software is free for all its users. This General Public License applies to most of the Free Software Foundation's software and to any other program whose authors commit to using it. (Some other Free Software Foundation software is covered by the GNU Library General Public License instead.) You can apply it to your programs, too.
When we speak of free software, we are referring to freedom, not price. Our General Public Licenses are designed to make sure that you have the freedom to distribute copies of free software (and charge for this service if you wish), that you receive source code or can get it if you want it, that you can change the software or use pieces of it in new free programs; and that you know you can do these things.
To protect your rights, we need to make restrictions that forbid anyone to deny you these rights or to ask you to surrender the rights. These restrictions translate to certain responsibilities for you if you distribute copies of the software, or if you modify it.
For example, if you distribute copies of such a program, whether gratis or for a fee, you must give the recipients all the rights that you have. You must make sure that they, too, receive or can get the source code. And you must show them these terms so they know their rights.
We protect your rights with two steps: (1) copyright the software, and (2) offer you this license which gives you legal permission to copy, distribute and/or modify the software.
Also, for each author's protection and ours, we want to make certain that everyone understands that there is no warranty for this free software. If the software is modified by someone else and passed on, we want its recipients to know that what they have is not the original, so that any problems introduced by others will not reflect on the original authors' reputations.
Finally, any free program is threatened constantly by software patents. We wish to avoid the danger that redistributors of a free program will individually obtain patent licenses, in effect making the program proprietary. To prevent this, we have made it clear that any patent must be licensed for everyone's free use or not licensed at all.
The precise terms and conditions for copying, distribution and modification follow.
TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
0. This License applies to any program or other work which contains a notice placed by the copyright holder saying it may be distributed under the terms of this General Public License. The "Program", below, refers to any such program or work, and a "work based on the Program" means either the Program or any derivative work under copyright law: that is to say, a work containing the Program or a portion of it, either verbatim or with modifications and/or translated into another language. (Hereinafter, translation is included without limitation in the term "modification".) Each licensee is addressed as "you".
Activities other than copying, distribution and modification are not covered by this License; they are outside its scope. The act of running the Program is not restricted, and the output from the Program is covered only if its contents constitute a work based on the Program (independent of having been made by running the Program). Whether that is true depends on what the Program does.
1. You may copy and distribute verbatim copies of the Program's source code as you receive it, in any medium, provided that you conspicuously and appropriately publish on each copy an appropriate copyright notice and disclaimer of warranty; keep intact all the notices that refer to this License and to the absence of any warranty; and give any other recipients of the Program a copy of this License along with the Program.
You may charge a fee for the physical act of transferring a copy, and you may at your option offer warranty protection in exchange for a fee.
2. You may modify your copy or copies of the Program or any portion of it, thus forming a work based on the Program, and copy and distribute such modifications or work under the terms of Section 1 above, provided that you also meet all of these conditions:
a) You must cause the modified files to carry prominent notices stating that you changed the files and the date of any change.
b) You must cause any work that you distribute or publish, that in whole or in part contains or is derived from the Program or any part thereof, to be licensed as a whole at no charge to all third parties under the terms of this License.
c) If the modified program normally reads commands interactively when run, you must cause it, when started running for such interactive use in the most ordinary way, to print or display an announcement including an appropriate copyright notice and a notice that there is no warranty (or else, saying that you provide a warranty) and that users may redistribute the program under these conditions, and telling the user how to view a copy of this License. (Exception: if the Program itself is interactive but does not normally print such an announcement, your work based on the Program is not required to print an announcement.)
These requirements apply to the modified work as a whole. If identifiable sections of that work are not derived from the Program, and can be reasonably considered independent and separate works in themselves, then this License, and its terms, do not apply to those sections when you distribute them as separate works. But when you distribute the same sections as part of a whole which is a work based on the Program, the distribution of the whole must be on the terms of this License, whose permissions for other licensees extend to the entire whole, and thus to each and every part regardless of who wrote it.
Thus, it is not the intent of this section to claim rights or contest your rights to work written entirely by you; rather, the intent is to exercise the right to control the distribution of derivative or collective works based on the Program.
In addition, mere aggregation of another work not based on the Program with the Program (or with a work based on the Program) on a volume of a storage or distribution medium does not bring the other work under the scope of this License.
3. You may copy and distribute the Program (or a work based on it, under Section 2) in object code or executable form under the terms of Sections 1 and 2 above provided that you also do one of the following:
a) Accompany it with the complete corresponding machine-readable source code, which must be distributed under the terms of Sections 1 and 2 above on a medium customarily used for software interchange; or,
b) Accompany it with a written offer, valid for at least three years, to give any third party, for a charge no more than your cost of physically performing source distribution, a complete machine-readable copy of the corresponding source code, to be distributed under the terms of Sections 1 and 2 above on a medium customarily used for software interchange; or,
c) Accompany it with the information you received as to the offer to distribute corresponding source code. (This alternative is allowed only for noncommercial distribution and only if you received the program in object code or executable form with such an offer, in accord with Subsection b above.)
The source code for a work means the preferred form of the work for making modifications to it. For an executable work, complete source code means all the source code for all modules it contains, plus any associated interface definition files, plus the scripts used to control compilation and installation of the executable. However, as a special exception, the source code distributed need not include anything that is normally distributed (in either source or binary form) with the major components (compiler, kernel, and so on) of the operating system on which the executable runs, unless that component itself accompanies the executable.
If distribution of executable or object code is made by offering access to copy from a designated place, then offering equivalent access to copy the source code from the same place counts as distribution of the source code, even though third parties are not compelled to copy the source along with the object code.
4. You may not copy, modify, sublicense, or distribute the Program except as expressly provided under this License. Any attempt otherwise to copy, modify, sublicense or distribute the Program is void, and will automatically terminate your rights under this License. However, parties who have received copies, or rights, from you under this License will not have their licenses terminated so long as such parties remain in full compliance.
5. You are not required to accept this License, since you have not signed it. However, nothing else grants you permission to modify or distribute the Program or its derivative works. These actions are prohibited by law if you do not accept this License. Therefore, by modifying or distributing the Program (or any work based on the Program), you indicate your acceptance of this License to do so, and all its terms and conditions for copying, distributing or modifying the Program or works based on it.
6. Each time you redistribute the Program (or any work based on the Program), the recipient automatically receives a license from the original licensor to copy, distribute or modify the Program subject to these terms and conditions. You may not impose any further restrictions on the recipients' exercise of the rights granted herein. You are not responsible for enforcing compliance by third parties to this License.
7. If, as a consequence of a court judgment or allegation of patent infringement or for any other reason (not limited to patent issues), conditions are imposed on you (whether by court order, agreement or otherwise) that contradict the conditions of this License, they do not excuse you from the conditions of this License. If you cannot distribute so as to satisfy simultaneously your obligations under this License and any other pertinent obligations, then as a consequence you may not distribute the Program at all. For example, if a patent license would not permit royalty-free redistribution of the Program by all those who receive copies directly or indirectly through you, then the only way you could satisfy both it and this License would be to refrain entirely from distribution of the Program.
If any portion of this section is held invalid or unenforceable under any particular circumstance, the balance of the section is intended to apply and the section as a whole is intended to apply in other circumstances.
It is not the purpose of this section to induce you to infringe any patents or other property right claims or to contest validity of any such claims; this section has the sole purpose of protecting the integrity of the free software distribution system, which is implemented by public license practices. Many people have made generous contributions to the wide range of software distributed through that system in reliance on consistent application of that system; it is up to the author/donor to decide if he or she is willing to distribute software through any other system and a licensee cannot impose that choice.
This section is intended to make thoroughly clear what is believed to be a consequence of the rest of this License.
8. If the distribution and/or use of the Program is restricted in certain countries either by patents or by copyrighted interfaces, the original copyright holder who places the Program under this License may add an explicit geographical distribution limitation excluding those countries, so that distribution is permitted only in or among countries not thus excluded. In such case, this License incorporates the limitation as if written in the body of this License.
9. The Free Software Foundation may publish revised and/or new versions of the General Public License from time to time. Such new versions will be similar in spirit to the present version, but may differ in detail to address new problems or concerns.
Each version is given a distinguishing version number. If the Program specifies a version number of this License which applies to it and "any later version", you have the option of following the terms and conditions either of that version or of any later version published by the Free Software Foundation. If the Program does not specify a version number of this License, you may choose any version ever published by the Free Software Foundation.
10. If you wish to incorporate parts of the Program into other free programs whose distribution conditions are different, write to the author to ask for permission. For software which is copyrighted by the Free Software Foundation, write to the Free Software Foundation; we sometimes make exceptions for this. Our decision will be guided by the two goals of preserving the free status of all derivatives of our free software and of promoting the sharing and reuse of software generally.
NO WARRANTY
11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES.
END OF TERMS AND CONDITIONS
How to Apply These Terms to Your New Programs
If you develop a new program, and you want it to be of the greatest possible use to the public, the best way to achieve this is to make it free software which everyone can redistribute and change under these terms.
To do so, attach the following notices to the program. It is safest to attach them to the start of each source file to most effectively convey the exclusion of warranty; and each file should have at least the "copyright" line and a pointer to where the full notice is found.
One line to give the program's name and a brief idea of what it does.
Copyright (C)
This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version.
This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
Also add information on how to contact you by electronic and paper mail.
If the program is interactive, make it output a short notice like this when it starts in an interactive mode:
Gnomovision version 69, Copyright (C) year name of author
Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. This is free software, and you are welcome to redistribute it under certain conditions; type `show c' for details.
The hypothetical commands `show w' and `show c' should show the appropriate parts of the General Public License. Of course, the commands you use may be called something other than `show w' and `show c'; they could even be mouse-clicks or menu items--whatever suits your program.
You should also get your employer (if you work as a programmer) or your school, if any, to sign a "copyright disclaimer" for the program, if necessary. Here is a sample; alter the names:
Yoyodyne, Inc., hereby disclaims all copyright interest in the program `Gnomovision' (which makes passes at compilers) written by James Hacker.
signature of Ty Coon, 1 April 1989
Ty Coon, President of Vice
This General Public License does not permit incorporating your program into proprietary programs. If your program is a subroutine library, you may consider it more useful to permit linking proprietary applications with the library. If this is what you want to do, use the GNU Library General Public License instead of this License.
"CLASSPATH" EXCEPTION TO THE GPL VERSION 2
Certain source files distributed by Sun Microsystems, Inc. are subject to the following clarification and special exception to the GPL Version 2, but only where Sun has expressly included in the particular source file's header the words
"Sun designates this particular file as subject to the "Classpath" exception as provided by Sun in the License file that accompanied this code."
Linking this library statically or dynamically with other modules is making a combined work based on this library. Thus, the terms and conditions of the GNU General Public License Version 2 cover the whole combination.
As a special exception, the copyright holders of this library give you permission to link this library with independent modules to produce an executable, regardless of the license terms of these independent modules, and to copy and distribute the resulting executable under terms of your choice, provided that you also meet, for each linked independent module, the terms and conditions of the license of that module.? An independent module is a module which is not derived from or based on this library.? If you modify this library, you may extend this exception to your version of the library, but you are not obligated to do so.? If you do not wish to do so, delete this exception statement from your version.

2
lucene/licenses/javax.servlet-api-NOTICE.txt Normal file
View File

@ -0,0 +1,2 @@
Servlet-api.jar is under the CDDL license, the original source
code for this can be found at http://www.eclipse.org/jetty/downloads.php

1
lucene/licenses/jcl-over-slf4j-1.6.6.jar.sha1 Normal file
View File

@ -0,0 +1 @@
ec497945fdcaf7fd970ae9931b9bbfaf735d385e

21
lucene/licenses/jcl-over-slf4j-LICENSE-BSD_LIKE.txt Normal file
View File

@ -0,0 +1,21 @@
Copyright (c) 2004-2008 QOS.ch
All rights reserved.
Permission is hereby granted, free of charge, to any person obtaining
a copy of this software and associated documentation files (the
"Software"), to deal in the Software without restriction, including
without limitation the rights to use, copy, modify, merge, publish,
distribute, sublicense, and/or sell copies of the Software, and to
permit persons to whom the Software is furnished to do so, subject to
the following conditions:
The above copyright notice and this permission notice shall be
included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

25
lucene/licenses/jcl-over-slf4j-NOTICE.txt Normal file
View File

@ -0,0 +1,25 @@
=========================================================================
== SLF4J Notice -- http://www.slf4j.org/license.html ==
=========================================================================
Copyright (c) 2004-2008 QOS.ch
All rights reserved.
Permission is hereby granted, free of charge, to any person obtaining
a copy of this software and associated documentation files (the
"Software"), to deal in the Software without restriction, including
without limitation the rights to use, copy, modify, merge, publish,
distribute, sublicense, and/or sell copies of the Software, and to
permit persons to whom the Software is furnished to do so, subject to
the following conditions:
The above copyright notice and this permission notice shall be
included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

202
lucene/licenses/jetty-LICENSE-ASL.txt Normal file
View File

@ -0,0 +1,202 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright [yyyy] [name of copyright owner]
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

111
lucene/licenses/jetty-NOTICE.txt Normal file
View File

@ -0,0 +1,111 @@
<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0//EN">
<html><head>
<meta http-equiv="Content-Type" content="text/html; charset=ISO-8859-1">
<title>Eclipse.org Software User Agreement</title>
</head><body lang="EN-US" link="blue" vlink="purple">
<h2>Eclipse Foundation Software User Agreement</h2>
<p>March 17, 2005</p>
<h3>Usage Of Content</h3>
<p>THE ECLIPSE FOUNDATION MAKES AVAILABLE SOFTWARE, DOCUMENTATION, INFORMATION AND/OR OTHER MATERIALS FOR OPEN SOURCE PROJECTS
(COLLECTIVELY "CONTENT"). USE OF THE CONTENT IS GOVERNED BY THE TERMS AND CONDITIONS OF THIS AGREEMENT AND/OR THE TERMS AND
CONDITIONS OF LICENSE AGREEMENTS OR NOTICES INDICATED OR REFERENCED BELOW. BY USING THE CONTENT, YOU AGREE THAT YOUR USE
OF THE CONTENT IS GOVERNED BY THIS AGREEMENT AND/OR THE TERMS AND CONDITIONS OF ANY APPLICABLE LICENSE AGREEMENTS OR
NOTICES INDICATED OR REFERENCED BELOW. IF YOU DO NOT AGREE TO THE TERMS AND CONDITIONS OF THIS AGREEMENT AND THE TERMS AND
CONDITIONS OF ANY APPLICABLE LICENSE AGREEMENTS OR NOTICES INDICATED OR REFERENCED BELOW, THEN YOU MAY NOT USE THE CONTENT.</p>
<h3>Applicable Licenses</h3>
<p>Unless otherwise indicated, all Content made available by the
Eclipse Foundation is provided to you under the terms and conditions of
the Eclipse Public License Version 1.0 ("EPL"). A copy of the EPL is
provided with this Content and is also available at <a href="http://www.eclipse.org/legal/epl-v10.html">http://www.eclipse.org/legal/epl-v10.html</a>.
For purposes of the EPL, "Program" will mean the Content.</p>
<p>Content includes, but is not limited to, source code, object code,
documentation and other files maintained in the Eclipse.org CVS
repository ("Repository") in CVS modules ("Modules") and made available
as downloadable archives ("Downloads").</p>
<ul>
<li>Content may be structured and packaged into modules to
facilitate delivering, extending, and upgrading the Content. Typical
modules may include plug-ins ("Plug-ins"), plug-in fragments
("Fragments"), and features ("Features").</li>
<li>Each Plug-in or Fragment may be packaged as a sub-directory or JAR (Java&#8482; ARchive) in a directory named "plugins".</li>
<li>A
Feature is a bundle of one or more Plug-ins and/or Fragments and
associated material. Each Feature may be packaged as a sub-directory in
a directory named "features". Within a Feature, files named
"feature.xml" may contain a list of the names and version numbers of
the Plug-ins and/or Fragments associated with that Feature.</li>
<li>Features
may also include other Features ("Included Features"). Within a
Feature, files named "feature.xml" may contain a list of the names and
version numbers of Included Features.</li>
</ul>
<p>The terms and conditions governing Plug-ins and Fragments should be
contained in files named "about.html" ("Abouts"). The terms and
conditions governing Features and
Included Features should be contained in files named "license.html"
("Feature Licenses"). Abouts and Feature Licenses may be located in any
directory of a Download or Module
including, but not limited to the following locations:</p>
<ul>
<li>The top-level (root) directory</li>
<li>Plug-in and Fragment directories</li>
<li>Inside Plug-ins and Fragments packaged as JARs</li>
<li>Sub-directories of the directory named "src" of certain Plug-ins</li>
<li>Feature directories</li>
</ul>
<p>Note: if a Feature made available by the Eclipse Foundation is
installed using the Eclipse Update Manager, you must agree to a license
("Feature Update License") during the
installation process. If the Feature contains Included Features, the
Feature Update License should either provide you with the terms and
conditions governing the Included Features or
inform you where you can locate them. Feature Update Licenses may be
found in the "license" property of files named "feature.properties"
found within a Feature.
Such Abouts, Feature Licenses, and Feature Update Licenses contain the
terms and conditions (or references to such terms and conditions) that
govern your use of the associated Content in
that directory.</p>
<p>THE ABOUTS, FEATURE LICENSES, AND FEATURE UPDATE LICENSES MAY REFER
TO THE EPL OR OTHER LICENSE AGREEMENTS, NOTICES OR TERMS AND
CONDITIONS. SOME OF THESE
OTHER LICENSE AGREEMENTS MAY INCLUDE (BUT ARE NOT LIMITED TO):</p>
<ul>
<li>Common Public License Version 1.0 (available at <a href="http://www.eclipse.org/legal/cpl-v10.html">http://www.eclipse.org/legal/cpl-v10.html</a>)</li>
<li>Apache Software License 1.1 (available at <a href="http://www.apache.org/licenses/LICENSE">http://www.apache.org/licenses/LICENSE</a>)</li>
<li>Apache Software License 2.0 (available at <a href="http://www.apache.org/licenses/LICENSE-2.0">http://www.apache.org/licenses/LICENSE-2.0</a>)</li>
<li>IBM Public License 1.0 (available at <a href="http://oss.software.ibm.com/developerworks/opensource/license10.html">http://oss.software.ibm.com/developerworks/opensource/license10.html</a>)</li>
<li>Metro Link Public License 1.00 (available at <a href="http://www.opengroup.org/openmotif/supporters/metrolink/license.html">http://www.opengroup.org/openmotif/supporters/metrolink/license.html</a>)</li>
<li>Mozilla Public License Version 1.1 (available at <a href="http://www.mozilla.org/MPL/MPL-1.1.html">http://www.mozilla.org/MPL/MPL-1.1.html</a>)</li>
</ul>
<p>IT IS YOUR OBLIGATION TO READ AND ACCEPT ALL SUCH TERMS AND
CONDITIONS PRIOR TO USE OF THE CONTENT. If no About, Feature License,
or Feature Update License is provided, please
contact the Eclipse Foundation to determine what terms and conditions
govern that particular Content.</p>
<h3>Cryptography</h3>
<p>Content may contain encryption software. The country in which you
are currently may have restrictions on the import, possession, and use,
and/or re-export to another country, of encryption software. BEFORE
using any encryption software, please check the country's laws,
regulations and policies concerning the import, possession, or use, and
re-export of encryption software, to see if this is permitted.</p>
<small>Java and all Java-based trademarks are trademarks of Sun Microsystems, Inc. in the United States, other countries, or both.</small>
</body></html>

1
lucene/licenses/jetty-continuation-8.1.10.v20130312.jar.sha1 Normal file
View File

@ -0,0 +1 @@
c0e26574ddcac7a86486f19a8b3782657acfd961

1
lucene/licenses/jetty-http-8.1.10.v20130312.jar.sha1 Normal file
View File

@ -0,0 +1 @@
d9eb53007e04d6338f12f3ded60fad1f7bfcb40e

1
lucene/licenses/jetty-io-8.1.10.v20130312.jar.sha1 Normal file
View File

@ -0,0 +1 @@
e829c768f2b9de5d9fae3bc0aba3996bd0344f56

1
lucene/licenses/jetty-server-8.1.10.v20130312.jar.sha1 Normal file
View File

@ -0,0 +1 @@
13ca9587bc1645f8fac89454b15252a2ad5bdcf5

1
lucene/licenses/jetty-servlet-8.1.10.v20130312.jar.sha1 Normal file
View File

@ -0,0 +1 @@
98f8029fe7236e9c66381c04f292b5319f47ca84

1
lucene/licenses/jetty-util-8.1.10.v20130312.jar.sha1 Normal file
View File

@ -0,0 +1 @@
d198a8ad8ea20b4fb74c781175c48500ec2b8b7a

1
lucene/licenses/servlet-api-3.0.jar.sha1 Normal file
View File

@ -0,0 +1 @@
0aaaa85845fb5c59da00193f06b8e5278d8bf3f8

21
lucene/licenses/slf4j-LICENSE-BSD_LIKE.txt Normal file
View File

@ -0,0 +1,21 @@
Copyright (c) 2004-2008 QOS.ch
All rights reserved.
Permission is hereby granted, free of charge, to any person obtaining
a copy of this software and associated documentation files (the
"Software"), to deal in the Software without restriction, including
without limitation the rights to use, copy, modify, merge, publish,
distribute, sublicense, and/or sell copies of the Software, and to
permit persons to whom the Software is furnished to do so, subject to
the following conditions:
The above copyright notice and this permission notice shall be
included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

25
lucene/licenses/slf4j-NOTICE.txt Normal file
View File

@ -0,0 +1,25 @@
=========================================================================
== SLF4J Notice -- http://www.slf4j.org/license.html ==
=========================================================================
Copyright (c) 2004-2008 QOS.ch
All rights reserved.
Permission is hereby granted, free of charge, to any person obtaining
a copy of this software and associated documentation files (the
"Software"), to deal in the Software without restriction, including
without limitation the rights to use, copy, modify, merge, publish,
distribute, sublicense, and/or sell copies of the Software, and to
permit persons to whom the Software is furnished to do so, subject to
the following conditions:
The above copyright notice and this permission notice shall be
included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

1
lucene/licenses/slf4j-api-1.6.6.jar.sha1 Normal file
View File

@ -0,0 +1 @@
ce53b0a0e2cfbb27e8a59d38f79a18a5c6a8d2b0

24
lucene/module-build.xml
View File

@ -220,7 +220,29 @@
</ant>
<property name="facet-javadocs.uptodate" value="true"/>
</target>
<property name="replicator.jar" value="${common.dir}/build/replicator/lucene-replicator-${version}.jar"/>
<target name="check-replicator-uptodate" unless="replicator.uptodate">
<module-uptodate name="replicator" jarfile="${replicator.jar}" property="replicator.uptodate"/>
</target>
<target name="jar-replicator" unless="replicator.uptodate" depends="check-replicator-uptodate">
<ant dir="${common.dir}/replicator" target="jar-core" inheritall="false">
<propertyset refid="uptodate.and.compiled.properties"/>
</ant>
<property name="replicator.uptodate" value="true"/>
</target>
<property name="replicator-javadoc.jar" value="${common.dir}/build/replicator/lucene-replicator-${version}-javadoc.jar"/>
<target name="check-replicator-javadocs-uptodate" unless="replicator-javadocs.uptodate">
<module-uptodate name="replicator" jarfile="${replicator-javadoc.jar}" property="replicator-javadocs.uptodate"/>
</target>
<target name="javadocs-replicator" unless="replicator-javadocs.uptodate" depends="check-replicator-javadocs-uptodate">
<ant dir="${common.dir}/replicator" target="javadocs" inheritAll="false">
<propertyset refid="uptodate.and.compiled.properties"/>
</ant>
<property name="replicator-javadocs.uptodate" value="true"/>
</target>
<property name="analyzers-icu.jar" value="${common.dir}/build/analysis/icu/lucene-analyzers-icu-${version}.jar"/>
<target name="check-analyzers-icu-uptodate" unless="analyzers-icu.uptodate">
<module-uptodate name="analysis/icu" jarfile="${analyzers-icu.jar}" property="analyzers-icu.uptodate"/>

49
lucene/replicator/build.xml Normal file
View File

@ -0,0 +1,49 @@
<?xml version="1.0"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<project name="replicator" default="default" xmlns:ivy="antlib:org.apache.ivy.ant">
<description>
Files replication utility
</description>
<import file="../module-build.xml"/>
<path id="classpath">
<fileset dir="lib" />
<pathelement path="${facet.jar}"/>
<path refid="base.classpath"/>
</path>
<target name="resolve" depends="common.resolve">
<sequential>
<!-- servlet-api.jar -->
<ivy:retrieve conf="servlet" log="download-only" type="orbit" pattern="lib/servlet-api-3.0.jar"/>
</sequential>
</target>
<target name="init" depends="module-build.init,jar-facet"/>
<target name="javadocs" depends="javadocs-facet,compile-core">
<invoke-module-javadoc>
<links>
<link href="../facet"/>
</links>
</invoke-module-javadoc>
</target>
</project>

50
lucene/replicator/ivy.xml Normal file
View File

@ -0,0 +1,50 @@
<!--
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
<!DOCTYPE ivy-module [
<!ENTITY jetty.version "8.1.10.v20130312">
]>
<ivy-module version="2.0">
<info organisation="org.apache.lucene" module="replicator"/>
<configurations>
<conf name="http" description="httpclient jars"/>
<conf name="jetty" description="jetty jars"/>
<conf name="start" description="jetty start jar"/>
<conf name="servlet" description="servlet-api jar"/>
<conf name="logging" description="logging setup"/>
</configurations>
<dependencies>
<dependency org="org.apache.httpcomponents" name="httpclient" rev="4.2.3" transitive="false" conf="http->default"/>
<dependency org="org.apache.httpcomponents" name="httpcore" rev="4.2.2" transitive="false" conf="http->default"/>
<dependency org="org.eclipse.jetty" name="jetty-server" rev="&jetty.version;" transitive="false" conf="jetty->default"/>
<dependency org="org.eclipse.jetty" name="jetty-servlet" rev="&jetty.version;" transitive="false" conf="jetty->default"/>
<dependency org="org.eclipse.jetty" name="jetty-util" rev="&jetty.version;" transitive="false" conf="jetty->default"/>
<dependency org="org.eclipse.jetty" name="jetty-io" rev="&jetty.version;" transitive="false" conf="jetty->default"/>
<dependency org="org.eclipse.jetty" name="jetty-continuation" rev="&jetty.version;" transitive="false" conf="jetty->default"/>
<dependency org="org.eclipse.jetty" name="jetty-http" rev="&jetty.version;" transitive="false" conf="jetty->default"/>
<dependency org="org.slf4j" name="slf4j-api" rev="1.6.6" transitive="false" conf="logging->default"/>
<dependency org="org.slf4j" name="jcl-over-slf4j" rev="1.6.6" transitive="false" conf="logging->default"/>
<dependency org="org.eclipse.jetty.orbit" name="javax.servlet" rev="3.0.0.v201112011016" transitive="false" conf="servlet->default">
<artifact name="javax.servlet" type="orbit" ext="jar"/>
</dependency>
<exclude org="*" ext="*" matcher="regexp" type="${ivy.exclude.types}"/>
</dependencies>
</ivy-module>

191
lucene/replicator/src/java/org/apache/lucene/replicator/IndexAndTaxonomyReplicationHandler.java Executable file
View File

@ -0,0 +1,191 @@
package org.apache.lucene.replicator;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.IOException;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.concurrent.Callable;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexCommit;
import org.apache.lucene.replicator.ReplicationClient.ReplicationHandler;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.util.InfoStream;
/**
* A {@link ReplicationHandler} for replication of an index and taxonomy pair.
* See {@link IndexReplicationHandler} for more detail. This handler ensures
* that the search and taxonomy indexes are replicated in a consistent way.
* <p>
* <b>NOTE:</b> if you intend to recreate a taxonomy index, you should make sure
* to reopen an IndexSearcher and TaxonomyReader pair via the provided callback,
* to guarantee that both indexes are in sync. This handler does not prevent
* replicating such index and taxonomy pairs, and if they are reopened by a
* different thread, unexpected errors can occur, as well as inconsistency
* between the taxonomy and index readers.
*
* @see IndexReplicationHandler
*
* @lucene.experimental
*/
public class IndexAndTaxonomyReplicationHandler implements ReplicationHandler {
/**
* The component used to log messages to the {@link InfoStream#getDefault()
* default} {@link InfoStream}.
*/
public static final String INFO_STREAM_COMPONENT = "IndexAndTaxonomyReplicationHandler";
private final Directory indexDir;
private final Directory taxoDir;
private final Callable<Boolean> callback;
private volatile Map<String,List<RevisionFile>> currentRevisionFiles;
private volatile String currentVersion;
private volatile InfoStream infoStream = InfoStream.getDefault();
/**
* Constructor with the given index directory and callback to notify when the
* indexes were updated.
*/
public IndexAndTaxonomyReplicationHandler(Directory indexDir, Directory taxoDir, Callable<Boolean> callback)
throws IOException {
this.callback = callback;
this.indexDir = indexDir;
this.taxoDir = taxoDir;
currentRevisionFiles = null;
currentVersion = null;
final boolean indexExists = DirectoryReader.indexExists(indexDir);
final boolean taxoExists = DirectoryReader.indexExists(taxoDir);
if (indexExists != taxoExists) {
throw new IllegalStateException("search and taxonomy indexes must either both exist or not: index=" + indexExists
+ " taxo=" + taxoExists);
}
if (indexExists) { // both indexes exist
final IndexCommit indexCommit = IndexReplicationHandler.getLastCommit(indexDir);
final IndexCommit taxoCommit = IndexReplicationHandler.getLastCommit(taxoDir);
currentRevisionFiles = IndexAndTaxonomyRevision.revisionFiles(indexCommit, taxoCommit);
currentVersion = IndexAndTaxonomyRevision.revisionVersion(indexCommit, taxoCommit);
final InfoStream infoStream = InfoStream.getDefault();
if (infoStream.isEnabled(INFO_STREAM_COMPONENT)) {
infoStream.message(INFO_STREAM_COMPONENT, "constructor(): currentVersion=" + currentVersion
+ " currentRevisionFiles=" + currentRevisionFiles);
infoStream.message(INFO_STREAM_COMPONENT, "constructor(): indexCommit=" + indexCommit
+ " taxoCommit=" + taxoCommit);
}
}
}
@Override
public String currentVersion() {
return currentVersion;
}
@Override
public Map<String,List<RevisionFile>> currentRevisionFiles() {
return currentRevisionFiles;
}
@Override
public void revisionReady(String version, Map<String,List<RevisionFile>> revisionFiles,
Map<String,List<String>> copiedFiles, Map<String,Directory> sourceDirectory) throws IOException {
Directory taxoClientDir = sourceDirectory.get(IndexAndTaxonomyRevision.TAXONOMY_SOURCE);
Directory indexClientDir = sourceDirectory.get(IndexAndTaxonomyRevision.INDEX_SOURCE);
List<String> taxoFiles = copiedFiles.get(IndexAndTaxonomyRevision.TAXONOMY_SOURCE);
List<String> indexFiles = copiedFiles.get(IndexAndTaxonomyRevision.INDEX_SOURCE);
String taxoSegmentsFile = IndexReplicationHandler.getSegmentsFile(taxoFiles, true);
String indexSegmentsFile = IndexReplicationHandler.getSegmentsFile(indexFiles, false);
boolean success = false;
try {
// copy taxonomy files before index files
IndexReplicationHandler.copyFiles(taxoClientDir, taxoDir, taxoFiles);
IndexReplicationHandler.copyFiles(indexClientDir, indexDir, indexFiles);
// fsync all copied files (except segmentsFile)
if (!taxoFiles.isEmpty()) {
taxoDir.sync(taxoFiles);
}
indexDir.sync(indexFiles);
// now copy and fsync segmentsFile, taxonomy first because it is ok if a
// reader sees a more advanced taxonomy than the index.
if (taxoSegmentsFile != null) {
taxoClientDir.copy(taxoDir, taxoSegmentsFile, taxoSegmentsFile, IOContext.READONCE);
}
indexClientDir.copy(indexDir, indexSegmentsFile, indexSegmentsFile, IOContext.READONCE);
if (taxoSegmentsFile != null) {
taxoDir.sync(Collections.singletonList(taxoSegmentsFile));
}
indexDir.sync(Collections.singletonList(indexSegmentsFile));
success = true;
} finally {
if (!success) {
taxoFiles.add(taxoSegmentsFile); // add it back so it gets deleted too
IndexReplicationHandler.cleanupFilesOnFailure(taxoDir, taxoFiles);
indexFiles.add(indexSegmentsFile); // add it back so it gets deleted too
IndexReplicationHandler.cleanupFilesOnFailure(indexDir, indexFiles);
}
}
// all files have been successfully copied + sync'd. update the handler's state
currentRevisionFiles = revisionFiles;
currentVersion = version;
if (infoStream.isEnabled(INFO_STREAM_COMPONENT)) {
infoStream.message(INFO_STREAM_COMPONENT, "revisionReady(): currentVersion=" + currentVersion
+ " currentRevisionFiles=" + currentRevisionFiles);
}
// update the segments.gen file
IndexReplicationHandler.writeSegmentsGen(taxoSegmentsFile, taxoDir);
IndexReplicationHandler.writeSegmentsGen(indexSegmentsFile, indexDir);
// Cleanup the index directory from old and unused index files.
// NOTE: we don't use IndexWriter.deleteUnusedFiles here since it may have
// side-effects, e.g. if it hits sudden IO errors while opening the index
// (and can end up deleting the entire index). It is not our job to protect
// against those errors, app will probably hit them elsewhere.
IndexReplicationHandler.cleanupOldIndexFiles(indexDir, indexSegmentsFile);
IndexReplicationHandler.cleanupOldIndexFiles(taxoDir, taxoSegmentsFile);
// successfully updated the index, notify the callback that the index is
// ready.
if (callback != null) {
try {
callback.call();
} catch (Exception e) {
throw new IOException(e);
}
}
}
/** Sets the {@link InfoStream} to use for logging messages. */
public void setInfoStream(InfoStream infoStream) {
if (infoStream == null) {
infoStream = InfoStream.NO_OUTPUT;
}
this.infoStream = infoStream;
}
}

219
lucene/replicator/src/java/org/apache/lucene/replicator/IndexAndTaxonomyRevision.java Executable file
View File

@ -0,0 +1,219 @@
package org.apache.lucene.replicator;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.IOException;
import java.io.InputStream;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter;
import org.apache.lucene.facet.taxonomy.writercache.TaxonomyWriterCache;
import org.apache.lucene.index.IndexCommit;
import org.apache.lucene.index.IndexDeletionPolicy;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.index.SnapshotDeletionPolicy;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
/**
* A {@link Revision} of a single index and taxonomy index files which comprises
* the list of files from both indexes. This revision should be used whenever a
* pair of search and taxonomy indexes need to be replicated together to
* guarantee consistency of both on the replicating (client) side.
*
* @see IndexRevision
*
* @lucene.experimental
*/
public class IndexAndTaxonomyRevision implements Revision {
/**
* A {@link DirectoryTaxonomyWriter} which sets the underlying
* {@link IndexWriter}'s {@link IndexDeletionPolicy} to
* {@link SnapshotDeletionPolicy}.
*/
public static final class SnapshotDirectoryTaxonomyWriter extends DirectoryTaxonomyWriter {
private SnapshotDeletionPolicy sdp;
private IndexWriter writer;
/**
* @see DirectoryTaxonomyWriter#DirectoryTaxonomyWriter(Directory,
* IndexWriterConfig.OpenMode, TaxonomyWriterCache)
*/
public SnapshotDirectoryTaxonomyWriter(Directory directory, OpenMode openMode, TaxonomyWriterCache cache)
throws IOException {
super(directory, openMode, cache);
}
/** @see DirectoryTaxonomyWriter#DirectoryTaxonomyWriter(Directory, IndexWriterConfig.OpenMode) */
public SnapshotDirectoryTaxonomyWriter(Directory directory, OpenMode openMode) throws IOException {
super(directory, openMode);
}
/** @see DirectoryTaxonomyWriter#DirectoryTaxonomyWriter(Directory) */
public SnapshotDirectoryTaxonomyWriter(Directory d) throws IOException {
super(d);
}
@Override
protected IndexWriterConfig createIndexWriterConfig(OpenMode openMode) {
IndexWriterConfig conf = super.createIndexWriterConfig(openMode);
conf.setIndexDeletionPolicy(new SnapshotDeletionPolicy(conf.getIndexDeletionPolicy()));
return conf;
}
@Override
protected IndexWriter openIndexWriter(Directory directory, IndexWriterConfig config) throws IOException {
writer = super.openIndexWriter(directory, config);
// must set it here because IndexWriter clones the config
sdp = (SnapshotDeletionPolicy) writer.getConfig().getIndexDeletionPolicy();
return writer;
}
/** Returns the {@link SnapshotDeletionPolicy} used by the underlying {@link IndexWriter}. */
public SnapshotDeletionPolicy getDeletionPolicy() {
return sdp;
}
/** Returns the {@link IndexWriter} used by this {@link DirectoryTaxonomyWriter}. */
public IndexWriter getIndexWriter() {
return writer;
}
}
private static final int RADIX = 16;
public static final String INDEX_SOURCE = "index";
public static final String TAXONOMY_SOURCE = "taxo";
private final IndexWriter indexWriter;
private final SnapshotDirectoryTaxonomyWriter taxoWriter;
private final IndexCommit indexCommit, taxoCommit;
private final SnapshotDeletionPolicy indexSDP, taxoSDP;
private final String version;
private final Map<String,List<RevisionFile>> sourceFiles;
/** Returns a singleton map of the revision files from the given {@link IndexCommit}. */
public static Map<String, List<RevisionFile>> revisionFiles(IndexCommit indexCommit, IndexCommit taxoCommit)
throws IOException {
HashMap<String,List<RevisionFile>> files = new HashMap<String,List<RevisionFile>>();
files.put(INDEX_SOURCE, IndexRevision.revisionFiles(indexCommit).values().iterator().next());
files.put(TAXONOMY_SOURCE, IndexRevision.revisionFiles(taxoCommit).values().iterator().next());
return files;
}
/**
* Returns a String representation of a revision's version from the given
* {@link IndexCommit}s of the search and taxonomy indexes.
*/
public static String revisionVersion(IndexCommit indexCommit, IndexCommit taxoCommit) {
return Long.toString(indexCommit.getGeneration(), RADIX) + ":" + Long.toString(taxoCommit.getGeneration(), RADIX);
}
/**
* Constructor over the given {@link IndexWriter}. Uses the last
* {@link IndexCommit} found in the {@link Directory} managed by the given
* writer.
*/
public IndexAndTaxonomyRevision(IndexWriter indexWriter, SnapshotDirectoryTaxonomyWriter taxoWriter)
throws IOException {
IndexDeletionPolicy delPolicy = indexWriter.getConfig().getIndexDeletionPolicy();
if (!(delPolicy instanceof SnapshotDeletionPolicy)) {
throw new IllegalArgumentException("IndexWriter must be created with SnapshotDeletionPolicy");
}
this.indexWriter = indexWriter;
this.taxoWriter = taxoWriter;
this.indexSDP = (SnapshotDeletionPolicy) delPolicy;
this.taxoSDP = taxoWriter.getDeletionPolicy();
this.indexCommit = indexSDP.snapshot();
this.taxoCommit = taxoSDP.snapshot();
this.version = revisionVersion(indexCommit, taxoCommit);
this.sourceFiles = revisionFiles(indexCommit, taxoCommit);
}
@Override
public int compareTo(String version) {
final String[] parts = version.split(":");
final long indexGen = Long.parseLong(parts[0], RADIX);
final long taxoGen = Long.parseLong(parts[1], RADIX);
final long indexCommitGen = indexCommit.getGeneration();
final long taxoCommitGen = taxoCommit.getGeneration();
// if the index generation is not the same as this commit's generation,
// compare by it. Otherwise, compare by the taxonomy generation.
if (indexCommitGen < indexGen) {
return -1;
} else if (indexCommitGen > indexGen) {
return 1;
} else {
return taxoCommitGen < taxoGen ? -1 : (taxoCommitGen > taxoGen ? 1 : 0);
}
}
@Override
public int compareTo(Revision o) {
IndexAndTaxonomyRevision other = (IndexAndTaxonomyRevision) o;
int cmp = indexCommit.compareTo(other.indexCommit);
return cmp != 0 ? cmp : taxoCommit.compareTo(other.taxoCommit);
}
@Override
public String getVersion() {
return version;
}
@Override
public Map<String,List<RevisionFile>> getSourceFiles() {
return sourceFiles;
}
@Override
public InputStream open(String source, String fileName) throws IOException {
assert source.equals(INDEX_SOURCE) || source.equals(TAXONOMY_SOURCE) : "invalid source; expected=(" + INDEX_SOURCE
+ " or " + TAXONOMY_SOURCE + ") got=" + source;
IndexCommit ic = source.equals(INDEX_SOURCE) ? indexCommit : taxoCommit;
return new IndexInputInputStream(ic.getDirectory().openInput(fileName, IOContext.READONCE));
}
@Override
public void release() throws IOException {
try {
indexSDP.release(indexCommit);
} finally {
taxoSDP.release(taxoCommit);
}
try {
indexWriter.deleteUnusedFiles();
} finally {
taxoWriter.getIndexWriter().deleteUnusedFiles();
}
}
@Override
public String toString() {
return "IndexAndTaxonomyRevision version=" + version + " files=" + sourceFiles;
}
}

92
lucene/replicator/src/java/org/apache/lucene/replicator/IndexInputInputStream.java Executable file
View File

@ -0,0 +1,92 @@
package org.apache.lucene.replicator;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.IOException;
import java.io.InputStream;
import org.apache.lucene.store.IndexInput;
/**
* An {@link InputStream} which wraps an {@link IndexInput}.
*
* @lucene.experimental
*/
public final class IndexInputInputStream extends InputStream {
private final IndexInput in;
private long remaining;
public IndexInputInputStream(IndexInput in) {
this.in = in;
remaining = in.length();
}
@Override
public int read() throws IOException {
if (remaining == 0) {
return -1;
} else {
--remaining;
return in.readByte();
}
}
@Override
public int available() throws IOException {
return (int) in.length();
}
@Override
public void close() throws IOException {
in.close();
}
@Override
public int read(byte[] b) throws IOException {
return read(b, 0, b.length);
}
@Override
public int read(byte[] b, int off, int len) throws IOException {
if (remaining == 0) {
return -1;
}
if (remaining < len) {
len = (int) remaining;
}
in.readBytes(b, off, len);
remaining -= len;
return len;
}
@Override
public long skip(long n) throws IOException {
if (remaining == 0) {
return -1;
}
if (remaining < n) {
n = remaining;
}
in.seek(in.getFilePointer() + n);
remaining -= n;
return n;
}
}

308
lucene/replicator/src/java/org/apache/lucene/replicator/IndexReplicationHandler.java Executable file
View File

@ -0,0 +1,308 @@
package org.apache.lucene.replicator;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.IOException;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.Callable;
import java.util.regex.Matcher;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexCommit;
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.IndexNotFoundException;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.SegmentInfos;
import org.apache.lucene.replicator.ReplicationClient.ReplicationHandler;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.util.InfoStream;
/**
* A {@link ReplicationHandler} for replication of an index. Implements
* {@link #revisionReady} by copying the files pointed by the client resolver to
* the index {@link Directory} and then touches the index with
* {@link IndexWriter} to make sure any unused files are deleted.
* <p>
* <b>NOTE:</b> this handler assumes that {@link IndexWriter} is not opened by
* another process on the index directory. In fact, opening an
* {@link IndexWriter} on the same directory to which files are copied can lead
* to undefined behavior, where some or all the files will be deleted, override
* other files or simply create a mess. When you replicate an index, it is best
* if the index is never modified by {@link IndexWriter}, except the one that is
* open on the source index, from which you replicate.
* <p>
* This handler notifies the application via a provided {@link Callable} when an
* updated index commit was made available for it.
*
* @lucene.experimental
*/
public class IndexReplicationHandler implements ReplicationHandler {
/**
* The component used to log messages to the {@link InfoStream#getDefault()
* default} {@link InfoStream}.
*/
public static final String INFO_STREAM_COMPONENT = "IndexReplicationHandler";
private final Directory indexDir;
private final Callable<Boolean> callback;
private volatile Map<String,List<RevisionFile>> currentRevisionFiles;
private volatile String currentVersion;
private volatile InfoStream infoStream = InfoStream.getDefault();
/**
* Returns the last {@link IndexCommit} found in the {@link Directory}, or
* {@code null} if there are no commits.
*/
public static IndexCommit getLastCommit(Directory dir) throws IOException {
try {
if (DirectoryReader.indexExists(dir)) {
List<IndexCommit> commits = DirectoryReader.listCommits(dir);
// listCommits guarantees that we get at least one commit back, or
// IndexNotFoundException which we handle below
return commits.get(commits.size() - 1);
}
} catch (IndexNotFoundException e) {
// ignore the exception and return null
}
return null;
}
/**
* Verifies that the last file is segments_N and fails otherwise. It also
* removes and returns the file from the list, because it needs to be handled
* last, after all files. This is important in order to guarantee that if a
* reader sees the new segments_N, all other segment files are already on
* stable storage.
* <p>
* The reason why the code fails instead of putting segments_N file last is
* that this indicates an error in the Revision implementation.
*/
public static String getSegmentsFile(List<String> files, boolean allowEmpty) {
if (files.isEmpty()) {
if (allowEmpty) {
return null;
} else {
throw new IllegalStateException("empty list of files not allowed");
}
}
String segmentsFile = files.remove(files.size() - 1);
if (!segmentsFile.startsWith(IndexFileNames.SEGMENTS) || segmentsFile.equals(IndexFileNames.SEGMENTS_GEN)) {
throw new IllegalStateException("last file to copy+sync must be segments_N but got " + segmentsFile
+ "; check your Revision implementation!");
}
return segmentsFile;
}
/**
* Cleanup the index directory by deleting all given files. Called when file
* copy or sync failed.
*/
public static void cleanupFilesOnFailure(Directory dir, List<String> files) {
for (String file : files) {
try {
if (dir.fileExists(file)) {
dir.deleteFile(file);
}
} catch (Throwable t) {
// suppress any exception because if we're here, it means copy
// failed, and we must cleanup after ourselves.
}
}
}
/**
* Cleans up the index directory from old index files. This method uses the
* last commit found by {@link #getLastCommit(Directory)}. If it matches the
* expected segmentsFile, then all files not referenced by this commit point
* are deleted.
* <p>
* <b>NOTE:</b> this method does a best effort attempt to clean the index
* directory. It suppresses any exceptions that occur, as this can be retried
* the next time.
*/
public static void cleanupOldIndexFiles(Directory dir, String segmentsFile) {
try {
IndexCommit commit = getLastCommit(dir);
// commit == null means weird IO errors occurred, ignore them
// if there were any IO errors reading the expected commit point (i.e.
// segments files mismatch), then ignore that commit either.
if (commit != null && commit.getSegmentsFileName().equals(segmentsFile)) {
Set<String> commitFiles = new HashSet<String>();
commitFiles.addAll(commit.getFileNames());
commitFiles.add(IndexFileNames.SEGMENTS_GEN);
Matcher matcher = IndexFileNames.CODEC_FILE_PATTERN.matcher("");
for (String file : dir.listAll()) {
if (!commitFiles.contains(file)
&& (matcher.reset(file).matches() || file.startsWith(IndexFileNames.SEGMENTS))) {
try {
dir.deleteFile(file);
} catch (Throwable t) {
// suppress, it's just a best effort
}
}
}
}
} catch (Throwable t) {
// ignore any errors that happens during this state and only log it. this
// cleanup will have a chance to succeed the next time we get a new
// revision.
}
}
/**
* Copies the files from the source directory to the target one, if they are
* not the same.
*/
public static void copyFiles(Directory source, Directory target, List<String> files) throws IOException {
if (!source.equals(target)) {
for (String file : files) {
source.copy(target, file, file, IOContext.READONCE);
}
}
}
/**
* Writes {@link IndexFileNames#SEGMENTS_GEN} file to the directory, reading
* the generation from the given {@code segmentsFile}. If it is {@code null},
* this method deletes segments.gen from the directory.
*/
public static void writeSegmentsGen(String segmentsFile, Directory dir) {
if (segmentsFile != null) {
SegmentInfos.writeSegmentsGen(dir, SegmentInfos.generationFromSegmentsFileName(segmentsFile));
} else {
try {
if (dir.fileExists(IndexFileNames.SEGMENTS_GEN)) {
dir.deleteFile(IndexFileNames.SEGMENTS_GEN);
}
} catch (Throwable t) {
// suppress any errors while deleting this file.
}
}
}
/**
* Constructor with the given index directory and callback to notify when the
* indexes were updated.
*/
public IndexReplicationHandler(Directory indexDir, Callable<Boolean> callback) throws IOException {
this.callback = callback;
this.indexDir = indexDir;
currentRevisionFiles = null;
currentVersion = null;
if (DirectoryReader.indexExists(indexDir)) {
final List<IndexCommit> commits = DirectoryReader.listCommits(indexDir);
final IndexCommit commit = commits.get(commits.size() - 1);
currentRevisionFiles = IndexRevision.revisionFiles(commit);
currentVersion = IndexRevision.revisionVersion(commit);
final InfoStream infoStream = InfoStream.getDefault();
if (infoStream.isEnabled(INFO_STREAM_COMPONENT)) {
infoStream.message(INFO_STREAM_COMPONENT, "constructor(): currentVersion=" + currentVersion
+ " currentRevisionFiles=" + currentRevisionFiles);
infoStream.message(INFO_STREAM_COMPONENT, "constructor(): commit=" + commit);
}
}
}
@Override
public String currentVersion() {
return currentVersion;
}
@Override
public Map<String,List<RevisionFile>> currentRevisionFiles() {
return currentRevisionFiles;
}
@Override
public void revisionReady(String version, Map<String,List<RevisionFile>> revisionFiles,
Map<String,List<String>> copiedFiles, Map<String,Directory> sourceDirectory) throws IOException {
if (revisionFiles.size() > 1) {
throw new IllegalArgumentException("this handler handles only a single source; got " + revisionFiles.keySet());
}
Directory clientDir = sourceDirectory.values().iterator().next();
List<String> files = copiedFiles.values().iterator().next();
String segmentsFile = getSegmentsFile(files, false);
boolean success = false;
try {
// copy files from the client to index directory
copyFiles(clientDir, indexDir, files);
// fsync all copied files (except segmentsFile)
indexDir.sync(files);
// now copy and fsync segmentsFile
clientDir.copy(indexDir, segmentsFile, segmentsFile, IOContext.READONCE);
indexDir.sync(Collections.singletonList(segmentsFile));
success = true;
} finally {
if (!success) {
files.add(segmentsFile); // add it back so it gets deleted too
cleanupFilesOnFailure(indexDir, files);
}
}
// all files have been successfully copied + sync'd. update the handler's state
currentRevisionFiles = revisionFiles;
currentVersion = version;
if (infoStream.isEnabled(INFO_STREAM_COMPONENT)) {
infoStream.message(INFO_STREAM_COMPONENT, "revisionReady(): currentVersion=" + currentVersion
+ " currentRevisionFiles=" + currentRevisionFiles);
}
// update the segments.gen file
writeSegmentsGen(segmentsFile, indexDir);
// Cleanup the index directory from old and unused index files.
// NOTE: we don't use IndexWriter.deleteUnusedFiles here since it may have
// side-effects, e.g. if it hits sudden IO errors while opening the index
// (and can end up deleting the entire index). It is not our job to protect
// against those errors, app will probably hit them elsewhere.
cleanupOldIndexFiles(indexDir, segmentsFile);
// successfully updated the index, notify the callback that the index is
// ready.
if (callback != null) {
try {
callback.call();
} catch (Exception e) {
throw new IOException(e);
}
}
}
/** Sets the {@link InfoStream} to use for logging messages. */
public void setInfoStream(InfoStream infoStream) {
if (infoStream == null) {
infoStream = InfoStream.NO_OUTPUT;
}
this.infoStream = infoStream;
}
}

150
lucene/replicator/src/java/org/apache/lucene/replicator/IndexRevision.java Executable file
View File

@ -0,0 +1,150 @@
package org.apache.lucene.replicator;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import org.apache.lucene.index.IndexCommit;
import org.apache.lucene.index.IndexDeletionPolicy;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.SnapshotDeletionPolicy;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
/**
* A {@link Revision} of a single index files which comprises the list of files
* that are part of the current {@link IndexCommit}. To ensure the files are not
* deleted by {@link IndexWriter} for as long as this revision stays alive (i.e.
* until {@link #release()}), the current commit point is snapshotted, using
* {@link SnapshotDeletionPolicy} (this means that the given writer's
* {@link IndexWriterConfig#getIndexDeletionPolicy() config} should return
* {@link SnapshotDeletionPolicy}).
* <p>
* When this revision is {@link #release() released}, it releases the obtained
* snapshot as well as calls {@link IndexWriter#deleteUnusedFiles()} so that the
* snapshotted files are deleted (if they are no longer needed).
*
* @lucene.experimental
*/
public class IndexRevision implements Revision {
private static final int RADIX = 16;
private static final String SOURCE = "index";
private final IndexWriter writer;
private final IndexCommit commit;
private final SnapshotDeletionPolicy sdp;
private final String version;
private final Map<String,List<RevisionFile>> sourceFiles;
// returns a RevisionFile with some metadata
private static RevisionFile newRevisionFile(String file, Directory dir) throws IOException {
RevisionFile revFile = new RevisionFile(file);
revFile.size = dir.fileLength(file);
return revFile;
}
/** Returns a singleton map of the revision files from the given {@link IndexCommit}. */
public static Map<String,List<RevisionFile>> revisionFiles(IndexCommit commit) throws IOException {
Collection<String> commitFiles = commit.getFileNames();
List<RevisionFile> revisionFiles = new ArrayList<RevisionFile>(commitFiles.size());
String segmentsFile = commit.getSegmentsFileName();
Directory dir = commit.getDirectory();
for (String file : commitFiles) {
if (!file.equals(segmentsFile)) {
revisionFiles.add(newRevisionFile(file, dir));
}
}
revisionFiles.add(newRevisionFile(segmentsFile, dir)); // segments_N must be last
return Collections.singletonMap(SOURCE, revisionFiles);
}
/**
* Returns a String representation of a revision's version from the given
* {@link IndexCommit}.
*/
public static String revisionVersion(IndexCommit commit) {
return Long.toString(commit.getGeneration(), RADIX);
}
/**
* Constructor over the given {@link IndexWriter}. Uses the last
* {@link IndexCommit} found in the {@link Directory} managed by the given
* writer.
*/
public IndexRevision(IndexWriter writer) throws IOException {
IndexDeletionPolicy delPolicy = writer.getConfig().getIndexDeletionPolicy();
if (!(delPolicy instanceof SnapshotDeletionPolicy)) {
throw new IllegalArgumentException("IndexWriter must be created with SnapshotDeletionPolicy");
}
this.writer = writer;
this.sdp = (SnapshotDeletionPolicy) delPolicy;
this.commit = sdp.snapshot();
this.version = revisionVersion(commit);
this.sourceFiles = revisionFiles(commit);
}
@Override
public int compareTo(String version) {
long gen = Long.parseLong(version, RADIX);
long commitGen = commit.getGeneration();
return commitGen < gen ? -1 : (commitGen > gen ? 1 : 0);
}
@Override
public int compareTo(Revision o) {
IndexRevision other = (IndexRevision) o;
return commit.compareTo(other.commit);
}
@Override
public String getVersion() {
return version;
}
@Override
public Map<String,List<RevisionFile>> getSourceFiles() {
return sourceFiles;
}
@Override
public InputStream open(String source, String fileName) throws IOException {
assert source.equals(SOURCE) : "invalid source; expected=" + SOURCE + " got=" + source;
return new IndexInputInputStream(commit.getDirectory().openInput(fileName, IOContext.READONCE));
}
@Override
public void release() throws IOException {
sdp.release(commit);
writer.deleteUnusedFiles();
}
@Override
public String toString() {
return "IndexRevision version=" + version + " files=" + sourceFiles;
}
}

247
lucene/replicator/src/java/org/apache/lucene/replicator/LocalReplicator.java Executable file
View File

@ -0,0 +1,247 @@
package org.apache.lucene.replicator;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.lucene.store.AlreadyClosedException;
/**
* A {@link Replicator} implementation for use by the side that publishes
* {@link Revision}s, as well for clients to {@link #checkForUpdate(String)
* check for updates}. When a client needs to be updated, it is returned a
* {@link SessionToken} through which it can
* {@link #obtainFile(String, String, String) obtain} the files of that
* revision. As long as a revision is being replicated, this replicator
* guarantees that it will not be {@link Revision#release() released}.
* <p>
* Replication sessions expire by default after
* {@link #DEFAULT_SESSION_EXPIRATION_THRESHOLD}, and the threshold can be
* configured through {@link #setExpirationThreshold(long)}.
*
* @lucene.experimental
*/
public class LocalReplicator implements Replicator {
private static class RefCountedRevision {
private final AtomicInteger refCount = new AtomicInteger(1);
public final Revision revision;
public RefCountedRevision(Revision revision) {
this.revision = revision;
}
public void decRef() throws IOException {
if (refCount.get() <= 0) {
throw new IllegalStateException("this revision is already released");
}
final int rc = refCount.decrementAndGet();
if (rc == 0) {
boolean success = false;
try {
revision.release();
success = true;
} finally {
if (!success) {
// Put reference back on failure
refCount.incrementAndGet();
}
}
} else if (rc < 0) {
throw new IllegalStateException("too many decRef calls: refCount is " + rc + " after decrement");
}
}
public void incRef() {
refCount.incrementAndGet();
}
}
private static class ReplicationSession {
public final SessionToken session;
public final RefCountedRevision revision;
private volatile long lastAccessTime;
ReplicationSession(SessionToken session, RefCountedRevision revision) {
this.session = session;
this.revision = revision;
lastAccessTime = System.currentTimeMillis();
}
boolean isExpired(long expirationThreshold) {
return lastAccessTime < (System.currentTimeMillis() - expirationThreshold);
}
void markAccessed() {
lastAccessTime = System.currentTimeMillis();
}
}
/** Threshold for expiring inactive sessions. Defaults to 30 minutes. */
public static final long DEFAULT_SESSION_EXPIRATION_THRESHOLD = 1000 * 60 * 30;
private long expirationThresholdMilllis = LocalReplicator.DEFAULT_SESSION_EXPIRATION_THRESHOLD;
private volatile RefCountedRevision currentRevision;
private volatile boolean closed = false;
private final AtomicInteger sessionToken = new AtomicInteger(0);
private final Map<String, ReplicationSession> sessions = new HashMap<String, ReplicationSession>();
private void checkExpiredSessions() throws IOException {
// make a "to-delete" list so we don't risk deleting from the map while iterating it
final ArrayList<ReplicationSession> toExpire = new ArrayList<ReplicationSession>();
for (ReplicationSession token : sessions.values()) {
if (token.isExpired(expirationThresholdMilllis)) {
toExpire.add(token);
}
}
for (ReplicationSession token : toExpire) {
releaseSession(token.session.id);
}
}
private void releaseSession(String sessionID) throws IOException {
ReplicationSession session = sessions.remove(sessionID);
// if we're called concurrently by close() and release(), could be that one
// thread beats the other to release the session.
if (session != null) {
session.revision.decRef();
}
}
/** Ensure that replicator is still open, or throw {@link AlreadyClosedException} otherwise. */
protected final synchronized void ensureOpen() {
if (closed) {
throw new AlreadyClosedException("This replicator has already been closed");
}
}
@Override
public synchronized SessionToken checkForUpdate(String currentVersion) {
ensureOpen();
if (currentRevision == null) { // no published revisions yet
return null;
}
if (currentVersion != null && currentRevision.revision.compareTo(currentVersion) <= 0) {
// currentVersion is newer or equal to latest published revision
return null;
}
// currentVersion is either null or older than latest published revision
currentRevision.incRef();
final String sessionID = Integer.toString(sessionToken.incrementAndGet());
final SessionToken sessionToken = new SessionToken(sessionID, currentRevision.revision);
final ReplicationSession timedSessionToken = new ReplicationSession(sessionToken, currentRevision);
sessions.put(sessionID, timedSessionToken);
return sessionToken;
}
@Override
public synchronized void close() throws IOException {
if (!closed) {
// release all managed revisions
for (ReplicationSession session : sessions.values()) {
session.revision.decRef();
}
sessions.clear();
closed = true;
}
}
/**
* Returns the expiration threshold.
*
* @see #setExpirationThreshold(long)
*/
public long getExpirationThreshold() {
return expirationThresholdMilllis;
}
@Override
public synchronized InputStream obtainFile(String sessionID, String source, String fileName) throws IOException {
ensureOpen();
ReplicationSession session = sessions.get(sessionID);
if (session != null && session.isExpired(expirationThresholdMilllis)) {
releaseSession(sessionID);
session = null;
}
// session either previously expired, or we just expired it
if (session == null) {
throw new SessionExpiredException("session (" + sessionID + ") expired while obtaining file: source=" + source
+ " file=" + fileName);
}
sessions.get(sessionID).markAccessed();
return session.revision.revision.open(source, fileName);
}
@Override
public synchronized void publish(Revision revision) throws IOException {
ensureOpen();
if (currentRevision != null) {
int compare = revision.compareTo(currentRevision.revision);
if (compare == 0) {
// same revision published again, ignore but release it
revision.release();
return;
}
if (compare < 0) {
revision.release();
throw new IllegalArgumentException("Cannot publish an older revision: rev=" + revision + " current="
+ currentRevision);
}
}
// swap revisions
final RefCountedRevision oldRevision = currentRevision;
currentRevision = new RefCountedRevision(revision);
if (oldRevision != null) {
oldRevision.decRef();
}
// check for expired sessions
checkExpiredSessions();
}
@Override
public synchronized void release(String sessionID) throws IOException {
ensureOpen();
releaseSession(sessionID);
}
/**
* Modify session expiration time - if a replication session is inactive that
* long it is automatically expired, and further attempts to operate within
* this session will throw a {@link SessionExpiredException}.
*/
public synchronized void setExpirationThreshold(long expirationThreshold) throws IOException {
ensureOpen();
this.expirationThresholdMilllis = expirationThreshold;
checkExpiredSessions();
}
}

77
lucene/replicator/src/java/org/apache/lucene/replicator/PerSessionDirectoryFactory.java Executable file
View File

@ -0,0 +1,77 @@
package org.apache.lucene.replicator;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.File;
import java.io.IOException;
import org.apache.lucene.replicator.ReplicationClient.SourceDirectoryFactory;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
/**
* A {@link SourceDirectoryFactory} which returns {@link FSDirectory} under a
* dedicated session directory. When a session is over, the entire directory is
* deleted.
*
* @lucene.experimental
*/
public class PerSessionDirectoryFactory implements SourceDirectoryFactory {
private final File workDir;
/** Constructor with the given sources mapping. */
public PerSessionDirectoryFactory(File workDir) {
this.workDir = workDir;
}
private void rm(File file) throws IOException {
if (file.isDirectory()) {
for (File f : file.listFiles()) {
rm(f);
}
}
// This should be either an empty directory, or a file
if (!file.delete() && file.exists()) {
throw new IOException("failed to delete " + file);
}
}
@Override
public Directory getDirectory(String sessionID, String source) throws IOException {
File sessionDir = new File(workDir, sessionID);
if (!sessionDir.exists() && !sessionDir.mkdirs()) {
throw new IOException("failed to create session directory " + sessionDir);
}
File sourceDir = new File(sessionDir, source);
if (!sourceDir.mkdirs()) {
throw new IOException("failed to create source directory " + sourceDir);
}
return FSDirectory.open(sourceDir);
}
@Override
public void cleanupSession(String sessionID) throws IOException {
if (sessionID.isEmpty()) { // protect against deleting workDir entirely!
throw new IllegalArgumentException("sessionID cannot be empty");
}
rm(new File(workDir, sessionID));
}
}

416
lucene/replicator/src/java/org/apache/lucene/replicator/ReplicationClient.java Executable file
View File

@ -0,0 +1,416 @@
package org.apache.lucene.replicator;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.Closeable;
import java.io.IOException;
import java.io.InputStream;
import java.io.PrintWriter;
import java.io.StringWriter;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
import org.apache.lucene.store.AlreadyClosedException;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.InfoStream;
import org.apache.lucene.util.ThreadInterruptedException;
/**
* A client which monitors and obtains new revisions from a {@link Replicator}.
* It can be used to either periodically check for updates by invoking
* {@link #startUpdateThread}, or manually by calling {@link #updateNow()}.
* <p>
* Whenever a new revision is available, the {@link #requiredFiles(Map)} are
* copied to the {@link Directory} specified by {@link PerSessionDirectoryFactory} and
* a handler is notified.
*
* @lucene.experimental
*/
public class ReplicationClient implements Closeable {
private class ReplicationThread extends Thread {
private final long interval;
// client uses this to stop us
final CountDownLatch stop = new CountDownLatch(1);
public ReplicationThread(long interval) {
this.interval = interval;
}
@SuppressWarnings("synthetic-access")
@Override
public void run() {
while (true) {
long time = System.currentTimeMillis();
updateLock.lock();
try {
doUpdate();
} catch (Throwable t) {
handleUpdateException(t);
} finally {
updateLock.unlock();
}
time = System.currentTimeMillis() - time;
// adjust timeout to compensate the time spent doing the replication.
final long timeout = interval - time;
if (timeout > 0) {
try {
// this will return immediately if we were ordered to stop (count=0)
// or the timeout has elapsed. if it returns true, it means count=0,
// so terminate.
if (stop.await(timeout, TimeUnit.MILLISECONDS)) {
return;
}
} catch (InterruptedException e) {
// if we were interruted, somebody wants to terminate us, so just
// throw the exception further.
Thread.currentThread().interrupt();
throw new ThreadInterruptedException(e);
}
}
}
}
}
/** Handler for revisions obtained by the client. */
public static interface ReplicationHandler {
/** Returns the current revision files held by the handler. */
public Map<String,List<RevisionFile>> currentRevisionFiles();
/** Returns the current revision version held by the handler. */
public String currentVersion();
/**
* Called when a new revision was obtained and is available (i.e. all needed
* files were successfully copied).
*
* @param version
* the version of the {@link Revision} that was copied
* @param revisionFiles
* the files contained by this {@link Revision}
* @param copiedFiles
* the files that were actually copied
* @param sourceDirectory
* a mapping from a source of files to the {@link Directory} they
* were copied into
*/
public void revisionReady(String version, Map<String,List<RevisionFile>> revisionFiles,
Map<String,List<String>> copiedFiles, Map<String, Directory> sourceDirectory) throws IOException;
}
/**
* Resolves a session and source into a {@link Directory} to use for copying
* the session files to.
*/
public static interface SourceDirectoryFactory {
/**
* Called to denote that the replication actions for this session were finished and the directory is no longer needed.
*/
public void cleanupSession(String sessionID) throws IOException;
/**
* Returns the {@link Directory} to use for the given session and source.
* Implementations may e.g. return different directories for different
* sessions, or the same directory for all sessions. In that case, it is
* advised to clean the directory before it is used for a new session.
*
* @see #cleanupSession(String)
*/
public Directory getDirectory(String sessionID, String source) throws IOException;
}
/** The component name to use with {@link InfoStream#isEnabled(String)}. */
public static final String INFO_STREAM_COMPONENT = "ReplicationThread";
private final Replicator replicator;
private final ReplicationHandler handler;
private final SourceDirectoryFactory factory;
private final byte[] copyBuffer = new byte[16384];
private final Lock updateLock = new ReentrantLock();
private volatile ReplicationThread updateThread;
private volatile boolean closed = false;
private volatile InfoStream infoStream = InfoStream.getDefault();
/**
* Constructor.
*
* @param replicator the {@link Replicator} used for checking for updates
* @param handler notified when new revisions are ready
* @param factory returns a {@link Directory} for a given source and session
*/
public ReplicationClient(Replicator replicator, ReplicationHandler handler, SourceDirectoryFactory factory) {
this.replicator = replicator;
this.handler = handler;
this.factory = factory;
}
private void copyBytes(IndexOutput out, InputStream in) throws IOException {
int numBytes;
while ((numBytes = in.read(copyBuffer)) > 0) {
out.writeBytes(copyBuffer, 0, numBytes);
}
}
private void doUpdate() throws IOException {
SessionToken session = null;
final Map<String,Directory> sourceDirectory = new HashMap<String,Directory>();
final Map<String,List<String>> copiedFiles = new HashMap<String,List<String>>();
boolean notify = false;
try {
final String version = handler.currentVersion();
session = replicator.checkForUpdate(version);
if (infoStream.isEnabled(INFO_STREAM_COMPONENT)) {
infoStream.message(INFO_STREAM_COMPONENT, "doUpdate(): handlerVersion=" + version + " session=" + session);
}
if (session == null) {
// already up to date
return;
}
Map<String,List<RevisionFile>> requiredFiles = requiredFiles(session.sourceFiles);
if (infoStream.isEnabled(INFO_STREAM_COMPONENT)) {
infoStream.message(INFO_STREAM_COMPONENT, "doUpdate(): requiredFiles=" + requiredFiles);
}
for (Entry<String,List<RevisionFile>> e : requiredFiles.entrySet()) {
String source = e.getKey();
Directory dir = factory.getDirectory(session.id, source);
sourceDirectory.put(source, dir);
List<String> cpFiles = new ArrayList<String>();
copiedFiles.put(source, cpFiles);
for (RevisionFile file : e.getValue()) {
if (closed) {
// if we're closed, abort file copy
if (infoStream.isEnabled(INFO_STREAM_COMPONENT)) {
infoStream.message(INFO_STREAM_COMPONENT, "doUpdate(): detected client was closed); abort file copy");
}
return;
}
InputStream in = null;
IndexOutput out = null;
try {
in = replicator.obtainFile(session.id, source, file.fileName);
out = dir.createOutput(file.fileName, IOContext.DEFAULT);
copyBytes(out, in);
cpFiles.add(file.fileName);
// TODO add some validation, on size / checksum
} finally {
IOUtils.close(in, out);
}
}
}
// only notify if all required files were successfully obtained.
notify = true;
} finally {
if (session != null) {
try {
replicator.release(session.id);
} finally {
if (!notify) { // cleanup after ourselves
IOUtils.close(sourceDirectory.values());
factory.cleanupSession(session.id);
}
}
}
}
// notify outside the try-finally above, so the session is released sooner.
// the handler may take time to finish acting on the copied files, but the
// session itself is no longer needed.
try {
if (notify && !closed ) { // no use to notify if we are closed already
handler.revisionReady(session.version, session.sourceFiles, copiedFiles, sourceDirectory);
}
} finally {
IOUtils.close(sourceDirectory.values());
if (session != null) {
factory.cleanupSession(session.id);
}
}
}
/** Throws {@link AlreadyClosedException} if the client has already been closed. */
protected final void ensureOpen() {
if (closed) {
throw new AlreadyClosedException("this update client has already been closed");
}
}
/**
* Called when an exception is hit by the replication thread. The default
* implementation prints the full stacktrace to the {@link InfoStream} set in
* {@link #setInfoStream(InfoStream)}, or the {@link InfoStream#getDefault()
* default} one. You can override to log the exception elswhere.
* <p>
* <b>NOTE:</b> if you override this method to throw the exception further,
* the replication thread will be terminated. The only way to restart it is to
* call {@link #stopUpdateThread()} followed by
* {@link #startUpdateThread(long, String)}.
*/
protected void handleUpdateException(Throwable t) {
final StringWriter sw = new StringWriter();
t.printStackTrace(new PrintWriter(sw));
if (infoStream.isEnabled(INFO_STREAM_COMPONENT)) {
infoStream.message(INFO_STREAM_COMPONENT, "an error occurred during revision update: " + sw.toString());
}
}
/**
* Returns the files required for replication. By default, this method returns
* all files that exist in the new revision, but not in the handler.
*/
protected Map<String,List<RevisionFile>> requiredFiles(Map<String,List<RevisionFile>> newRevisionFiles) {
Map<String,List<RevisionFile>> handlerRevisionFiles = handler.currentRevisionFiles();
if (handlerRevisionFiles == null) {
return newRevisionFiles;
}
Map<String,List<RevisionFile>> requiredFiles = new HashMap<String,List<RevisionFile>>();
for (Entry<String,List<RevisionFile>> e : handlerRevisionFiles.entrySet()) {
// put the handler files in a Set, for faster contains() checks later
Set<String> handlerFiles = new HashSet<String>();
for (RevisionFile file : e.getValue()) {
handlerFiles.add(file.fileName);
}
// make sure to preserve revisionFiles order
ArrayList<RevisionFile> res = new ArrayList<RevisionFile>();
String source = e.getKey();
assert newRevisionFiles.containsKey(source) : "source not found in newRevisionFiles: " + newRevisionFiles;
for (RevisionFile file : newRevisionFiles.get(source)) {
if (!handlerFiles.contains(file.fileName)) {
res.add(file);
}
}
requiredFiles.put(source, res);
}
return requiredFiles;
}
@Override
public synchronized void close() {
if (!closed) {
stopUpdateThread();
closed = true;
}
}
/**
* Start the update thread with the specified interval in milliseconds. For
* debugging purposes, you can optionally set the name to set on
* {@link Thread#setName(String)}. If you pass {@code null}, a default name
* will be set.
*
* @throws IllegalStateException if the thread has already been started
*/
public synchronized void startUpdateThread(long intervalMillis, String threadName) {
ensureOpen();
if (updateThread != null && updateThread.isAlive()) {
throw new IllegalStateException(
"cannot start an update thread when one is running, must first call 'stopUpdateThread()'");
}
threadName = threadName == null ? INFO_STREAM_COMPONENT : "ReplicationThread-" + threadName;
updateThread = new ReplicationThread(intervalMillis);
updateThread.setName(threadName);
updateThread.start();
// we rely on isAlive to return true in isUpdateThreadAlive, assert to be on the safe side
assert updateThread.isAlive() : "updateThread started but not alive?";
}
/**
* Stop the update thread. If the update thread is not running, silently does
* nothing. This method returns after the update thread has stopped.
*/
public synchronized void stopUpdateThread() {
if (updateThread != null) {
// this will trigger the thread to terminate if it awaits the lock.
// otherwise, if it's in the middle of replication, we wait for it to
// stop.
updateThread.stop.countDown();
try {
updateThread.join();
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
throw new ThreadInterruptedException(e);
}
updateThread = null;
}
}
/**
* Returns true if the update thread is alive. The update thread is alive if
* it has been {@link #startUpdateThread(long, String) started} and not
* {@link #stopUpdateThread() stopped}, as well as didn't hit an error which
* caused it to terminate (i.e. {@link #handleUpdateException(Throwable)}
* threw the exception further).
*/
public synchronized boolean isUpdateThreadAlive() {
return updateThread != null && updateThread.isAlive();
}
@Override
public String toString() {
String res = "ReplicationClient";
if (updateThread != null) {
res += " (" + updateThread.getName() + ")";
}
return res;
}
/**
* Executes the update operation immediately, irregardess if an update thread
* is running or not.
*/
public void updateNow() throws IOException {
ensureOpen();
updateLock.lock();
try {
doUpdate();
} finally {
updateLock.unlock();
}
}
/** Sets the {@link InfoStream} to use for logging messages. */
public void setInfoStream(InfoStream infoStream) {
if (infoStream == null) {
infoStream = InfoStream.NO_OUTPUT;
}
this.infoStream = infoStream;
}
}

80
lucene/replicator/src/java/org/apache/lucene/replicator/Replicator.java Executable file
View File

@ -0,0 +1,80 @@
package org.apache.lucene.replicator;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.Closeable;
import java.io.IOException;
import java.io.InputStream;
/**
* An interface for replicating files. Allows a producer to
* {@link #publish(Revision) publish} {@link Revision}s and consumers to
* {@link #checkForUpdate(String) check for updates}. When a client needs to be
* updated, it is given a {@link SessionToken} through which it can
* {@link #obtainFile(String, String, String) obtain} the files of that
* revision. After the client has finished obtaining all the files, it should
* {@link #release(String) release} the given session, so that the files can be
* reclaimed if they are not needed anymore.
* <p>
* A client is always updated to the newest revision available. That is, if a
* client is on revision <em>r1</em> and revisions <em>r2</em> and <em>r3</em>
* were published, then when the cllient will next check for update, it will
* receive <em>r3</em>.
*
* @lucene.experimental
*/
public interface Replicator extends Closeable {
/**
* Publish a new {@link Revision} for consumption by clients. It is the
* caller's responsibility to verify that the revision files exist and can be
* read by clients. When the revision is no longer needed, it will be
* {@link Revision#release() released} by the replicator.
*/
public void publish(Revision revision) throws IOException;
/**
* Check whether the given version is up-to-date and returns a
* {@link SessionToken} which can be used for fetching the revision files,
* otherwise returns {@code null}.
* <p>
* <b>NOTE:</b> when the returned session token is no longer needed, you
* should call {@link #release(String)} so that the session resources can be
* reclaimed, including the revision files.
*/
public SessionToken checkForUpdate(String currVersion) throws IOException;
/**
* Notify that the specified {@link SessionToken} is no longer needed by the
* caller.
*/
public void release(String sessionID) throws IOException;
/**
* Returns an {@link InputStream} for the requested file and source in the
* context of the given {@link SessionToken#id session}.
* <p>
* <b>NOTE:</b> it is the caller's responsibility to close the returned
* stream.
*
* @throws SessionExpiredException if the specified session has already
* expired
*/
public InputStream obtainFile(String sessionID, String source, String fileName) throws IOException;
}

75
lucene/replicator/src/java/org/apache/lucene/replicator/Revision.java Executable file
View File

@ -0,0 +1,75 @@
package org.apache.lucene.replicator;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.IOException;
import java.io.InputStream;
import java.util.List;
import java.util.Map;
import org.apache.lucene.store.IndexInput;
/**
* A revision comprises lists of files that come from different sources and need
* to be replicated together to e.g. guarantee that all resources are in sync.
* In most cases an application will replicate a single index, and so the
* revision will contain files from a single source. However, some applications
* may require to treat a collection of indexes as a single entity so that the
* files from all sources are replicated together, to guarantee consistency
* beween them. For example, an application which indexes facets will need to
* replicate both the search and taxonomy indexes together, to guarantee that
* they match at the client side.
*
* @lucene.experimental
*/
public interface Revision extends Comparable<Revision> {
/**
* Compares the revision to the given version string. Behaves like
* {@link Comparable#compareTo(Object)}.
*/
public int compareTo(String version);
/**
* Returns a string representation of the version of this revision. The
* version is used by {@link #compareTo(String)} as well as to
* serialize/deserialize revision information. Therefore it must be self
* descriptive as well as be able to identify one revision from another.
*/
public String getVersion();
/**
* Returns the files that comprise this revision, as a mapping from a source
* to a list of files.
*/
public Map<String,List<RevisionFile>> getSourceFiles();
/**
* Returns an {@link IndexInput} for the given fileName and source. It is the
* caller's respnsibility to close the {@link IndexInput} when it has been
* consumed.
*/
public InputStream open(String source, String fileName) throws IOException;
/**
* Called when this revision can be safely released, i.e. where there are no
* more references to it.
*/
public void release() throws IOException;
}

59
lucene/replicator/src/java/org/apache/lucene/replicator/RevisionFile.java Executable file
View File

@ -0,0 +1,59 @@
package org.apache.lucene.replicator;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* Describes a file in a {@link Revision}. A file has a source, which allows a
* single revision to contain files from multiple sources (e.g. multiple
* indexes).
*
* @lucene.experimental
*/
public class RevisionFile {
/** The name of the file. */
public final String fileName;
/** The size of the file denoted by {@link #fileName}. */
public long size = -1;
/** Constructor with the given file name. */
public RevisionFile(String fileName) {
if (fileName == null || fileName.isEmpty()) {
throw new IllegalArgumentException("fileName cannot be null or empty");
}
this.fileName = fileName;
}
@Override
public boolean equals(Object obj) {
RevisionFile other = (RevisionFile) obj;
return fileName.equals(other.fileName) && size == other.size;
}
@Override
public int hashCode() {
return fileName.hashCode() ^ (int) (size ^ (size >>> 32));
}
@Override
public String toString() {
return "fileName=" + fileName + " size=" + size;
}
}

54
lucene/replicator/src/java/org/apache/lucene/replicator/SessionExpiredException.java Executable file
View File

@ -0,0 +1,54 @@
package org.apache.lucene.replicator;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.IOException;
/**
* Exception indicating that a revision update session was expired due to lack
* of activity.
*
* @see LocalReplicator#DEFAULT_SESSION_EXPIRATION_THRESHOLD
* @see LocalReplicator#setExpirationThreshold(long)
*
* @lucene.experimental
*/
public class SessionExpiredException extends IOException {
/**
* @see IOException#IOException(String, Throwable)
*/
public SessionExpiredException(String message, Throwable cause) {
super(message, cause);
}
/**
* @see IOException#IOException(String)
*/
public SessionExpiredException(String message) {
super(message);
}
/**
* @see IOException#IOException(Throwable)
*/
public SessionExpiredException(Throwable cause) {
super(cause);
}
}

108
lucene/replicator/src/java/org/apache/lucene/replicator/SessionToken.java Executable file
View File

@ -0,0 +1,108 @@
package org.apache.lucene.replicator;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
/**
* Token for a replication session, for guaranteeing that source replicated
* files will be kept safe until the replication completes.
*
* @see Replicator#checkForUpdate(String)
* @see Replicator#release(String)
* @see LocalReplicator#DEFAULT_SESSION_EXPIRATION_THRESHOLD
*
* @lucene.experimental
*/
public final class SessionToken {
/**
* ID of this session.
* Should be passed when releasing the session, thereby acknowledging the
* {@link Replicator Replicator} that this session is no longer in use.
* @see Replicator#release(String)
*/
public final String id;
/**
* @see Revision#getVersion()
*/
public final String version;
/**
* @see Revision#getSourceFiles()
*/
public final Map<String,List<RevisionFile>> sourceFiles;
/** Constructor which deserializes from the given {@link DataInput}. */
public SessionToken(DataInput in) throws IOException {
this.id = in.readUTF();
this.version = in.readUTF();
this.sourceFiles = new HashMap<String,List<RevisionFile>>();
int numSources = in.readInt();
while (numSources > 0) {
String source = in.readUTF();
int numFiles = in.readInt();
List<RevisionFile> files = new ArrayList<RevisionFile>(numFiles);
for (int i = 0; i < numFiles; i++) {
String fileName = in.readUTF();
RevisionFile file = new RevisionFile(fileName);
file.size = in.readLong();
files.add(file);
}
this.sourceFiles.put(source, files);
--numSources;
}
}
/** Constructor with the given id and revision. */
public SessionToken(String id, Revision revision) {
this.id = id;
this.version = revision.getVersion();
this.sourceFiles = revision.getSourceFiles();
}
/** Serialize the token data for communication between server and client. */
public void serialize(DataOutput out) throws IOException {
out.writeUTF(id);
out.writeUTF(version);
out.writeInt(sourceFiles.size());
for (Entry<String,List<RevisionFile>> e : sourceFiles.entrySet()) {
out.writeUTF(e.getKey());
List<RevisionFile> files = e.getValue();
out.writeInt(files.size());
for (RevisionFile file : files) {
out.writeUTF(file.fileName);
out.writeLong(file.size);
}
}
}
@Override
public String toString() {
return "id=" + id + " version=" + version + " files=" + sourceFiles;
}
}

297
lucene/replicator/src/java/org/apache/lucene/replicator/http/HttpClientBase.java Executable file
View File

@ -0,0 +1,297 @@
package org.apache.lucene.replicator.http;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.Closeable;
import java.io.IOException;
import java.io.InputStream;
import java.io.ObjectInputStream;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.util.concurrent.Callable;
import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.HttpStatus;
import org.apache.http.StatusLine;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.conn.ClientConnectionManager;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.params.HttpConnectionParams;
import org.apache.http.util.EntityUtils;
import org.apache.lucene.store.AlreadyClosedException;
/**
* Base class for Http clients.
*
* @lucene.experimental
* */
public abstract class HttpClientBase implements Closeable {
/**
* Default connection timeout for this client, in milliseconds.
*
* @see #setConnectionTimeout(int)
*/
public static final int DEFAULT_CONNECTION_TIMEOUT = 1000;
/**
* Default socket timeout for this client, in milliseconds.
*
* @see #setSoTimeout(int)
*/
public static final int DEFAULT_SO_TIMEOUT = 60000;
// TODO compression?
/** The URL stting to execute requests against. */
protected final String url;
private volatile boolean closed = false;
private final HttpClient httpc;
/**
* @param conMgr connection manager to use for this http client.
* <b>NOTE:</b>The provided {@link ClientConnectionManager} will not be
* {@link ClientConnectionManager#shutdown()} by this class.
*/
protected HttpClientBase(String host, int port, String path, ClientConnectionManager conMgr) {
url = normalizedURL(host, port, path);
httpc = new DefaultHttpClient(conMgr);
setConnectionTimeout(DEFAULT_CONNECTION_TIMEOUT);
setSoTimeout(DEFAULT_SO_TIMEOUT);
}
/**
* Set the connection timeout for this client, in milliseconds. This setting
* is used to modify {@link HttpConnectionParams#setConnectionTimeout}.
*
* @param timeout timeout to set, in millisecopnds
*/
public void setConnectionTimeout(int timeout) {
HttpConnectionParams.setConnectionTimeout(httpc.getParams(), timeout);
}
/**
* Set the socket timeout for this client, in milliseconds. This setting
* is used to modify {@link HttpConnectionParams#setSoTimeout}.
*
* @param timeout timeout to set, in millisecopnds
*/
public void setSoTimeout(int timeout) {
HttpConnectionParams.setSoTimeout(httpc.getParams(), timeout);
}
/** Throws {@link AlreadyClosedException} if this client is already closed. */
protected final void ensureOpen() throws AlreadyClosedException {
if (closed) {
throw new AlreadyClosedException("HttpClient already closed");
}
}
/**
* Create a URL out of the given parameters, translate an empty/null path to '/'
*/
private static String normalizedURL(String host, int port, String path) {
if (path == null || path.length() == 0) {
path = "/";
}
return "http://" + host + ":" + port + path;
}
/**
* <b>Internal:</b> response status after invocation, and in case or error attempt to read the
* exception sent by the server.
*/
protected void verifyStatus(HttpResponse response) throws IOException {
StatusLine statusLine = response.getStatusLine();
if (statusLine.getStatusCode() != HttpStatus.SC_OK) {
throwKnownError(response, statusLine);
}
}
protected void throwKnownError(HttpResponse response, StatusLine statusLine) throws IOException {
ObjectInputStream in = null;
try {
in = new ObjectInputStream(response.getEntity().getContent());
} catch (Exception e) {
// the response stream is not an exception - could be an error in servlet.init().
throw new RuntimeException("Uknown error: " + statusLine);
}
Throwable t;
try {
t = (Throwable) in.readObject();
} catch (Exception e) {
//not likely
throw new RuntimeException("Failed to read exception object: " + statusLine, e);
} finally {
in.close();
}
if (t instanceof IOException) {
throw (IOException) t;
}
if (t instanceof RuntimeException) {
throw (RuntimeException) t;
}
throw new RuntimeException("unknown exception "+statusLine,t);
}
/**
* <b>internal:</b> execute a request and return its result
* The <code>params</code> argument is treated as: name1,value1,name2,value2,...
*/
protected HttpResponse executePOST(String request, HttpEntity entity, String... params) throws IOException {
ensureOpen();
HttpPost m = new HttpPost(queryString(request, params));
m.setEntity(entity);
HttpResponse response = httpc.execute(m);
verifyStatus(response);
return response;
}
/**
* <b>internal:</b> execute a request and return its result
* The <code>params</code> argument is treated as: name1,value1,name2,value2,...
*/
protected HttpResponse executeGET(String request, String... params) throws IOException {
ensureOpen();
HttpGet m = new HttpGet(queryString(request, params));
HttpResponse response = httpc.execute(m);
verifyStatus(response);
return response;
}
private String queryString(String request, String... params) throws UnsupportedEncodingException {
StringBuilder query = new StringBuilder(url).append('/').append(request).append('?');
if (params != null) {
for (int i = 0; i < params.length; i += 2) {
query.append(params[i]).append('=').append(URLEncoder.encode(params[i+1], "UTF8")).append('&');
}
}
return query.substring(0, query.length() - 1);
}
/** Internal utility: input stream of the provided response */
public InputStream responseInputStream(HttpResponse response) throws IOException {
return responseInputStream(response, false);
}
// TODO: can we simplify this Consuming !?!?!?
/**
* Internal utility: input stream of the provided response, which optionally
* consumes the response's resources when the input stream is exhausted.
*/
public InputStream responseInputStream(HttpResponse response, boolean consume) throws IOException {
final HttpEntity entity = response.getEntity();
final InputStream in = entity.getContent();
if (!consume) {
return in;
}
return new InputStream() {
private boolean consumed = false;
@Override
public int read() throws IOException {
final int res = in.read();
consume(res);
return res;
}
@Override
public void close() throws IOException {
super.close();
consume(-1);
}
@Override
public int read(byte[] b) throws IOException {
final int res = super.read(b);
consume(res);
return res;
}
@Override
public int read(byte[] b, int off, int len) throws IOException {
final int res = super.read(b, off, len);
consume(res);
return res;
}
private void consume(int minusOne) {
if (!consumed && minusOne==-1) {
try {
EntityUtils.consume(entity);
} catch (Exception e) {
// ignored on purpose
}
consumed = true;
}
}
};
}
/**
* Returns true iff this instance was {@link #close() closed}, otherwise
* returns false. Note that if you override {@link #close()}, you must call
* {@code super.close()}, in order for this instance to be properly closed.
*/
protected final boolean isClosed() {
return closed;
}
/**
* Same as {@link #doAction(HttpResponse, boolean, Callable)} but always do consume at the end.
*/
protected <T> T doAction(HttpResponse response, Callable<T> call) throws IOException {
return doAction(response, true, call);
}
/**
* Do a specific action and validate after the action that the status is still OK,
* and if not, attempt to extract the actual server side exception. Optionally
* release the response at exit, depending on <code>consume</code> parameter.
*/
protected <T> T doAction(HttpResponse response, boolean consume, Callable<T> call) throws IOException {
IOException error = null;
try {
return call.call();
} catch (IOException e) {
error = e;
} catch (Exception e) {
error = new IOException(e);
} finally {
try {
verifyStatus(response);
} finally {
if (consume) {
try {
EntityUtils.consume(response.getEntity());
} catch (Exception e) {
// ignoring on purpose
}
}
}
}
throw error; // should not get here
}
@Override
public void close() throws IOException {
closed = true;
}
}

105
lucene/replicator/src/java/org/apache/lucene/replicator/http/HttpReplicator.java Executable file
View File

@ -0,0 +1,105 @@
package org.apache.lucene.replicator.http;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.DataInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.concurrent.Callable;
import org.apache.http.HttpResponse;
import org.apache.http.conn.ClientConnectionManager;
import org.apache.lucene.replicator.Replicator;
import org.apache.lucene.replicator.Revision;
import org.apache.lucene.replicator.SessionToken;
import org.apache.lucene.replicator.http.ReplicationService.ReplicationAction;
/**
* An HTTP implementation of {@link Replicator}. Assumes the API supported by
* {@link ReplicationService}.
*
* @lucene.experimental
*/
public class HttpReplicator extends HttpClientBase implements Replicator {
/** Construct with specified connection manager. */
public HttpReplicator(String host, int port, String path, ClientConnectionManager conMgr) {
super(host, port, path, conMgr);
}
@Override
public SessionToken checkForUpdate(String currVersion) throws IOException {
String[] params = null;
if (currVersion != null) {
params = new String[] { ReplicationService.REPLICATE_VERSION_PARAM, currVersion };
}
final HttpResponse response = executeGET(ReplicationAction.UPDATE.name(), params);
return doAction(response, new Callable<SessionToken>() {
@Override
public SessionToken call() throws Exception {
final DataInputStream dis = new DataInputStream(responseInputStream(response));
try {
if (dis.readByte() == 0) {
return null;
} else {
return new SessionToken(dis);
}
} finally {
dis.close();
}
}
});
}
@Override
public InputStream obtainFile(String sessionID, String source, String fileName) throws IOException {
String[] params = new String[] {
ReplicationService.REPLICATE_SESSION_ID_PARAM, sessionID,
ReplicationService.REPLICATE_SOURCE_PARAM, source,
ReplicationService.REPLICATE_FILENAME_PARAM, fileName,
};
final HttpResponse response = executeGET(ReplicationAction.OBTAIN.name(), params);
return doAction(response, false, new Callable<InputStream>() {
@Override
public InputStream call() throws Exception {
return responseInputStream(response,true);
}
});
}
@Override
public void publish(Revision revision) throws IOException {
throw new UnsupportedOperationException(
"this replicator implementation does not support remote publishing of revisions");
}
@Override
public void release(String sessionID) throws IOException {
String[] params = new String[] {
ReplicationService.REPLICATE_SESSION_ID_PARAM, sessionID
};
final HttpResponse response = executeGET(ReplicationAction.RELEASE.name(), params);
doAction(response, new Callable<Object>() {
@Override
public Object call() throws Exception {
return null; // do not remove this call: as it is still validating for us!
}
});
}
}

198
lucene/replicator/src/java/org/apache/lucene/replicator/http/ReplicationService.java Executable file
View File

@ -0,0 +1,198 @@
package org.apache.lucene.replicator.http;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.DataOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.ObjectOutputStream;
import java.io.OutputStream;
import java.util.ArrayList;
import java.util.Locale;
import java.util.Map;
import java.util.StringTokenizer;
import javax.servlet.ServletException;
import javax.servlet.ServletOutputStream;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.apache.http.HttpStatus;
import org.apache.lucene.replicator.Replicator;
import org.apache.lucene.replicator.SessionToken;
/**
* A server-side service for handling replication requests. The service assumes
* requests are sent in the format
* <code>/&lt;context&gt;/&lt;shard&gt;/&lt;action&gt;</code> where
* <ul>
* <li>{@code context} is the servlet context, e.g. {@link #REPLICATION_CONTEXT}
* <li>{@code shard} is the ID of the shard, e.g. "s1"
* <li>{@code action} is one of {@link ReplicationAction} values
* </ul>
* For example, to check whether there are revision updates for shard "s1" you
* should send the request: <code>http://host:port/replicate/s1/update</code>.
* <p>
* This service is written like a servlet, and
* {@link #perform(HttpServletRequest, HttpServletResponse)} takes servlet
* request and response accordingly, so it is quite easy to embed in your
* application's servlet.
*
* @lucene.experimental
*/
public class ReplicationService {
/** Actions supported by the {@link ReplicationService}. */
public enum ReplicationAction {
OBTAIN, RELEASE, UPDATE
}
/** The context path for the servlet. */
public static final String REPLICATION_CONTEXT = "/replicate";
/** Request parameter name for providing the revision version. */
public final static String REPLICATE_VERSION_PARAM = "version";
/** Request parameter name for providing a session ID. */
public final static String REPLICATE_SESSION_ID_PARAM = "sessionid";
/** Request parameter name for providing the file's source. */
public final static String REPLICATE_SOURCE_PARAM = "source";
/** Request parameter name for providing the file's name. */
public final static String REPLICATE_FILENAME_PARAM = "filename";
private static final int SHARD_IDX = 0, ACTION_IDX = 1;
private final Map<String,Replicator> replicators;
public ReplicationService(Map<String,Replicator> replicators) {
super();
this.replicators = replicators;
}
/**
* Returns the path elements that were given in the servlet request, excluding
* the servlet's action context.
*/
private String[] getPathElements(HttpServletRequest req) {
String path = req.getServletPath();
String pathInfo = req.getPathInfo();
if (pathInfo != null) {
path += pathInfo;
}
int actionLen = REPLICATION_CONTEXT.length();
int startIdx = actionLen;
if (path.length() > actionLen && path.charAt(actionLen) == '/') {
++startIdx;
}
// split the string on '/' and remove any empty elements. This is better
// than using String.split() since the latter may return empty elements in
// the array
StringTokenizer stok = new StringTokenizer(path.substring(startIdx), "/");
ArrayList<String> elements = new ArrayList<String>();
while (stok.hasMoreTokens()) {
elements.add(stok.nextToken());
}
return elements.toArray(new String[0]);
}
private static String extractRequestParam(HttpServletRequest req, String paramName) throws ServletException {
String param = req.getParameter(paramName);
if (param == null) {
throw new ServletException("Missing mandatory parameter: " + paramName);
}
return param;
}
private static void copy(InputStream in, OutputStream out) throws IOException {
byte[] buf = new byte[16384];
int numRead;
while ((numRead = in.read(buf)) != -1) {
out.write(buf, 0, numRead);
}
}
/** Executes the replication task. */
public void perform(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException {
String[] pathElements = getPathElements(req);
if (pathElements.length != 2) {
throw new ServletException("invalid path, must contain shard ID and action, e.g. */s1/update");
}
final ReplicationAction action;
try {
action = ReplicationAction.valueOf(pathElements[ACTION_IDX].toUpperCase(Locale.ENGLISH));
} catch (IllegalArgumentException e) {
throw new ServletException("Unsupported action provided: " + pathElements[ACTION_IDX]);
}
final Replicator replicator = replicators.get(pathElements[SHARD_IDX]);
if (replicator == null) {
throw new ServletException("unrecognized shard ID " + pathElements[SHARD_IDX]);
}
ServletOutputStream resOut = resp.getOutputStream();
try {
switch (action) {
case OBTAIN:
final String sessionID = extractRequestParam(req, REPLICATE_SESSION_ID_PARAM);
final String fileName = extractRequestParam(req, REPLICATE_FILENAME_PARAM);
final String source = extractRequestParam(req, REPLICATE_SOURCE_PARAM);
InputStream in = replicator.obtainFile(sessionID, source, fileName);
try {
copy(in, resOut);
} finally {
in.close();
}
break;
case RELEASE:
replicator.release(extractRequestParam(req, REPLICATE_SESSION_ID_PARAM));
break;
case UPDATE:
String currVersion = req.getParameter(REPLICATE_VERSION_PARAM);
SessionToken token = replicator.checkForUpdate(currVersion);
if (token == null) {
resOut.write(0); // marker for null token
} else {
resOut.write(1); // marker for null token
token.serialize(new DataOutputStream(resOut));
}
break;
}
} catch (Exception e) {
resp.setStatus(HttpStatus.SC_INTERNAL_SERVER_ERROR); // propagate the failure
try {
/*
* Note: it is assumed that "identified exceptions" are thrown before
* anything was written to the stream.
*/
ObjectOutputStream oos = new ObjectOutputStream(resOut);
oos.writeObject(e);
oos.flush();
} catch (Exception e2) {
throw new IOException("Could not serialize", e2);
}
} finally {
resp.flushBuffer();
}
}
}

28
lucene/replicator/src/java/org/apache/lucene/replicator/http/package.html Executable file
View File

@ -0,0 +1,28 @@
<html>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<head>
<title>HTTP replication implementation</title>
</head>
<body>
<h1>HTTP replication implementation</h1>
</body>
</html>

79
lucene/replicator/src/java/org/apache/lucene/replicator/package.html Executable file
View File

@ -0,0 +1,79 @@
<html>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<head>
<title>Files replication framework</title>
</head>
<body>
<h1>Files replication framework</h1>
The
<a href="Replicator.html">Replicator</a> allows replicating files between a server and client(s). Producers publish
<a href="Revision.html">revisions</a> and consumers update to the latest revision available.
<a href="ReplicationClient.html">ReplicationClient</a> is a helper utility for performing the update operation. It can
be invoked either
<a href="ReplicationClient.html#updateNow()">manually</a> or periodically by
<a href="ReplicationClient.html#startUpdateThread(long, java.lang.String)">starting an update thread</a>.
<a href="http/HttpReplicator.html">HttpReplicator</a> can be used to replicate revisions by consumers that reside on
a different node than the producer.
<p />
The replication framework supports replicating any type of files, with built-in support for a single search index as
well as an index and taxonomy pair. For a single index, the application should publish an
<a href="IndexRevision.html">IndexRevision</a> and set
<a href="IndexReplicationHandler.html">IndexReplicationHandler</a> on the client. For an index and taxonomy pair, the
application should publish an <a href="IndexAndTaxonomyRevision.html">IndexAndTaxonomyRevision</a> and set
<a href="IndexAndTaxonomyReplicationHandler.html">IndexAndTaxonomyReplicationHandler</a> on the client.
<p />
When the replication client detects that there is a newer revision available, it copies the files of the revision and
then invokes the handler to complete the operation (e.g. copy the files to the index directory, fsync them, reopen an
index reader etc.). By default, only files that do not exist in the handler's
<a href="ReplicationClient.ReplicationHandler.html#currentRevisionFiles()">current revision files</a> are copied,
however this can be overridden by extending the client.
<p />
An example usage of the Replicator:
<pre class="prettyprint lang-java">
// ++++++++++++++ SERVER SIDE ++++++++++++++ //
IndexWriter publishWriter; // the writer used for indexing
Replicator replicator = new LocalReplicator();
replicator.publish(new IndexRevision(publishWriter));
// ++++++++++++++ CLIENT SIDE ++++++++++++++ //
// either LocalReplictor, or HttpReplicator if client and server are on different nodes
Replicator replicator;
// callback invoked after handler finished handling the revision and e.g. can reopen the reader.
Callable&lt;Boolean&gt; callback = null; // can also be null if no callback is needed
ReplicationHandler handler = new IndexReplicationHandler(indexDir, callback);
SourceDirectoryFactory factory = new PerSessionDirectoryFactory(workDir);
ReplicationClient client = new ReplicationClient(replicator, handler, factory);
// invoke client manually
client.updateNow();
// or, periodically
client.startUpdateThread(100); // check for update every 100 milliseconds
</pre>
</body>
</html>

26
lucene/replicator/src/java/overview.html Executable file
View File

@ -0,0 +1,26 @@
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<html>
<head>
<title>
replicator
</title>
</head>
<body>
Provides index files replication capabilities.
</body>
</html>

444
lucene/replicator/src/test/org/apache/lucene/replicator/IndexAndTaxonomyReplicationClientTest.java Executable file
View File

@ -0,0 +1,444 @@
package org.apache.lucene.replicator;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.Closeable;
import java.io.File;
import java.io.IOException;
import java.util.Collections;
import java.util.HashMap;
import java.util.concurrent.Callable;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.lucene.document.Document;
import org.apache.lucene.facet.index.FacetFields;
import org.apache.lucene.facet.params.FacetIndexingParams;
import org.apache.lucene.facet.params.FacetSearchParams;
import org.apache.lucene.facet.search.CountFacetRequest;
import org.apache.lucene.facet.search.DrillDownQuery;
import org.apache.lucene.facet.search.FacetsCollector;
import org.apache.lucene.facet.taxonomy.CategoryPath;
import org.apache.lucene.facet.taxonomy.TaxonomyReader;
import org.apache.lucene.facet.taxonomy.TaxonomyWriter;
import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyReader;
import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.SnapshotDeletionPolicy;
import org.apache.lucene.replicator.IndexAndTaxonomyRevision.SnapshotDirectoryTaxonomyWriter;
import org.apache.lucene.replicator.ReplicationClient.ReplicationHandler;
import org.apache.lucene.replicator.ReplicationClient.SourceDirectoryFactory;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.MatchAllDocsQuery;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.MockDirectoryWrapper;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.ThreadInterruptedException;
import org.apache.lucene.util._TestUtil;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
public class IndexAndTaxonomyReplicationClientTest extends ReplicatorTestCase {
private static class IndexAndTaxonomyReadyCallback implements Callable<Boolean>, Closeable {
private final Directory indexDir, taxoDir;
private DirectoryReader indexReader;
private DirectoryTaxonomyReader taxoReader;
private long lastIndexGeneration = -1;
public IndexAndTaxonomyReadyCallback(Directory indexDir, Directory taxoDir) throws IOException {
this.indexDir = indexDir;
this.taxoDir = taxoDir;
if (DirectoryReader.indexExists(indexDir)) {
indexReader = DirectoryReader.open(indexDir);
lastIndexGeneration = indexReader.getIndexCommit().getGeneration();
taxoReader = new DirectoryTaxonomyReader(taxoDir);
}
}
@Override
public Boolean call() throws Exception {
if (indexReader == null) {
indexReader = DirectoryReader.open(indexDir);
lastIndexGeneration = indexReader.getIndexCommit().getGeneration();
taxoReader = new DirectoryTaxonomyReader(taxoDir);
} else {
// verify search index
DirectoryReader newReader = DirectoryReader.openIfChanged(indexReader);
assertNotNull("should not have reached here if no changes were made to the index", newReader);
long newGeneration = newReader.getIndexCommit().getGeneration();
assertTrue("expected newer generation; current=" + lastIndexGeneration + " new=" + newGeneration, newGeneration > lastIndexGeneration);
indexReader.close();
indexReader = newReader;
lastIndexGeneration = newGeneration;
_TestUtil.checkIndex(indexDir);
// verify taxonomy index
DirectoryTaxonomyReader newTaxoReader = TaxonomyReader.openIfChanged(taxoReader);
if (newTaxoReader != null) {
taxoReader.close();
taxoReader = newTaxoReader;
}
_TestUtil.checkIndex(taxoDir);
// verify faceted search
int id = Integer.parseInt(indexReader.getIndexCommit().getUserData().get(VERSION_ID), 16);
CategoryPath cp = new CategoryPath("A", Integer.toString(id, 16));
IndexSearcher searcher = new IndexSearcher(indexReader);
FacetsCollector fc = FacetsCollector.create(new FacetSearchParams(new CountFacetRequest(cp, 10)), indexReader, taxoReader);
searcher.search(new MatchAllDocsQuery(), fc);
assertEquals(1, (int) fc.getFacetResults().get(0).getFacetResultNode().value);
DrillDownQuery drillDown = new DrillDownQuery(FacetIndexingParams.DEFAULT);
drillDown.add(cp);
TopDocs docs = searcher.search(drillDown, 10);
assertEquals(1, docs.totalHits);
}
return null;
}
@Override
public void close() throws IOException {
IOUtils.close(indexReader, taxoReader);
}
}
private Directory publishIndexDir, publishTaxoDir;
private MockDirectoryWrapper handlerIndexDir, handlerTaxoDir;
private Replicator replicator;
private SourceDirectoryFactory sourceDirFactory;
private ReplicationClient client;
private ReplicationHandler handler;
private IndexWriter publishIndexWriter;
private SnapshotDirectoryTaxonomyWriter publishTaxoWriter;
private IndexAndTaxonomyReadyCallback callback;
private File clientWorkDir;
private static final String VERSION_ID = "version";
private void assertHandlerRevision(int expectedID, Directory dir) throws IOException {
// loop as long as client is alive. test-framework will terminate us if
// there's a serious bug, e.g. client doesn't really update. otherwise,
// introducing timeouts is not good, can easily lead to false positives.
while (client.isUpdateThreadAlive()) {
// give client a chance to update
try {
Thread.sleep(100);
} catch (InterruptedException e) {
throw new ThreadInterruptedException(e);
}
try {
DirectoryReader reader = DirectoryReader.open(dir);
try {
int handlerID = Integer.parseInt(reader.getIndexCommit().getUserData().get(VERSION_ID), 16);
if (expectedID == handlerID) {
return;
}
} finally {
reader.close();
}
} catch (Exception e) {
// we can hit IndexNotFoundException or e.g. EOFException (on
// segments_N) because it is being copied at the same time it is read by
// DirectoryReader.open().
}
}
}
private Revision createRevision(final int id) throws IOException {
publishIndexWriter.addDocument(newDocument(publishTaxoWriter, id));
publishIndexWriter.setCommitData(new HashMap<String, String>() {{
put(VERSION_ID, Integer.toString(id, 16));
}});
publishIndexWriter.commit();
publishTaxoWriter.commit();
return new IndexAndTaxonomyRevision(publishIndexWriter, publishTaxoWriter);
}
private Document newDocument(TaxonomyWriter taxoWriter, int id) throws IOException {
Document doc = new Document();
FacetFields facetFields = new FacetFields(taxoWriter);
facetFields.addFields(doc, Collections.singleton(new CategoryPath("A", Integer.toString(id, 16))));
return doc;
}
@Override
@Before
public void setUp() throws Exception {
super.setUp();
publishIndexDir = newDirectory();
publishTaxoDir = newDirectory();
handlerIndexDir = newMockDirectory();
handlerTaxoDir = newMockDirectory();
clientWorkDir = _TestUtil.getTempDir("replicationClientTest");
sourceDirFactory = new PerSessionDirectoryFactory(clientWorkDir);
replicator = new LocalReplicator();
callback = new IndexAndTaxonomyReadyCallback(handlerIndexDir, handlerTaxoDir);
handler = new IndexAndTaxonomyReplicationHandler(handlerIndexDir, handlerTaxoDir, callback);
client = new ReplicationClient(replicator, handler, sourceDirFactory);
IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, null);
conf.setIndexDeletionPolicy(new SnapshotDeletionPolicy(conf.getIndexDeletionPolicy()));
publishIndexWriter = new IndexWriter(publishIndexDir, conf);
publishTaxoWriter = new SnapshotDirectoryTaxonomyWriter(publishTaxoDir);
}
@After
@Override
public void tearDown() throws Exception {
IOUtils.close(client, callback, publishIndexWriter, publishTaxoWriter, replicator, publishIndexDir, publishTaxoDir,
handlerIndexDir, handlerTaxoDir);
super.tearDown();
}
@Test
public void testNoUpdateThread() throws Exception {
assertNull("no version expected at start", handler.currentVersion());
// Callback validates the replicated index
replicator.publish(createRevision(1));
client.updateNow();
// make sure updating twice, when in fact there's nothing to update, works
client.updateNow();
replicator.publish(createRevision(2));
client.updateNow();
// Publish two revisions without update, handler should be upgraded to latest
replicator.publish(createRevision(3));
replicator.publish(createRevision(4));
client.updateNow();
}
@Test
public void testRestart() throws Exception {
replicator.publish(createRevision(1));
client.updateNow();
replicator.publish(createRevision(2));
client.updateNow();
client.stopUpdateThread();
client.close();
client = new ReplicationClient(replicator, handler, sourceDirFactory);
// Publish two revisions without update, handler should be upgraded to latest
replicator.publish(createRevision(3));
replicator.publish(createRevision(4));
client.updateNow();
}
@Test
public void testUpdateThread() throws Exception {
client.startUpdateThread(10, "indexTaxo");
replicator.publish(createRevision(1));
assertHandlerRevision(1, handlerIndexDir);
replicator.publish(createRevision(2));
assertHandlerRevision(2, handlerIndexDir);
// Publish two revisions without update, handler should be upgraded to latest
replicator.publish(createRevision(3));
replicator.publish(createRevision(4));
assertHandlerRevision(4, handlerIndexDir);
}
@Test
public void testRecreateTaxonomy() throws Exception {
replicator.publish(createRevision(1));
client.updateNow();
// recreate index and taxonomy
Directory newTaxo = newDirectory();
new DirectoryTaxonomyWriter(newTaxo).close();
publishTaxoWriter.replaceTaxonomy(newTaxo);
publishIndexWriter.deleteAll();
replicator.publish(createRevision(2));
client.updateNow();
newTaxo.close();
}
/*
* This test verifies that the client and handler do not end up in a corrupt
* index if exceptions are thrown at any point during replication. Either when
* a client copies files from the server to the temporary space, or when the
* handler copies them to the index directory.
*/
@Test
public void testConsistencyOnExceptions() throws Exception {
// so the handler's index isn't empty
replicator.publish(createRevision(1));
client.updateNow();
client.close();
callback.close();
// Replicator violates write-once policy. It may be that the
// handler copies files to the index dir, then fails to copy a
// file and reverts the copy operation. On the next attempt, it
// will copy the same file again. There is nothing wrong with this
// in a real system, but it does violate write-once, and MDW
// doesn't like it. Disabling it means that we won't catch cases
// where the handler overwrites an existing index file, but
// there's nothing currently we can do about it, unless we don't
// use MDW.
handlerIndexDir.setPreventDoubleWrite(false);
handlerTaxoDir.setPreventDoubleWrite(false);
// wrap sourceDirFactory to return a MockDirWrapper so we can simulate errors
final SourceDirectoryFactory in = sourceDirFactory;
final AtomicInteger failures = new AtomicInteger(atLeast(10));
sourceDirFactory = new SourceDirectoryFactory() {
private long clientMaxSize = 100, handlerIndexMaxSize = 100, handlerTaxoMaxSize = 100;
private double clientExRate = 1.0, handlerIndexExRate = 1.0, handlerTaxoExRate = 1.0;
@Override
public void cleanupSession(String sessionID) throws IOException {
in.cleanupSession(sessionID);
}
@SuppressWarnings("synthetic-access")
@Override
public Directory getDirectory(String sessionID, String source) throws IOException {
Directory dir = in.getDirectory(sessionID, source);
if (random().nextBoolean() && failures.get() > 0) { // client should fail, return wrapped dir
MockDirectoryWrapper mdw = new MockDirectoryWrapper(random(), dir);
mdw.setRandomIOExceptionRateOnOpen(clientExRate);
mdw.setMaxSizeInBytes(clientMaxSize);
mdw.setRandomIOExceptionRate(clientExRate);
mdw.setCheckIndexOnClose(false);
clientMaxSize *= 2;
clientExRate /= 2;
return mdw;
}
if (failures.get() > 0 && random().nextBoolean()) { // handler should fail
if (random().nextBoolean()) { // index dir fail
handlerIndexDir.setMaxSizeInBytes(handlerIndexMaxSize);
handlerIndexDir.setRandomIOExceptionRate(handlerIndexExRate);
handlerIndexDir.setRandomIOExceptionRateOnOpen(handlerIndexExRate);
handlerIndexMaxSize *= 2;
handlerIndexExRate /= 2;
} else { // taxo dir fail
handlerTaxoDir.setMaxSizeInBytes(handlerTaxoMaxSize);
handlerTaxoDir.setRandomIOExceptionRate(handlerTaxoExRate);
handlerTaxoDir.setRandomIOExceptionRateOnOpen(handlerTaxoExRate);
handlerTaxoDir.setCheckIndexOnClose(false);
handlerTaxoMaxSize *= 2;
handlerTaxoExRate /= 2;
}
} else {
// disable all errors
handlerIndexDir.setMaxSizeInBytes(0);
handlerIndexDir.setRandomIOExceptionRate(0.0);
handlerIndexDir.setRandomIOExceptionRateOnOpen(0.0);
handlerTaxoDir.setMaxSizeInBytes(0);
handlerTaxoDir.setRandomIOExceptionRate(0.0);
handlerTaxoDir.setRandomIOExceptionRateOnOpen(0.0);
}
return dir;
}
};
handler = new IndexAndTaxonomyReplicationHandler(handlerIndexDir, handlerTaxoDir, new Callable<Boolean>() {
@Override
public Boolean call() throws Exception {
if (random().nextDouble() < 0.2 && failures.get() > 0) {
throw new RuntimeException("random exception from callback");
}
return null;
}
});
// wrap handleUpdateException so we can act on the thrown exception
client = new ReplicationClient(replicator, handler, sourceDirFactory) {
@SuppressWarnings("synthetic-access")
@Override
protected void handleUpdateException(Throwable t) {
if (t instanceof IOException) {
try {
if (VERBOSE) {
System.out.println("hit exception during update: " + t);
t.printStackTrace(System.out);
}
// test that the index can be read and also some basic statistics
DirectoryReader reader = DirectoryReader.open(handlerIndexDir.getDelegate());
try {
int numDocs = reader.numDocs();
int version = Integer.parseInt(reader.getIndexCommit().getUserData().get(VERSION_ID), 16);
assertEquals(numDocs, version);
} finally {
reader.close();
}
// verify index is fully consistent
_TestUtil.checkIndex(handlerIndexDir.getDelegate());
// verify taxonomy index is fully consistent (since we only add one
// category to all documents, there's nothing much more to validate
_TestUtil.checkIndex(handlerTaxoDir.getDelegate());
} catch (IOException e) {
throw new RuntimeException(e);
} finally {
// count-down number of failures
failures.decrementAndGet();
assert failures.get() >= 0 : "handler failed too many times: " + failures.get();
if (VERBOSE) {
if (failures.get() == 0) {
System.out.println("no more failures expected");
} else {
System.out.println("num failures left: " + failures.get());
}
}
}
} else {
if (t instanceof RuntimeException) throw (RuntimeException) t;
throw new RuntimeException(t);
}
}
};
client.startUpdateThread(10, "indexAndTaxo");
final Directory baseHandlerIndexDir = handlerIndexDir.getDelegate();
int numRevisions = atLeast(20) + 2;
for (int i = 2; i < numRevisions; i++) {
replicator.publish(createRevision(i));
assertHandlerRevision(i, baseHandlerIndexDir);
}
// disable errors -- maybe randomness didn't exhaust all allowed failures,
// and we don't want e.g. CheckIndex to hit false errors.
handlerIndexDir.setMaxSizeInBytes(0);
handlerIndexDir.setRandomIOExceptionRate(0.0);
handlerIndexDir.setRandomIOExceptionRateOnOpen(0.0);
handlerTaxoDir.setMaxSizeInBytes(0);
handlerTaxoDir.setRandomIOExceptionRate(0.0);
handlerTaxoDir.setRandomIOExceptionRateOnOpen(0.0);
}
}

170
lucene/replicator/src/test/org/apache/lucene/replicator/IndexAndTaxonomyRevisionTest.java Executable file
View File

@ -0,0 +1,170 @@
package org.apache.lucene.replicator;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.IOException;
import java.io.InputStream;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import org.apache.lucene.document.Document;
import org.apache.lucene.facet.index.FacetFields;
import org.apache.lucene.facet.taxonomy.CategoryPath;
import org.apache.lucene.facet.taxonomy.TaxonomyWriter;
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.SnapshotDeletionPolicy;
import org.apache.lucene.replicator.IndexAndTaxonomyRevision.SnapshotDirectoryTaxonomyWriter;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.util.IOUtils;
import org.junit.Test;
public class IndexAndTaxonomyRevisionTest extends ReplicatorTestCase {
private Document newDocument(TaxonomyWriter taxoWriter) throws IOException {
Document doc = new Document();
FacetFields ff = new FacetFields(taxoWriter);
ff.addFields(doc, Collections.singleton(new CategoryPath("A")));
return doc;
}
@Test
public void testNoCommit() throws Exception {
Directory indexDir = newDirectory();
IndexWriterConfig conf = new IndexWriterConfig(TEST_VERSION_CURRENT, null);
conf.setIndexDeletionPolicy(new SnapshotDeletionPolicy(conf.getIndexDeletionPolicy()));
IndexWriter indexWriter = new IndexWriter(indexDir, conf);
Directory taxoDir = newDirectory();
SnapshotDirectoryTaxonomyWriter taxoWriter = new SnapshotDirectoryTaxonomyWriter(taxoDir);
try {
assertNotNull(new IndexAndTaxonomyRevision(indexWriter, taxoWriter));
fail("should have failed when there are no commits to snapshot");
} catch (IllegalStateException e) {
// expected
} finally {
IOUtils.close(indexWriter, taxoWriter, taxoDir, indexDir);
}
}
@Test
public void testRevisionRelease() throws Exception {
Directory indexDir = newDirectory();
IndexWriterConfig conf = new IndexWriterConfig(TEST_VERSION_CURRENT, null);
conf.setIndexDeletionPolicy(new SnapshotDeletionPolicy(conf.getIndexDeletionPolicy()));
IndexWriter indexWriter = new IndexWriter(indexDir, conf);
Directory taxoDir = newDirectory();
SnapshotDirectoryTaxonomyWriter taxoWriter = new SnapshotDirectoryTaxonomyWriter(taxoDir);
try {
indexWriter.addDocument(newDocument(taxoWriter));
indexWriter.commit();
taxoWriter.commit();
Revision rev1 = new IndexAndTaxonomyRevision(indexWriter, taxoWriter);
// releasing that revision should not delete the files
rev1.release();
assertTrue(indexDir.fileExists(IndexFileNames.SEGMENTS + "_1"));
assertTrue(taxoDir.fileExists(IndexFileNames.SEGMENTS + "_1"));
rev1 = new IndexAndTaxonomyRevision(indexWriter, taxoWriter); // create revision again, so the files are snapshotted
indexWriter.addDocument(newDocument(taxoWriter));
indexWriter.commit();
taxoWriter.commit();
assertNotNull(new IndexAndTaxonomyRevision(indexWriter, taxoWriter));
rev1.release(); // this release should trigger the delete of segments_1
assertFalse(indexDir.fileExists(IndexFileNames.SEGMENTS + "_1"));
} finally {
IOUtils.close(indexWriter, taxoWriter, taxoDir, indexDir);
}
}
@Test
public void testSegmentsFileLast() throws Exception {
Directory indexDir = newDirectory();
IndexWriterConfig conf = new IndexWriterConfig(TEST_VERSION_CURRENT, null);
conf.setIndexDeletionPolicy(new SnapshotDeletionPolicy(conf.getIndexDeletionPolicy()));
IndexWriter indexWriter = new IndexWriter(indexDir, conf);
Directory taxoDir = newDirectory();
SnapshotDirectoryTaxonomyWriter taxoWriter = new SnapshotDirectoryTaxonomyWriter(taxoDir);
try {
indexWriter.addDocument(newDocument(taxoWriter));
indexWriter.commit();
taxoWriter.commit();
Revision rev = new IndexAndTaxonomyRevision(indexWriter, taxoWriter);
Map<String,List<RevisionFile>> sourceFiles = rev.getSourceFiles();
assertEquals(2, sourceFiles.size());
for (List<RevisionFile> files : sourceFiles.values()) {
String lastFile = files.get(files.size() - 1).fileName;
assertTrue(lastFile.startsWith(IndexFileNames.SEGMENTS) && !lastFile.equals(IndexFileNames.SEGMENTS_GEN));
}
} finally {
IOUtils.close(indexWriter, taxoWriter, taxoDir, indexDir);
}
}
@Test
public void testOpen() throws Exception {
Directory indexDir = newDirectory();
IndexWriterConfig conf = new IndexWriterConfig(TEST_VERSION_CURRENT, null);
conf.setIndexDeletionPolicy(new SnapshotDeletionPolicy(conf.getIndexDeletionPolicy()));
IndexWriter indexWriter = new IndexWriter(indexDir, conf);
Directory taxoDir = newDirectory();
SnapshotDirectoryTaxonomyWriter taxoWriter = new SnapshotDirectoryTaxonomyWriter(taxoDir);
try {
indexWriter.addDocument(newDocument(taxoWriter));
indexWriter.commit();
taxoWriter.commit();
Revision rev = new IndexAndTaxonomyRevision(indexWriter, taxoWriter);
for (Entry<String,List<RevisionFile>> e : rev.getSourceFiles().entrySet()) {
String source = e.getKey();
Directory dir = source.equals(IndexAndTaxonomyRevision.INDEX_SOURCE) ? indexDir : taxoDir;
for (RevisionFile file : e.getValue()) {
IndexInput src = dir.openInput(file.fileName, IOContext.READONCE);
InputStream in = rev.open(source, file.fileName);
assertEquals(src.length(), in.available());
byte[] srcBytes = new byte[(int) src.length()];
byte[] inBytes = new byte[(int) src.length()];
int offset = 0;
if (random().nextBoolean()) {
int skip = random().nextInt(10);
if (skip >= src.length()) {
skip = 0;
}
in.skip(skip);
src.seek(skip);
offset = skip;
}
src.readBytes(srcBytes, offset, srcBytes.length - offset);
in.read(inBytes, offset, inBytes.length - offset);
assertArrayEquals(srcBytes, inBytes);
IOUtils.close(src, in);
}
}
} finally {
IOUtils.close(indexWriter, taxoWriter, taxoDir, indexDir);
}
}
}

347
lucene/replicator/src/test/org/apache/lucene/replicator/IndexReplicationClientTest.java Executable file
View File

@ -0,0 +1,347 @@
package org.apache.lucene.replicator;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.Closeable;
import java.io.IOException;
import java.util.HashMap;
import java.util.concurrent.Callable;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.SnapshotDeletionPolicy;
import org.apache.lucene.replicator.ReplicationClient.ReplicationHandler;
import org.apache.lucene.replicator.ReplicationClient.SourceDirectoryFactory;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.MockDirectoryWrapper;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.ThreadInterruptedException;
import org.apache.lucene.util._TestUtil;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
public class IndexReplicationClientTest extends ReplicatorTestCase {
private static class IndexReadyCallback implements Callable<Boolean>, Closeable {
private final Directory indexDir;
private DirectoryReader reader;
private long lastGeneration = -1;
public IndexReadyCallback(Directory indexDir) throws IOException {
this.indexDir = indexDir;
if (DirectoryReader.indexExists(indexDir)) {
reader = DirectoryReader.open(indexDir);
lastGeneration = reader.getIndexCommit().getGeneration();
}
}
@Override
public Boolean call() throws Exception {
if (reader == null) {
reader = DirectoryReader.open(indexDir);
lastGeneration = reader.getIndexCommit().getGeneration();
} else {
DirectoryReader newReader = DirectoryReader.openIfChanged(reader);
assertNotNull("should not have reached here if no changes were made to the index", newReader);
long newGeneration = newReader.getIndexCommit().getGeneration();
assertTrue("expected newer generation; current=" + lastGeneration + " new=" + newGeneration, newGeneration > lastGeneration);
reader.close();
reader = newReader;
lastGeneration = newGeneration;
_TestUtil.checkIndex(indexDir);
}
return null;
}
@Override
public void close() throws IOException {
IOUtils.close(reader);
}
}
private MockDirectoryWrapper publishDir, handlerDir;
private Replicator replicator;
private SourceDirectoryFactory sourceDirFactory;
private ReplicationClient client;
private ReplicationHandler handler;
private IndexWriter publishWriter;
private IndexReadyCallback callback;
private static final String VERSION_ID = "version";
private void assertHandlerRevision(int expectedID, Directory dir) throws IOException {
// loop as long as client is alive. test-framework will terminate us if
// there's a serious bug, e.g. client doesn't really update. otherwise,
// introducing timeouts is not good, can easily lead to false positives.
while (client.isUpdateThreadAlive()) {
// give client a chance to update
try {
Thread.sleep(100);
} catch (InterruptedException e) {
throw new ThreadInterruptedException(e);
}
try {
DirectoryReader reader = DirectoryReader.open(dir);
try {
int handlerID = Integer.parseInt(reader.getIndexCommit().getUserData().get(VERSION_ID), 16);
if (expectedID == handlerID) {
return;
} else if (VERBOSE) {
System.out.println("expectedID=" + expectedID + " actual=" + handlerID + " generation=" + reader.getIndexCommit().getGeneration());
}
} finally {
reader.close();
}
} catch (Exception e) {
// we can hit IndexNotFoundException or e.g. EOFException (on
// segments_N) because it is being copied at the same time it is read by
// DirectoryReader.open().
}
}
}
private Revision createRevision(final int id) throws IOException {
publishWriter.addDocument(new Document());
publishWriter.setCommitData(new HashMap<String, String>() {{
put(VERSION_ID, Integer.toString(id, 16));
}});
publishWriter.commit();
return new IndexRevision(publishWriter);
}
@Override
@Before
public void setUp() throws Exception {
super.setUp();
publishDir = newMockDirectory();
handlerDir = newMockDirectory();
sourceDirFactory = new PerSessionDirectoryFactory(_TestUtil.getTempDir("replicationClientTest"));
replicator = new LocalReplicator();
callback = new IndexReadyCallback(handlerDir);
handler = new IndexReplicationHandler(handlerDir, callback);
client = new ReplicationClient(replicator, handler, sourceDirFactory);
IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, null);
conf.setIndexDeletionPolicy(new SnapshotDeletionPolicy(conf.getIndexDeletionPolicy()));
publishWriter = new IndexWriter(publishDir, conf);
}
@After
@Override
public void tearDown() throws Exception {
IOUtils.close(client, callback, publishWriter, replicator, publishDir, handlerDir);
super.tearDown();
}
@Test
public void testNoUpdateThread() throws Exception {
assertNull("no version expected at start", handler.currentVersion());
// Callback validates the replicated index
replicator.publish(createRevision(1));
client.updateNow();
replicator.publish(createRevision(2));
client.updateNow();
// Publish two revisions without update, handler should be upgraded to latest
replicator.publish(createRevision(3));
replicator.publish(createRevision(4));
client.updateNow();
}
@Test
public void testUpdateThread() throws Exception {
client.startUpdateThread(10, "index");
replicator.publish(createRevision(1));
assertHandlerRevision(1, handlerDir);
replicator.publish(createRevision(2));
assertHandlerRevision(2, handlerDir);
// Publish two revisions without update, handler should be upgraded to latest
replicator.publish(createRevision(3));
replicator.publish(createRevision(4));
assertHandlerRevision(4, handlerDir);
}
@Test
public void testRestart() throws Exception {
replicator.publish(createRevision(1));
client.updateNow();
replicator.publish(createRevision(2));
client.updateNow();
client.stopUpdateThread();
client.close();
client = new ReplicationClient(replicator, handler, sourceDirFactory);
// Publish two revisions without update, handler should be upgraded to latest
replicator.publish(createRevision(3));
replicator.publish(createRevision(4));
client.updateNow();
}
/*
* This test verifies that the client and handler do not end up in a corrupt
* index if exceptions are thrown at any point during replication. Either when
* a client copies files from the server to the temporary space, or when the
* handler copies them to the index directory.
*/
@Test
public void testConsistencyOnExceptions() throws Exception {
// so the handler's index isn't empty
replicator.publish(createRevision(1));
client.updateNow();
client.close();
callback.close();
// Replicator violates write-once policy. It may be that the
// handler copies files to the index dir, then fails to copy a
// file and reverts the copy operation. On the next attempt, it
// will copy the same file again. There is nothing wrong with this
// in a real system, but it does violate write-once, and MDW
// doesn't like it. Disabling it means that we won't catch cases
// where the handler overwrites an existing index file, but
// there's nothing currently we can do about it, unless we don't
// use MDW.
handlerDir.setPreventDoubleWrite(false);
// wrap sourceDirFactory to return a MockDirWrapper so we can simulate errors
final SourceDirectoryFactory in = sourceDirFactory;
final AtomicInteger failures = new AtomicInteger(atLeast(10));
sourceDirFactory = new SourceDirectoryFactory() {
private long clientMaxSize = 100, handlerMaxSize = 100;
private double clientExRate = 1.0, handlerExRate = 1.0;
@Override
public void cleanupSession(String sessionID) throws IOException {
in.cleanupSession(sessionID);
}
@SuppressWarnings("synthetic-access")
@Override
public Directory getDirectory(String sessionID, String source) throws IOException {
Directory dir = in.getDirectory(sessionID, source);
if (random().nextBoolean() && failures.get() > 0) { // client should fail, return wrapped dir
MockDirectoryWrapper mdw = new MockDirectoryWrapper(random(), dir);
mdw.setRandomIOExceptionRateOnOpen(clientExRate);
mdw.setMaxSizeInBytes(clientMaxSize);
mdw.setRandomIOExceptionRate(clientExRate);
mdw.setCheckIndexOnClose(false);
clientMaxSize *= 2;
clientExRate /= 2;
return mdw;
}
if (failures.get() > 0 && random().nextBoolean()) { // handler should fail
handlerDir.setMaxSizeInBytes(handlerMaxSize);
handlerDir.setRandomIOExceptionRateOnOpen(handlerExRate);
handlerDir.setRandomIOExceptionRate(handlerExRate);
handlerMaxSize *= 2;
handlerExRate /= 2;
} else {
// disable errors
handlerDir.setMaxSizeInBytes(0);
handlerDir.setRandomIOExceptionRate(0.0);
handlerDir.setRandomIOExceptionRateOnOpen(0.0);
}
return dir;
}
};
handler = new IndexReplicationHandler(handlerDir, new Callable<Boolean>() {
@Override
public Boolean call() throws Exception {
if (random().nextDouble() < 0.2 && failures.get() > 0) {
throw new RuntimeException("random exception from callback");
}
return null;
}
});
// wrap handleUpdateException so we can act on the thrown exception
client = new ReplicationClient(replicator, handler, sourceDirFactory) {
@SuppressWarnings("synthetic-access")
@Override
protected void handleUpdateException(Throwable t) {
if (t instanceof IOException) {
if (VERBOSE) {
System.out.println("hit exception during update: " + t);
t.printStackTrace(System.out);
}
try {
// test that the index can be read and also some basic statistics
DirectoryReader reader = DirectoryReader.open(handlerDir.getDelegate());
try {
int numDocs = reader.numDocs();
int version = Integer.parseInt(reader.getIndexCommit().getUserData().get(VERSION_ID), 16);
assertEquals(numDocs, version);
} finally {
reader.close();
}
// verify index consistency
_TestUtil.checkIndex(handlerDir.getDelegate());
} catch (IOException e) {
// exceptions here are bad, don't ignore them
throw new RuntimeException(e);
} finally {
// count-down number of failures
failures.decrementAndGet();
assert failures.get() >= 0 : "handler failed too many times: " + failures.get();
if (VERBOSE) {
if (failures.get() == 0) {
System.out.println("no more failures expected");
} else {
System.out.println("num failures left: " + failures.get());
}
}
}
} else {
if (t instanceof RuntimeException) throw (RuntimeException) t;
throw new RuntimeException(t);
}
}
};
client.startUpdateThread(10, "index");
final Directory baseHandlerDir = handlerDir.getDelegate();
int numRevisions = atLeast(20);
for (int i = 2; i < numRevisions; i++) {
replicator.publish(createRevision(i));
assertHandlerRevision(i, baseHandlerDir);
}
// disable errors -- maybe randomness didn't exhaust all allowed failures,
// and we don't want e.g. CheckIndex to hit false errors.
handlerDir.setMaxSizeInBytes(0);
handlerDir.setRandomIOExceptionRate(0.0);
handlerDir.setRandomIOExceptionRateOnOpen(0.0);
}
}

155
lucene/replicator/src/test/org/apache/lucene/replicator/IndexRevisionTest.java Executable file
View File

@ -0,0 +1,155 @@
package org.apache.lucene.replicator;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.InputStream;
import java.util.List;
import java.util.Map;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.KeepOnlyLastCommitDeletionPolicy;
import org.apache.lucene.index.SnapshotDeletionPolicy;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.util.IOUtils;
import org.junit.Test;
public class IndexRevisionTest extends ReplicatorTestCase {
@Test
public void testNoSnapshotDeletionPolicy() throws Exception {
Directory dir = newDirectory();
IndexWriterConfig conf = new IndexWriterConfig(TEST_VERSION_CURRENT, null);
conf.setIndexDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy());
IndexWriter writer = new IndexWriter(dir, conf);
try {
assertNotNull(new IndexRevision(writer));
fail("should have failed when IndexDeletionPolicy is not Snapshot");
} catch (IllegalArgumentException e) {
// expected
} finally {
IOUtils.close(writer, dir);
}
}
@Test
public void testNoCommit() throws Exception {
Directory dir = newDirectory();
IndexWriterConfig conf = new IndexWriterConfig(TEST_VERSION_CURRENT, null);
conf.setIndexDeletionPolicy(new SnapshotDeletionPolicy(conf.getIndexDeletionPolicy()));
IndexWriter writer = new IndexWriter(dir, conf);
try {
assertNotNull(new IndexRevision(writer));
fail("should have failed when there are no commits to snapshot");
} catch (IllegalStateException e) {
// expected
} finally {
IOUtils.close(writer, dir);
}
}
@Test
public void testRevisionRelease() throws Exception {
Directory dir = newDirectory();
IndexWriterConfig conf = new IndexWriterConfig(TEST_VERSION_CURRENT, null);
conf.setIndexDeletionPolicy(new SnapshotDeletionPolicy(conf.getIndexDeletionPolicy()));
IndexWriter writer = new IndexWriter(dir, conf);
try {
writer.addDocument(new Document());
writer.commit();
Revision rev1 = new IndexRevision(writer);
// releasing that revision should not delete the files
rev1.release();
assertTrue(dir.fileExists(IndexFileNames.SEGMENTS + "_1"));
rev1 = new IndexRevision(writer); // create revision again, so the files are snapshotted
writer.addDocument(new Document());
writer.commit();
assertNotNull(new IndexRevision(writer));
rev1.release(); // this release should trigger the delete of segments_1
assertFalse(dir.fileExists(IndexFileNames.SEGMENTS + "_1"));
} finally {
IOUtils.close(writer, dir);
}
}
@Test
public void testSegmentsFileLast() throws Exception {
Directory dir = newDirectory();
IndexWriterConfig conf = new IndexWriterConfig(TEST_VERSION_CURRENT, null);
conf.setIndexDeletionPolicy(new SnapshotDeletionPolicy(conf.getIndexDeletionPolicy()));
IndexWriter writer = new IndexWriter(dir, conf);
try {
writer.addDocument(new Document());
writer.commit();
Revision rev = new IndexRevision(writer);
@SuppressWarnings("unchecked")
Map<String, List<RevisionFile>> sourceFiles = rev.getSourceFiles();
assertEquals(1, sourceFiles.size());
List<RevisionFile> files = sourceFiles.values().iterator().next();
String lastFile = files.get(files.size() - 1).fileName;
assertTrue(lastFile.startsWith(IndexFileNames.SEGMENTS) && !lastFile.equals(IndexFileNames.SEGMENTS_GEN));
} finally {
IOUtils.close(writer, dir);
}
}
@Test
public void testOpen() throws Exception {
Directory dir = newDirectory();
IndexWriterConfig conf = new IndexWriterConfig(TEST_VERSION_CURRENT, null);
conf.setIndexDeletionPolicy(new SnapshotDeletionPolicy(conf.getIndexDeletionPolicy()));
IndexWriter writer = new IndexWriter(dir, conf);
try {
writer.addDocument(new Document());
writer.commit();
Revision rev = new IndexRevision(writer);
@SuppressWarnings("unchecked")
Map<String, List<RevisionFile>> sourceFiles = rev.getSourceFiles();
String source = sourceFiles.keySet().iterator().next();
for (RevisionFile file : sourceFiles.values().iterator().next()) {
IndexInput src = dir.openInput(file.fileName, IOContext.READONCE);
InputStream in = rev.open(source, file.fileName);
assertEquals(src.length(), in.available());
byte[] srcBytes = new byte[(int) src.length()];
byte[] inBytes = new byte[(int) src.length()];
int offset = 0;
if (random().nextBoolean()) {
int skip = random().nextInt(10);
if (skip >= src.length()) {
skip = 0;
}
in.skip(skip);
src.seek(skip);
offset = skip;
}
src.readBytes(srcBytes, offset, srcBytes.length - offset);
in.read(inBytes, offset, inBytes.length - offset);
assertArrayEquals(srcBytes, inBytes);
IOUtils.close(src, in);
}
} finally {
IOUtils.close(writer, dir);
}
}
}

196
lucene/replicator/src/test/org/apache/lucene/replicator/LocalReplicatorTest.java Executable file
View File

@ -0,0 +1,196 @@
package org.apache.lucene.replicator;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.HashMap;
import java.util.List;
import java.util.Map.Entry;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.SnapshotDeletionPolicy;
import org.apache.lucene.store.AlreadyClosedException;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.IOUtils;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
public class LocalReplicatorTest extends ReplicatorTestCase {
private static final String VERSION_ID = "version";
private LocalReplicator replicator;
private Directory sourceDir;
private IndexWriter sourceWriter;
@Before
@Override
public void setUp() throws Exception {
super.setUp();
sourceDir = newDirectory();
IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, null);
conf.setIndexDeletionPolicy(new SnapshotDeletionPolicy(conf.getIndexDeletionPolicy()));
sourceWriter = new IndexWriter(sourceDir, conf);
replicator = new LocalReplicator();
}
@After
@Override
public void tearDown() throws Exception {
IOUtils.close(replicator, sourceWriter, sourceDir);
super.tearDown();
}
private Revision createRevision(final int id) throws IOException {
sourceWriter.addDocument(new Document());
sourceWriter.setCommitData(new HashMap<String, String>() {{
put(VERSION_ID, Integer.toString(id, 16));
}});
sourceWriter.commit();
return new IndexRevision(sourceWriter);
}
@Test
public void testCheckForUpdateNoRevisions() throws Exception {
assertNull(replicator.checkForUpdate(null));
}
@Test
public void testObtainFileAlreadyClosed() throws IOException {
replicator.publish(createRevision(1));
SessionToken res = replicator.checkForUpdate(null);
assertNotNull(res);
assertEquals(1, res.sourceFiles.size());
Entry<String,List<RevisionFile>> entry = res.sourceFiles.entrySet().iterator().next();
replicator.close();
try {
replicator.obtainFile(res.id, entry.getKey(), entry.getValue().get(0).fileName);
fail("should have failed on AlreadyClosedException");
} catch (AlreadyClosedException e) {
// expected
}
}
@Test
public void testPublishAlreadyClosed() throws IOException {
replicator.close();
try {
replicator.publish(createRevision(2));
fail("should have failed on AlreadyClosedException");
} catch (AlreadyClosedException e) {
// expected
}
}
@Test
public void testUpdateAlreadyClosed() throws IOException {
replicator.close();
try {
replicator.checkForUpdate(null);
fail("should have failed on AlreadyClosedException");
} catch (AlreadyClosedException e) {
// expected
}
}
@Test
public void testPublishSameRevision() throws IOException {
Revision rev = createRevision(1);
replicator.publish(rev);
SessionToken res = replicator.checkForUpdate(null);
assertNotNull(res);
assertEquals(rev.getVersion(), res.version);
replicator.release(res.id);
replicator.publish(new IndexRevision(sourceWriter));
res = replicator.checkForUpdate(res.version);
assertNull(res);
// now make sure that publishing same revision doesn't leave revisions
// "locked", i.e. that replicator releases revisions even when they are not
// kept
replicator.publish(createRevision(2));
assertEquals(1, DirectoryReader.listCommits(sourceDir).size());
}
@Test
public void testPublishOlderRev() throws IOException {
replicator.publish(createRevision(1));
Revision old = new IndexRevision(sourceWriter);
replicator.publish(createRevision(2));
try {
replicator.publish(old);
fail("should have failed to publish an older revision");
} catch (IllegalArgumentException e) {
// expected
}
assertEquals(1, DirectoryReader.listCommits(sourceDir).size());
}
@Test
public void testObtainMissingFile() throws IOException {
replicator.publish(createRevision(1));
SessionToken res = replicator.checkForUpdate(null);
try {
replicator.obtainFile(res.id, res.sourceFiles.keySet().iterator().next(), "madeUpFile");
fail("should have failed obtaining an unrecognized file");
} catch (FileNotFoundException e) {
// expected
}
}
@Test
public void testSessionExpiration() throws IOException, InterruptedException {
replicator.publish(createRevision(1));
SessionToken session = replicator.checkForUpdate(null);
replicator.setExpirationThreshold(5); // expire quickly
Thread.sleep(50); // sufficient for expiration
try {
replicator.obtainFile(session.id, session.sourceFiles.keySet().iterator().next(), session.sourceFiles.values().iterator().next().get(0).fileName);
fail("should have failed to obtain a file for an expired session");
} catch (SessionExpiredException e) {
// expected
}
}
@Test
public void testUpdateToLatest() throws IOException {
replicator.publish(createRevision(1));
Revision rev = createRevision(2);
replicator.publish(rev);
SessionToken res = replicator.checkForUpdate(null);
assertNotNull(res);
assertEquals(0, rev.compareTo(res.version));
}
@Test
public void testRevisionRelease() throws Exception {
replicator.publish(createRevision(1));
assertTrue(sourceDir.fileExists(IndexFileNames.SEGMENTS + "_1"));
replicator.publish(createRevision(2));
// now the files of revision 1 can be deleted
assertTrue(sourceDir.fileExists(IndexFileNames.SEGMENTS + "_2"));
assertFalse("segments_1 should not be found in index directory after revision is released", sourceDir.fileExists(IndexFileNames.SEGMENTS + "_1"));
}
}

118
lucene/replicator/src/test/org/apache/lucene/replicator/ReplicatorTestCase.java Executable file
View File

@ -0,0 +1,118 @@
package org.apache.lucene.replicator;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.net.SocketException;
import org.apache.http.conn.ClientConnectionManager;
import org.apache.http.impl.conn.PoolingClientConnectionManager;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.LuceneTestCase.SuppressCodecs;
import org.eclipse.jetty.server.Connector;
import org.eclipse.jetty.server.Handler;
import org.eclipse.jetty.server.Server;
import org.eclipse.jetty.util.thread.QueuedThreadPool;
import org.junit.AfterClass;
@SuppressCodecs("Lucene3x")
public class ReplicatorTestCase extends LuceneTestCase {
private static final int BASE_PORT = 7000;
// if a test calls newServer() multiple times, or some ports already failed,
// don't start from BASE_PORT again
private static int lastPortUsed = -1;
private static ClientConnectionManager clientConnectionManager;
@AfterClass
public static void afterClassReplicatorTestCase() throws Exception {
if (clientConnectionManager != null) {
clientConnectionManager.shutdown();
clientConnectionManager = null;
}
}
/**
* Returns a new {@link Server HTTP Server} instance. To obtain its port, use
* {@link #serverPort(Server)}.
*/
public static synchronized Server newHttpServer(Handler handler) throws Exception {
int port = lastPortUsed == -1 ? BASE_PORT : lastPortUsed + 1;
Server server = null;
while (true) {
try {
server = new Server(port);
server.setHandler(handler);
QueuedThreadPool threadPool = new QueuedThreadPool();
threadPool.setDaemon(true);
threadPool.setMaxIdleTimeMs(0);
server.setThreadPool(threadPool);
// this will test the port
server.start();
// if here, port is available
lastPortUsed = port;
return server;
} catch (SocketException e) {
stopHttpServer(server);
// this is ok, we'll try the next port until successful.
++port;
}
}
}
/**
* Returns a {@link Server}'s port. This method assumes that no
* {@link Connector}s were added to the Server besides the default one.
*/
public static int serverPort(Server httpServer) {
return httpServer.getConnectors()[0].getPort();
}
/**
* Stops the given HTTP Server instance. This method does its best to guarantee
* that no threads will be left running following this method.
*/
public static void stopHttpServer(Server httpServer) throws Exception {
httpServer.stop();
httpServer.join();
}
/**
* Returns a {@link ClientConnectionManager}.
* <p>
* <b>NOTE:</b> do not {@link ClientConnectionManager#shutdown()} this
* connection manager, it will be shutdown automatically after all tests have
* finished.
*/
public static synchronized ClientConnectionManager getClientConnectionManager() {
if (clientConnectionManager == null) {
PoolingClientConnectionManager ccm = new PoolingClientConnectionManager();
ccm.setDefaultMaxPerRoute(128);
ccm.setMaxTotal(128);
clientConnectionManager = ccm;
}
return clientConnectionManager;
}
}

64
lucene/replicator/src/test/org/apache/lucene/replicator/SessionTokenTest.java Executable file
View File

@ -0,0 +1,64 @@
package org.apache.lucene.replicator;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.IOException;
import java.util.List;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.SnapshotDeletionPolicy;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.IOUtils;
import org.junit.Test;
public class SessionTokenTest extends ReplicatorTestCase {
@Test
public void testSerialization() throws IOException {
Directory dir = newDirectory();
IndexWriterConfig conf = new IndexWriterConfig(TEST_VERSION_CURRENT, null);
conf.setIndexDeletionPolicy(new SnapshotDeletionPolicy(conf.getIndexDeletionPolicy()));
IndexWriter writer = new IndexWriter(dir, conf);
writer.addDocument(new Document());
writer.commit();
Revision rev = new IndexRevision(writer);
SessionToken session1 = new SessionToken("17", rev);
ByteArrayOutputStream baos = new ByteArrayOutputStream();
session1.serialize(new DataOutputStream(baos));
byte[] b = baos.toByteArray();
SessionToken session2 = new SessionToken(new DataInputStream(new ByteArrayInputStream(b)));
assertEquals(session1.id, session2.id);
assertEquals(session1.version, session2.version);
assertEquals(1, session2.sourceFiles.size());
assertEquals(session1.sourceFiles.size(), session2.sourceFiles.size());
assertEquals(session1.sourceFiles.keySet(), session2.sourceFiles.keySet());
List<RevisionFile> files1 = session1.sourceFiles.values().iterator().next();
List<RevisionFile> files2 = session2.sourceFiles.values().iterator().next();
assertEquals(files1, files2);
IOUtils.close(writer, dir);
}
}

120
lucene/replicator/src/test/org/apache/lucene/replicator/http/HttpReplicatorTest.java Executable file
View File

@ -0,0 +1,120 @@
package org.apache.lucene.replicator.http;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.File;
import java.io.IOException;
import java.util.Collections;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.SnapshotDeletionPolicy;
import org.apache.lucene.replicator.IndexReplicationHandler;
import org.apache.lucene.replicator.IndexRevision;
import org.apache.lucene.replicator.LocalReplicator;
import org.apache.lucene.replicator.PerSessionDirectoryFactory;
import org.apache.lucene.replicator.ReplicationClient;
import org.apache.lucene.replicator.Replicator;
import org.apache.lucene.replicator.ReplicatorTestCase;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util._TestUtil;
import org.eclipse.jetty.server.Server;
import org.eclipse.jetty.servlet.ServletHandler;
import org.eclipse.jetty.servlet.ServletHolder;
import org.junit.Before;
import org.junit.Test;
public class HttpReplicatorTest extends ReplicatorTestCase {
private File clientWorkDir;
private Replicator serverReplicator;
private IndexWriter writer;
private DirectoryReader reader;
private Server server;
private int port;
private Directory serverIndexDir, handlerIndexDir;
private void startServer() throws Exception {
ServletHandler replicationHandler = new ServletHandler();
ReplicationService service = new ReplicationService(Collections.singletonMap("s1", serverReplicator));
ServletHolder servlet = new ServletHolder(new ReplicationServlet(service));
replicationHandler.addServletWithMapping(servlet, ReplicationService.REPLICATION_CONTEXT + "/*");
server = newHttpServer(replicationHandler);
port = serverPort(server);
}
@Before
@Override
public void setUp() throws Exception {
super.setUp();
clientWorkDir = _TestUtil.getTempDir("httpReplicatorTest");
handlerIndexDir = newDirectory();
serverIndexDir = newDirectory();
serverReplicator = new LocalReplicator();
startServer();
IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, null);
conf.setIndexDeletionPolicy(new SnapshotDeletionPolicy(conf.getIndexDeletionPolicy()));
writer = new IndexWriter(serverIndexDir, conf);
reader = DirectoryReader.open(writer, false);
}
@Override
public void tearDown() throws Exception {
stopHttpServer(server);
IOUtils.close(reader, writer, handlerIndexDir, serverIndexDir);
super.tearDown();
}
private void publishRevision(int id) throws IOException {
Document doc = new Document();
writer.addDocument(doc);
writer.setCommitData(Collections.singletonMap("ID", Integer.toString(id, 16)));
writer.commit();
serverReplicator.publish(new IndexRevision(writer));
}
private void reopenReader() throws IOException {
DirectoryReader newReader = DirectoryReader.openIfChanged(reader);
assertNotNull(newReader);
reader.close();
reader = newReader;
}
@Test
public void testBasic() throws Exception {
Replicator replicator = new HttpReplicator("localhost", port, ReplicationService.REPLICATION_CONTEXT + "/s1",
getClientConnectionManager());
ReplicationClient client = new ReplicationClient(replicator, new IndexReplicationHandler(handlerIndexDir, null),
new PerSessionDirectoryFactory(clientWorkDir));
publishRevision(1);
client.updateNow();
reopenReader();
assertEquals(1, Integer.parseInt(reader.getIndexCommit().getUserData().get("ID"), 16));
publishRevision(2);
client.updateNow();
reopenReader();
assertEquals(2, Integer.parseInt(reader.getIndexCommit().getUserData().get("ID"), 16));
}
}

41
lucene/replicator/src/test/org/apache/lucene/replicator/http/ReplicationServlet.java Executable file
View File

@ -0,0 +1,41 @@
package org.apache.lucene.replicator.http;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.IOException;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
public class ReplicationServlet extends HttpServlet {
private final ReplicationService service;
public ReplicationServlet(ReplicationService service) {
super();
this.service = service;
}
@Override
protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException {
service.perform(req, resp);
}
}

8
lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java
View File

@ -372,7 +372,7 @@ public class MockDirectoryWrapper extends BaseDirectoryWrapper {
System.out.println(Thread.currentThread().getName() + ": MockDirectoryWrapper: now throw random exception" + (message == null ? "" : " (" + message + ")"));
new Throwable().printStackTrace(System.out);
}
throw new IOException("a random IOException" + (message == null ? "" : "(" + message + ")"));
throw new IOException("a random IOException" + (message == null ? "" : " (" + message + ")"));
}
}
@ -383,9 +383,9 @@ public class MockDirectoryWrapper extends BaseDirectoryWrapper {
new Throwable().printStackTrace(System.out);
}
if (randomState.nextBoolean()) {
throw new IOException("a random IOException");
throw new IOException("a random IOException (" + name + ")");
} else {
throw new FileNotFoundException("a random IOException");
throw new FileNotFoundException("a random IOException (" + name + ")");
}
}
}
@ -505,7 +505,7 @@ public class MockDirectoryWrapper extends BaseDirectoryWrapper {
if (throttling == Throttling.ALWAYS ||
(throttling == Throttling.SOMETIMES && randomState.nextInt(50) == 0) && !(delegate instanceof RateLimitedDirectoryWrapper)) {
if (LuceneTestCase.VERBOSE) {
System.out.println("MockDirectoryWrapper: throttling indexOutput");
System.out.println("MockDirectoryWrapper: throttling indexOutput (" + name + ")");
}
return throttledOutput.newFromDelegate(io);
} else {