USRealEstate
/
Rets-Io
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Init submit the change from fork and documents

This commit is contained in:
YuCheng Hu 2019-09-13 10:22:08 -04:00
parent 8a29cfe906
commit cafbafdbfd
606 changed files with 47858 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
build.gradle
View File

@ -1,14 +1,17 @@
plugins {
id 'maven-publish'
id "org.asciidoctor.convert" version "2.3.0"
}
apply plugin: 'java'
version = '1.0.0'
group = 'com.ossez.reoc'
version = '0.1.0'
allprojects {
repositories {
maven { url "https://maven.ossez.com/repository/internal" }
mavenLocal()
}
ext {
@ -54,7 +57,7 @@ task prepareAsciidocBuildZH(type: Sync) {
}
task('makePDFZH', type: org.asciidoctor.gradle.AsciidoctorTask){
task('makePDFZH', type: org.asciidoctor.gradle.AsciidoctorTask) {
dependsOn prepareAsciidocBuildZH
backends 'pdf'
sourceDir "$buildDir/asciidoc/assemble"
@ -67,12 +70,11 @@ task('makePDFZH', type: org.asciidoctor.gradle.AsciidoctorTask){
'sectanchors': '',
'sectnums': '',
'toc': '',
'source-highlighter' : 'coderay',
'source-highlighter': 'coderay',
revnumber: project.version
}
asciidoctor {
dependsOn makePDFZH
backends 'html5'
@ -101,5 +103,22 @@ asciidoctor {
'allow-uri-read': '',
revnumber: project.version
}
task sourcesJar(type: Jar) {
from sourceSets.main.allJava
archiveClassifier = 'sources'
}
task javadocJar(type: Jar) {
from javadoc
archiveClassifier = 'javadoc'
}
publishing {
publications {
maven(MavenPublication) {
from components.java
artifact sourcesJar
artifact javadocJar
}
}
}

132
docs/asciidoc/appendix.adoc Normal file
View File

@ -0,0 +1,132 @@
:batch-asciidoc: ./
:toc: left
:toclevels: 4
[[listOfReadersAndWriters]]
[appendix]
== List of ItemReaders and ItemWriters
[[itemReadersAppendix]]
=== Item Readers
.Available Item Readers
[options="header"]
|===============
|Item Reader|Description
|AbstractItemCountingItemStreamItemReader|Abstract base class that provides basic
restart capabilities by counting the number of items returned from
an `ItemReader`.
|AggregateItemReader|An `ItemReader` that delivers a list as its
item, storing up objects from the injected `ItemReader` until they
are ready to be packed out as a collection. This `ItemReader` should
mark the beginning and end of records with the constant values in
`FieldSetMapper AggregateItemReader#__$$BEGIN_RECORD$$__` and
`AggregateItemReader#__$$END_RECORD$$__`.
|AmqpItemReader|Given a Spring `AmqpTemplate`, it provides
synchronous receive methods. The `receiveAndConvert()` method
lets you receive POJO objects.
|KafkaItemReader|An `ItemReader` that reads messages from an Apache Kafka topic.
It can be configured to read messages from multiple partitions of the same topic.
This reader stores message offsets in the execution context to support restart capabilities.
|FlatFileItemReader|Reads from a flat file. Includes `ItemStream`
and `Skippable` functionality. See link:readersAndWriters.html#flatFileItemReader[`FlatFileItemReader`].
|HibernateCursorItemReader|Reads from a cursor based on an HQL query. See
link:readersAndWriters.html#cursorBasedItemReaders[`Cursor-based ItemReaders`].
|HibernatePagingItemReader|Reads from a paginated HQL query
|ItemReaderAdapter|Adapts any class to the
`ItemReader` interface.
|JdbcCursorItemReader|Reads from a database cursor via JDBC. See
link:readersAndWriters.html#cursorBasedItemReaders[`Cursor-based ItemReaders`].
|JdbcPagingItemReader|Given an SQL statement, pages through the rows,
such that large datasets can be read without running out of
memory.
|JmsItemReader|Given a Spring `JmsOperations` object and a JMS
Destination or destination name to which to send errors, provides items
received through the injected `JmsOperations#receive()`
method.
|JpaPagingItemReader|Given a JPQL statement, pages through the
rows, such that large datasets can be read without running out of
memory.
|ListItemReader|Provides the items from a list, one at a
time.
|MongoItemReader|Given a `MongoOperations` object and a JSON-based MongoDB
query, provides items received from the `MongoOperations#find()` method.
|Neo4jItemReader|Given a `Neo4jOperations` object and the components of a
Cyhper query, items are returned as the result of the Neo4jOperations.query
method.
|RepositoryItemReader|Given a Spring Data `PagingAndSortingRepository` object,
a `Sort`, and the name of method to execute, returns items provided by the
Spring Data repository implementation.
|StoredProcedureItemReader|Reads from a database cursor resulting from the
execution of a database stored procedure. See link:readersAndWriters.html#StoredProcedureItemReader[`StoredProcedureItemReader`]
|StaxEventItemReader|Reads via StAX. see link:readersAndWriters.html#StaxEventItemReader[`StaxEventItemReader`].
|JsonItemReader|Reads items from a Json document. see link:readersAndWriters.html#JsonItemReader[`JsonItemReader`].
|===============
[[itemWritersAppendix]]
=== Item Writers
.Available Item Writers
[options="header"]
|===============
|Item Writer|Description
|AbstractItemStreamItemWriter|Abstract base class that combines the
`ItemStream` and
`ItemWriter` interfaces.
|AmqpItemWriter|Given a Spring `AmqpTemplate`, it provides
for a synchronous `send` method. The `convertAndSend(Object)`
method lets you send POJO objects.
|CompositeItemWriter|Passes an item to the `write` method of each
in an injected `List` of `ItemWriter` objects.
|FlatFileItemWriter|Writes to a flat file. Includes `ItemStream` and
Skippable functionality. See link:readersAndWriters.html#flatFileItemWriter[`FlatFileItemWriter`].
|GemfireItemWriter|Using a `GemfireOperations` object, items are either written
or removed from the Gemfire instance based on the configuration of the delete
flag.
|HibernateItemWriter|This item writer is Hibernate-session aware
and handles some transaction-related work that a non-"hibernate-aware"
item writer would not need to know about and then delegates
to another item writer to do the actual writing.
|ItemWriterAdapter|Adapts any class to the
`ItemWriter` interface.
|JdbcBatchItemWriter|Uses batching features from a
`PreparedStatement`, if available, and can
take rudimentary steps to locate a failure during a
`flush`.
|JmsItemWriter|Using a `JmsOperations` object, items are written
to the default queue through the `JmsOperations#convertAndSend()` method.
|JpaItemWriter|This item writer is JPA EntityManager-aware
and handles some transaction-related work that a non-"JPA-aware"
`ItemWriter` would not need to know about and
then delegates to another writer to do the actual writing.
|KafkaItemWriter|Using a `KafkaTemplate` object, items are written to the default topic through the
`KafkaTemplate#sendDefault(Object, Object)` method using a `Converter` to map the key from the item.
A delete flag can also be configured to send delete events to the topic.
|MimeMessageItemWriter|Using Spring's `JavaMailSender`, items of type `MimeMessage`
are sent as mail messages.
|MongoItemWriter|Given a `MongoOperations` object, items are written
through the `MongoOperations.save(Object)` method. The actual write is delayed
until the last possible moment before the transaction commits.
|Neo4jItemWriter|Given a `Neo4jOperations` object, items are persisted through the
`save(Object)` method or deleted through the `delete(Object)` per the
`ItemWriter's` configuration
|PropertyExtractingDelegatingItemWriter|Extends `AbstractMethodInvokingDelegator`
creating arguments on the fly. Arguments are created by retrieving
the values from the fields in the item to be processed (through a
`SpringBeanWrapper`), based on an injected array of field
names.
|RepositoryItemWriter|Given a Spring Data `CrudRepository` implementation,
items are saved through the method specified in the configuration.
|StaxEventItemWriter|Uses a `Marshaller` implementation to
convert each item to XML and then writes it to an XML file using
StAX.
|JsonFileItemWriter|Uses a `JsonObjectMarshaller` implementation to
convert each item to Json and then writes it to an Json file.
|===============

727
docs/asciidoc/common-patterns.adoc Normal file
View File

@ -0,0 +1,727 @@
:batch-asciidoc: ./
:toc: left
:toclevels: 4
[[commonPatterns]]
== Common Batch Patterns
ifndef::onlyonetoggle[]
include::toggle.adoc[]
endif::onlyonetoggle[]
Some batch jobs can be assembled purely from off-the-shelf components in Spring Batch.
For instance, the `ItemReader` and `ItemWriter` implementations can be configured to
cover a wide range of scenarios. However, for the majority of cases, custom code must be
written. The main API entry points for application developers are the `Tasklet`, the
`ItemReader`, the `ItemWriter`, and the various listener interfaces. Most simple batch
jobs can use off-the-shelf input from a Spring Batch `ItemReader`, but it is often the
case that there are custom concerns in the processing and writing that require developers
to implement an `ItemWriter` or `ItemProcessor`.
In this chapter, we provide a few examples of common patterns in custom business logic.
These examples primarily feature the listener interfaces. It should be noted that an
`ItemReader` or `ItemWriter` can implement a listener interface as well, if appropriate.
[[loggingItemProcessingAndFailures]]
=== Logging Item Processing and Failures
A common use case is the need for special handling of errors in a step, item by item,
perhaps logging to a special channel or inserting a record into a database. A
chunk-oriented `Step` (created from the step factory beans) lets users implement this use
case with a simple `ItemReadListener` for errors on `read` and an `ItemWriteListener` for
errors on `write`. The following code snippet illustrates a listener that logs both read
and write failures:
[source, java]
----
public class ItemFailureLoggerListener extends ItemListenerSupport {
private static Log logger = LogFactory.getLog("item.error");
public void onReadError(Exception ex) {
logger.error("Encountered error on read", e);
}
public void onWriteError(Exception ex, List<? extends Object> items) {
logger.error("Encountered error on write", ex);
}
}
----
Having implemented this listener, it must be registered with a step, as shown in the
following example:
.XML Configuration
[source, xml, role="xmlContent"]
----
<step id="simpleStep">
...
<listeners>
<listener>
<bean class="org.example...ItemFailureLoggerListener"/>
</listener>
</listeners>
</step>
----
.Java Configuration
[source, java, role="javaContent"]
----
@Bean
public Step simpleStep() {
return this.stepBuilderFactory.get("simpleStep")
...
.listener(new ItemFailureLoggerListener())
.build();
}
----
IMPORTANT: if your listener does anything in an `onError()` method, it must be inside
a transaction that is going to be rolled back. If you need to use a transactional
resource, such as a database, inside an `onError()` method, consider adding a declarative
transaction to that method (see Spring Core Reference Guide for details), and giving its
propagation attribute a value of `REQUIRES_NEW`.
[[stoppingAJobManuallyForBusinessReasons]]
=== Stopping a Job Manually for Business Reasons
Spring Batch provides a `stop()` method through the `JobLauncher` interface, but this is
really for use by the operator rather than the application programmer. Sometimes, it is
more convenient or makes more sense to stop a job execution from within the business
logic.
The simplest thing to do is to throw a `RuntimeException` (one that is neither retried
indefinitely nor skipped). For example, a custom exception type could be used, as shown
in the following example:
[source, java]
----
public class PoisonPillItemProcessor<T> implements ItemProcessor<T, T> {
@Override
public T process(T item) throws Exception {
if (isPoisonPill(item)) {
throw new PoisonPillException("Poison pill detected: " + item);
}
return item;
}
}
----
Another simple way to stop a step from executing is to return `null` from the
`ItemReader`, as shown in the following example:
[source, java]
----
public class EarlyCompletionItemReader implements ItemReader<T> {
private ItemReader<T> delegate;
public void setDelegate(ItemReader<T> delegate) { ... }
public T read() throws Exception {
T item = delegate.read();
if (isEndItem(item)) {
return null; // end the step here
}
return item;
}
}
----
The previous example actually relies on the fact that there is a default implementation
of the `CompletionPolicy` strategy that signals a complete batch when the item to be
processed is `null`. A more sophisticated completion policy could be implemented and
injected into the `Step` through the `SimpleStepFactoryBean`, as shown in the following
example:
.XML Configuration
[source, xml, role="xmlContent"]
----
<step id="simpleStep">
<tasklet>
<chunk reader="reader" writer="writer" commit-interval="10"
chunk-completion-policy="completionPolicy"/>
</tasklet>
</step>
<bean id="completionPolicy" class="org.example...SpecialCompletionPolicy"/>
----
.Java Configuration
[source, java, role="javaContent"]
----
@Bean
public Step simpleStep() {
return this.stepBuilderFactory.get("simpleStep")
.<String, String>chunk(new SpecialCompletionPolicy())
.reader(reader())
.writer(writer())
.build();
}
----
An alternative is to set a flag in the `StepExecution`, which is checked by the `Step`
implementations in the framework in between item processing. To implement this
alternative, we need access to the current `StepExecution`, and this can be achieved by
implementing a `StepListener` and registering it with the `Step`. The following example
shows a listener that sets the flag:
[source, java]
----
public class CustomItemWriter extends ItemListenerSupport implements StepListener {
private StepExecution stepExecution;
public void beforeStep(StepExecution stepExecution) {
this.stepExecution = stepExecution;
}
public void afterRead(Object item) {
if (isPoisonPill(item)) {
stepExecution.setTerminateOnly();
}
}
}
----
When the flag is set, the default behavior is for the step to throw a
`JobInterruptedException`. This behavior can be controlled through the
`StepInterruptionPolicy`. However, the only choice is to throw or not throw an exception,
so this is always an abnormal ending to a job.
[[addingAFooterRecord]]
=== Adding a Footer Record
Often, when writing to flat files, a "footer" record must be appended to the end of the
file, after all processing has be completed. This can be achieved using the
`FlatFileFooterCallback` interface provided by Spring Batch. The `FlatFileFooterCallback`
(and its counterpart, the `FlatFileHeaderCallback`) are optional properties of the
`FlatFileItemWriter` and can be added to an item writer as shown in the following
example:
.XML Configuration
[source, xml, role="xmlContent"]
----
<bean id="itemWriter" class="org.spr...FlatFileItemWriter">
<property name="resource" ref="outputResource" />
<property name="lineAggregator" ref="lineAggregator"/>
<property name="headerCallback" ref="headerCallback" />
<property name="footerCallback" ref="footerCallback" />
</bean>
----
.Java Configuration
[source, java, role="javaContent"]
----
@Bean
public FlatFileItemWriter<String> itemWriter(Resource outputResource) {
return new FlatFileItemWriterBuilder<String>()
.name("itemWriter")
.resource(outputResource)
.lineAggregator(lineAggregator())
.headerCallback(headerCallback())
.footerCallback(footerCallback())
.build();
}
----
The footer callback interface has just one method that is called when the footer must be
written, as shown in the following interface definition:
[source, java]
----
public interface FlatFileFooterCallback {
void writeFooter(Writer writer) throws IOException;
}
----
[[writingASummaryFooter]]
==== Writing a Summary Footer
A common requirement involving footer records is to aggregate information during the
output process and to append this information to the end of the file. This footer often
serves as a summarization of the file or provides a checksum.
For example, if a batch job is writing `Trade` records to a flat file, and there is a
requirement that the total amount from all the `Trades` is placed in a footer, then the
following `ItemWriter` implementation can be used:
[source, java]
----
public class TradeItemWriter implements ItemWriter<Trade>,
FlatFileFooterCallback {
private ItemWriter<Trade> delegate;
private BigDecimal totalAmount = BigDecimal.ZERO;
public void write(List<? extends Trade> items) throws Exception {
BigDecimal chunkTotal = BigDecimal.ZERO;
for (Trade trade : items) {
chunkTotal = chunkTotal.add(trade.getAmount());
}
delegate.write(items);
// After successfully writing all items
totalAmount = totalAmount.add(chunkTotal);
}
public void writeFooter(Writer writer) throws IOException {
writer.write("Total Amount Processed: " + totalAmount);
}
public void setDelegate(ItemWriter delegate) {...}
}
----
This `TradeItemWriter` stores a `totalAmount` value that is increased with the `amount`
from each `Trade` item written. After the last `Trade` is processed, the framework calls
`writeFooter`, which puts the `totalAmount` into the file. Note that the `write` method
makes use of a temporary variable, `chunkTotal`, that stores the total of the
`Trade` amounts in the chunk. This is done to ensure that, if a skip occurs in the
`write` method, the `totalAmount` is left unchanged. It is only at the end of the `write`
method, once we are guaranteed that no exceptions are thrown, that we update the
`totalAmount`.
In order for the `writeFooter` method to be called, the `TradeItemWriter` (which
implements `FlatFileFooterCallback`) must be wired into the `FlatFileItemWriter` as the
`footerCallback`. The following example shows how to do so:
.XML Configuration
[source, xml, role="xmlContent"]
----
<bean id="tradeItemWriter" class="..TradeItemWriter">
<property name="delegate" ref="flatFileItemWriter" />
</bean>
<bean id="flatFileItemWriter" class="org.spr...FlatFileItemWriter">
<property name="resource" ref="outputResource" />
<property name="lineAggregator" ref="lineAggregator"/>
<property name="footerCallback" ref="tradeItemWriter" />
</bean>
----
.Java Configuration
[source, java, role="javaContent"]
----
@Bean
public TradeItemWriter tradeItemWriter() {
TradeItemWriter itemWriter = new TradeItemWriter();
itemWriter.setDelegate(flatFileItemWriter(null));
return itemWriter;
}
@Bean
public FlatFileItemWriter<String> flatFileItemWriter(Resource outputResource) {
return new FlatFileItemWriterBuilder<String>()
.name("itemWriter")
.resource(outputResource)
.lineAggregator(lineAggregator())
.footerCallback(tradeItemWriter())
.build();
}
----
The way that the `TradeItemWriter` has been written so far functions correctly only if
the `Step` is not restartable. This is because the class is stateful (since it stores the
`totalAmount`), but the `totalAmount` is not persisted to the database. Therefore, it
cannot be retrieved in the event of a restart. In order to make this class restartable,
the `ItemStream` interface should be implemented along with the methods `open` and
`update`, as shown in the following example:
[source, java]
----
public void open(ExecutionContext executionContext) {
if (executionContext.containsKey("total.amount") {
totalAmount = (BigDecimal) executionContext.get("total.amount");
}
}
public void update(ExecutionContext executionContext) {
executionContext.put("total.amount", totalAmount);
}
----
The update method stores the most current version of `totalAmount` to the
`ExecutionContext` just before that object is persisted to the database. The open method
retrieves any existing `totalAmount` from the `ExecutionContext` and uses it as the
starting point for processing, allowing the `TradeItemWriter` to pick up on restart where
it left off the previous time the `Step` was run.
[[drivingQueryBasedItemReaders]]
=== Driving Query Based ItemReaders
In the link:readersAndWriters.html[chapter on readers and writers], database input using
paging was discussed. Many database vendors, such as DB2, have extremely pessimistic
locking strategies that can cause issues if the table being read also needs to be used by
other portions of the online application. Furthermore, opening cursors over extremely
large datasets can cause issues on databases from certain vendors. Therefore, many
projects prefer to use a 'Driving Query' approach to reading in data. This approach works
by iterating over keys, rather than the entire object that needs to be returned, as the
following image illustrates:
.Driving Query Job
image::{batch-asciidoc}images/drivingQueryExample.png[Driving Query Job, scaledwidth="60%"]
As you can see, the example shown in the preceding image uses the same 'FOO' table as was
used in the cursor-based example. However, rather than selecting the entire row, only the
IDs were selected in the SQL statement. So, rather than a `FOO` object being returned
from `read`, an `Integer` is returned. This number can then be used to query for the
'details', which is a complete `Foo` object, as shown in the following image:
.Driving Query Example
image::{batch-asciidoc}images/drivingQueryJob.png[Driving Query Example, scaledwidth="60%"]
An `ItemProcessor` should be used to transform the key obtained from the driving query
into a full 'Foo' object. An existing DAO can be used to query for the full object based
on the key.
[[multiLineRecords]]
=== Multi-Line Records
While it is usually the case with flat files that each record is confined to a single
line, it is common that a file might have records spanning multiple lines with multiple
formats. The following excerpt from a file shows an example of such an arrangement:
----
HEA;0013100345;2007-02-15
NCU;Smith;Peter;;T;20014539;F
BAD;;Oak Street 31/A;;Small Town;00235;IL;US
FOT;2;2;267.34
----
Everything between the line starting with 'HEA' and the line starting with 'FOT' is
considered one record. There are a few considerations that must be made in order to
handle this situation correctly:
* Instead of reading one record at a time, the `ItemReader` must read every line of the
multi-line record as a group, so that it can be passed to the `ItemWriter` intact.
* Each line type may need to be tokenized differently.
Because a single record spans multiple lines and because we may not know how many lines
there are, the `ItemReader` must be careful to always read an entire record. In order to
do this, a custom `ItemReader` should be implemented as a wrapper for the
`FlatFileItemReader`, as shown in the following example:
.XML Configuration
[source, xml, role="xmlContent"]
----
<bean id="itemReader" class="org.spr...MultiLineTradeItemReader">
<property name="delegate">
<bean class="org.springframework.batch.item.file.FlatFileItemReader">
<property name="resource" value="data/iosample/input/multiLine.txt" />
<property name="lineMapper">
<bean class="org.spr...DefaultLineMapper">
<property name="lineTokenizer" ref="orderFileTokenizer"/>
<property name="fieldSetMapper" ref="orderFieldSetMapper"/>
</bean>
</property>
</bean>
</property>
</bean>
----
.Java Configuration
[source, java, role="javaContent"]
----
@Bean
public MultiLineTradeItemReader itemReader() {
MultiLineTradeItemReader itemReader = new MultiLineTradeItemReader();
itemReader.setDelegate(flatFileItemReader());
return itemReader;
}
@Bean
public FlatFileItemReader flatFileItemReader() {
FlatFileItemReader<Trade> reader = new FlatFileItemReaderBuilder<>()
.name("flatFileItemReader")
.resource(new ClassPathResource("data/iosample/input/multiLine.txt"))
.lineTokenizer(orderFileTokenizer())
.fieldSetMapper(orderFieldSetMapper())
.build();
return reader;
}
----
To ensure that each line is tokenized properly, which is especially important for
fixed-length input, the `PatternMatchingCompositeLineTokenizer` can be used on the
delegate `FlatFileItemReader`. See
link:readersAndWriters.html#flatFileItemReader[`FlatFileItemReader` in the Readers and
Writers chapter] for more details. The delegate reader then uses a
`PassThroughFieldSetMapper` to deliver a `FieldSet` for each line back to the wrapping
`ItemReader`, as shown in the following example:
.XML Content
[source, xml, role="xmlContent"]
----
<bean id="orderFileTokenizer" class="org.spr...PatternMatchingCompositeLineTokenizer">
<property name="tokenizers">
<map>
<entry key="HEA*" value-ref="headerRecordTokenizer" />
<entry key="FOT*" value-ref="footerRecordTokenizer" />
<entry key="NCU*" value-ref="customerLineTokenizer" />
<entry key="BAD*" value-ref="billingAddressLineTokenizer" />
</map>
</property>
</bean>
----
.Java Content
[source, java, role="javaContent"]
----
@Bean
public PatternMatchingCompositeLineTokenizer orderFileTokenizer() {
PatternMatchingCompositeLineTokenizer tokenizer =
new PatternMatchingCompositeLineTokenizer();
Map<String, LineTokenizer> tokenizers = new HashMap<>(4);
tokenizers.put("HEA*", headerRecordTokenizer());
tokenizers.put("FOT*", footerRecordTokenizer());
tokenizers.put("NCU*", customerLineTokenizer());
tokenizers.put("BAD*", billingAddressLineTokenizer());
tokenizer.setTokenizers(tokenizers);
return tokenizer;
}
----
This wrapper has to be able to recognize the end of a record so that it can continually
call `read()` on its delegate until the end is reached. For each line that is read, the
wrapper should build up the item to be returned. Once the footer is reached, the item can
be returned for delivery to the `ItemProcessor` and `ItemWriter`, as shown in the
following example:
[source, java]
----
private FlatFileItemReader<FieldSet> delegate;
public Trade read() throws Exception {
Trade t = null;
for (FieldSet line = null; (line = this.delegate.read()) != null;) {
String prefix = line.readString(0);
if (prefix.equals("HEA")) {
t = new Trade(); // Record must start with header
}
else if (prefix.equals("NCU")) {
Assert.notNull(t, "No header was found.");
t.setLast(line.readString(1));
t.setFirst(line.readString(2));
...
}
else if (prefix.equals("BAD")) {
Assert.notNull(t, "No header was found.");
t.setCity(line.readString(4));
t.setState(line.readString(6));
...
}
else if (prefix.equals("FOT")) {
return t; // Record must end with footer
}
}
Assert.isNull(t, "No 'END' was found.");
return null;
}
----
[[executingSystemCommands]]
=== Executing System Commands
Many batch jobs require that an external command be called from within the batch job.
Such a process could be kicked off separately by the scheduler, but the advantage of
common metadata about the run would be lost. Furthermore, a multi-step job would also
need to be split up into multiple jobs as well.
Because the need is so common, Spring Batch provides a `Tasklet` implementation for
calling system commands, as shown in the following example:
.XML Configuration
[source, xml, role="xmlContent"]
----
<bean class="org.springframework.batch.core.step.tasklet.SystemCommandTasklet">
<property name="command" value="echo hello" />
<!-- 5 second timeout for the command to complete -->
<property name="timeout" value="5000" />
</bean>
----
.Java Configuration
[source, java, role="javaContent"]
----
@Bean
public SystemCommandTasklet tasklet() {
SystemCommandTasklet tasklet = new SystemCommandTasklet();
tasklet.setCommand("echo hello");
tasklet.setTimeout(5000);
return tasklet;
}
----
[[handlingStepCompletionWhenNoInputIsFound]]
=== Handling Step Completion When No Input is Found
In many batch scenarios, finding no rows in a database or file to process is not
exceptional. The `Step` is simply considered to have found no work and completes with 0
items read. All of the `ItemReader` implementations provided out of the box in Spring
Batch default to this approach. This can lead to some confusion if nothing is written out
even when input is present (which usually happens if a file was misnamed or some similar
issue arises). For this reason, the metadata itself should be inspected to determine how
much work the framework found to be processed. However, what if finding no input is
considered exceptional? In this case, programmatically checking the metadata for no items
processed and causing failure is the best solution. Because this is a common use case,
Spring Batch provides a listener with exactly this functionality, as shown in
the class definition for `NoWorkFoundStepExecutionListener`:
[source, java]
----
public class NoWorkFoundStepExecutionListener extends StepExecutionListenerSupport {
public ExitStatus afterStep(StepExecution stepExecution) {
if (stepExecution.getReadCount() == 0) {
return ExitStatus.FAILED;
}
return null;
}
}
----
The preceding `StepExecutionListener` inspects the `readCount` property of the
`StepExecution` during the 'afterStep' phase to determine if no items were read. If that
is the case, an exit code of FAILED is returned, indicating that the `Step` should fail.
Otherwise, `null` is returned, which does not affect the status of the `Step`.
[[passingDataToFutureSteps]]
=== Passing Data to Future Steps
It is often useful to pass information from one step to another. This can be done through
the `ExecutionContext`. The catch is that there are two `ExecutionContexts`: one at the
`Step` level and one at the `Job` level. The `Step` `ExecutionContext` remains only as
long as the step, while the `Job` `ExecutionContext` remains through the whole `Job`. On
the other hand, the `Step` `ExecutionContext` is updated every time the `Step` commits a
chunk, while the `Job` `ExecutionContext` is updated only at the end of each `Step`.
The consequence of this separation is that all data must be placed in the `Step`
`ExecutionContext` while the `Step` is executing. Doing so ensures that the data is
stored properly while the `Step` runs. If data is stored to the `Job` `ExecutionContext`,
then it is not persisted during `Step` execution. If the `Step` fails, that data is lost.
[source, java]
----
public class SavingItemWriter implements ItemWriter<Object> {
private StepExecution stepExecution;
public void write(List<? extends Object> items) throws Exception {
// ...
ExecutionContext stepContext = this.stepExecution.getExecutionContext();
stepContext.put("someKey", someObject);
}
@BeforeStep
public void saveStepExecution(StepExecution stepExecution) {
this.stepExecution = stepExecution;
}
}
----
To make the data available to future `Steps`, it must be "promoted" to the `Job`
`ExecutionContext` after the step has finished. Spring Batch provides the
`ExecutionContextPromotionListener` for this purpose. The listener must be configured
with the keys related to the data in the `ExecutionContext` that must be promoted. It can
also, optionally, be configured with a list of exit code patterns for which the promotion
should occur (`COMPLETED` is the default). As with all listeners, it must be registered
on the `Step` as shown in the following example:
.XML Configuration
[source, xml, role="xmlContent"]
----
<job id="job1">
<step id="step1">
<tasklet>
<chunk reader="reader" writer="savingWriter" commit-interval="10"/>
</tasklet>
<listeners>
<listener ref="promotionListener"/>
</listeners>
</step>
<step id="step2">
...
</step>
</job>
<beans:bean id="promotionListener" class="org.spr....ExecutionContextPromotionListener">
<beans:property name="keys">
<list>
<value>someKey</value>
</list>
</beans:property>
</beans:bean>
----
.Java Configuration
[source, java, role="javaContent"]
----
@Bean
public Job job1() {
return this.jobBuilderFactory.get("job1")
.start(step1())
.next(step1())
.build();
}
@Bean
public Step step1() {
return this.stepBuilderFactory.get("step1")
.<String, String>chunk(10)
.reader(reader())
.writer(savingWriter())
.listener(promotionListener())
.build();
}
@Bean
public ExecutionContextPromotionListener promotionListener() {
ExecutionContextPromotionListener listener = new ExecutionContextPromotionListener();
listener.setKeys(new String[] {"someKey" });
return listener;
}
----
Finally, the saved values must be retrieved from the `Job` `ExecutionContext`, as shown
in the following example:
[source, java]
----
public class RetrievingItemWriter implements ItemWriter<Object> {
private Object someObject;
public void write(List<? extends Object> items) throws Exception {
// ...
}
@BeforeStep
public void retrieveInterstepData(StepExecution stepExecution) {
JobExecution jobExecution = stepExecution.getJobExecution();
ExecutionContext jobContext = jobExecution.getExecutionContext();
this.someObject = jobContext.get("someKey");
}
}
----

3
docs/asciidoc/docinfo.html Normal file
View File

@ -0,0 +1,3 @@
<!-- Matomo Image Tracker-->
<img src="https://analytics.ossez.com/matomo.php?idsite=2&amp;rec=1" style="border:0" alt="" />
<!-- End Matomo -->

657
docs/asciidoc/domain.adoc Normal file
View File

@ -0,0 +1,657 @@
:batch-asciidoc: ./
:toc: left
:toclevels: 4
[[domainLanguageOfBatch]]
== The Domain Language of Batch
ifndef::onlyonetoggle[]
include::toggle.adoc[]
endif::onlyonetoggle[]
To any experienced batch architect, the overall concepts of batch processing used in
Spring Batch should be familiar and comfortable. There are "Jobs" and "Steps" and
developer-supplied processing units called `ItemReader` and `ItemWriter`. However,
because of the Spring patterns, operations, templates, callbacks, and idioms, there are
opportunities for the following:
* Significant improvement in adherence to a clear separation of concerns.
* Clearly delineated architectural layers and services provided as interfaces.
* Simple and default implementations that allow for quick adoption and ease of use
out-of-the-box.
* Significantly enhanced extensibility.
The following diagram is a simplified version of the batch reference architecture that
has been used for decades. It provides an overview of the components that make up the
domain language of batch processing. This architecture framework is a blueprint that has
been proven through decades of implementations on the last several generations of
platforms (COBOL/Mainframe, C++/Unix, and now Java/anywhere). JCL and COBOL developers
are likely to be as comfortable with the concepts as C++, C#, and Java developers. Spring
Batch provides a physical implementation of the layers, components, and technical
services commonly found in the robust, maintainable systems that are used to address the
creation of simple to complex batch applications, with the infrastructure and extensions
to address very complex processing needs.
.Batch Stereotypes
image::{batch-asciidoc}images/spring-batch-reference-model.png[Figure 2.1: Batch Stereotypes, scaledwidth="60%"]
The preceding diagram highlights the key concepts that make up the domain language of
Spring Batch. A Job has one to many steps, each of which has exactly one `ItemReader`,
one `ItemProcessor`, and one `ItemWriter`. A job needs to be launched (with
`JobLauncher`), and metadata about the currently running process needs to be stored (in
`JobRepository`).
=== Job
This section describes stereotypes relating to the concept of a batch job. A `Job` is an
entity that encapsulates an entire batch process. As is common with other Spring
projects, a `Job` is wired together with either an XML configuration file or Java-based
configuration. This configuration may be referred to as the "job configuration". However,
`Job` is just the top of an overall hierarchy, as shown in the following diagram:
.Job Hierarchy
image::{batch-asciidoc}images/job-heirarchy.png[Job Hierarchy, scaledwidth="60%"]
In Spring Batch, a `Job` is simply a container for `Step` instances. It combines multiple
steps that belong logically together in a flow and allows for configuration of properties
global to all steps, such as restartability. The job configuration contains:
* The simple name of the job.
* Definition and ordering of `Step` instances.
* Whether or not the job is restartable.
ifdef::backend-html5[]
[role="javaContent"]
A default simple implementation of the Job interface is provided by Spring Batch in the
form of the `SimpleJob` class, which creates some standard functionality on top of `Job`.
When using java based configuration, a collection of builders is made available for the
instantiation of a `Job`, as shown in the following example:
[source, java, role="javaContent"]
----
@Bean
public Job footballJob() {
return this.jobBuilderFactory.get("footballJob")
.start(playerLoad())
.next(gameLoad())
.next(playerSummarization())
.end()
.build();
}
----
[role="xmlContent"]
A default simple implementation of the `Job` interface is provided by Spring Batch in the
form of the `SimpleJob` class, which creates some standard functionality on top of `Job`.
However, the batch namespace abstracts away the need to instantiate it directly. Instead,
the `<job>` tag can be used as shown in the following example:
[source, xml, role="xmlContent"]
----
<job id="footballJob">
<step id="playerload" next="gameLoad"/>
<step id="gameLoad" next="playerSummarization"/>
<step id="playerSummarization"/>
</job>
----
endif::backend-html5[]
ifdef::backend-pdf[]
A default simple implementation of the Job interface is provided by Spring Batch in the
form of the `SimpleJob` class, which creates some standard functionality on top of `Job`.
When using java based configuration, a collection of builders are made available for the
instantiation of a `Job`, as shown in the following example:
[source, java]
----
@Bean
public Job footballJob() {
return this.jobBuilderFactory.get("footballJob")
.start(playerLoad())
.next(gameLoad())
.next(playerSummarization())
.end()
.build();
}
----
However, when using XML configuration, the batch namespace abstracts away the need to
instantiate it directly. Instead, the `<job>` tag can be used as shown in the following
example:
[source, xml]
----
<job id="footballJob">
<step id="playerload" next="gameLoad"/>
<step id="gameLoad" next="playerSummarization"/>
<step id="playerSummarization"/>
</job>
----
endif::backend-pdf[]
==== JobInstance
A `JobInstance` refers to the concept of a logical job run. Consider a batch job that
should be run once at the end of the day, such as the 'EndOfDay' `Job` from the preceding
diagram. There is one 'EndOfDay' job, but each individual run of the `Job` must be
tracked separately. In the case of this job, there is one logical `JobInstance` per day.
For example, there is a January 1st run, a January 2nd run, and so on. If the January 1st
run fails the first time and is run again the next day, it is still the January 1st run.
(Usually, this corresponds with the data it is processing as well, meaning the January
1st run processes data for January 1st). Therefore, each `JobInstance` can have multiple
executions (`JobExecution` is discussed in more detail later in this chapter), and only
one `JobInstance` corresponding to a particular `Job` and identifying `JobParameters` can
run at a given time.
The definition of a `JobInstance` has absolutely no bearing on the data the to be loaded.
It is entirely up to the `ItemReader` implementation to determine how data is loaded. For
example, in the EndOfDay scenario, there may be a column on the data that indicates the
'effective date' or 'schedule date' to which the data belongs. So, the January 1st run
would load only data from the 1st, and the January 2nd run would use only data from the
2nd. Because this determination is likely to be a business decision, it is left up to the
`ItemReader` to decide. However, using the same `JobInstance` determines whether or not
the 'state' (that is, the `ExecutionContext`, which is discussed later in this chapter)
from previous executions is used. Using a new `JobInstance` means 'start from the
beginning', and using an existing instance generally means 'start from where you left
off'.
==== JobParameters
Having discussed `JobInstance` and how it differs from Job, the natural question to ask
is: "How is one `JobInstance` distinguished from another?" The answer is:
`JobParameters`. A `JobParameters` object holds a set of parameters used to start a batch
job. They can be used for identification or even as reference data during the run, as
shown in the following image:
.Job Parameters
image::{batch-asciidoc}images/job-stereotypes-parameters.png[Job Parameters, scaledwidth="60%"]
In the preceding example, where there are two instances, one for January 1st, and another
for January 2nd, there is really only one `Job`, but it has two `JobParameter` objects:
one that was started with a job parameter of 01-01-2017 and another that was started with
a parameter of 01-02-2017. Thus, the contract can be defined as: `JobInstance` = `Job`
+ identifying `JobParameters`. This allows a developer to effectively control how a
`JobInstance` is defined, since they control what parameters are passed in.
NOTE: Not all job parameters are required to contribute to the identification of a
`JobInstance`. By default, they do so. However, the framework also allows the submission
of a `Job` with parameters that do not contribute to the identity of a `JobInstance`.
==== JobExecution
A `JobExecution` refers to the technical concept of a single attempt to run a Job. An
execution may end in failure or success, but the `JobInstance` corresponding to a given
execution is not considered to be complete unless the execution completes successfully.
Using the EndOfDay `Job` described previously as an example, consider a `JobInstance` for
01-01-2017 that failed the first time it was run. If it is run again with the same
identifying job parameters as the first run (01-01-2017), a new `JobExecution` is
created. However, there is still only one `JobInstance`.
A `Job` defines what a job is and how it is to be executed, and a `JobInstance` is a
purely organizational object to group executions together, primarily to enable correct
restart semantics. A `JobExecution`, however, is the primary storage mechanism for what
actually happened during a run and contains many more properties that must be controlled
and persisted, as shown in the following table:
.JobExecution Properties
|===
|Property |Definition
|Status
|A `BatchStatus` object that indicates the status of the execution. While running, it is
`BatchStatus#STARTED`. If it fails, it is `BatchStatus#FAILED`. If it finishes
successfully, it is `BatchStatus#COMPLETED`
|startTime
|A `java.util.Date` representing the current system time when the execution was started.
This field is empty if the job has yet to start.
|endTime
|A `java.util.Date` representing the current system time when the execution finished,
regardless of whether or not it was successful. The field is empty if the job has yet to
finish.
|exitStatus
|The `ExitStatus`, indicating the result of the run. It is most important, because it
contains an exit code that is returned to the caller. See chapter 5 for more details. The
field is empty if the job has yet to finish.
|createTime
|A `java.util.Date` representing the current system time when the `JobExecution` was
first persisted. The job may not have been started yet (and thus has no start time), but
it always has a createTime, which is required by the framework for managing job level
`ExecutionContexts`.
|lastUpdated
|A `java.util.Date` representing the last time a `JobExecution` was persisted. This field
is empty if the job has yet to start.
|executionContext
|The "property bag" containing any user data that needs to be persisted between
executions.
|failureExceptions
|The list of exceptions encountered during the execution of a `Job`. These can be useful
if more than one exception is encountered during the failure of a `Job`.
|===
These properties are important because they are persisted and can be used to completely
determine the status of an execution. For example, if the EndOfDay job for 01-01 is
executed at 9:00 PM and fails at 9:30, the following entries are made in the batch
metadata tables:
.BATCH_JOB_INSTANCE
|===
|JOB_INST_ID |JOB_NAME
|1
|EndOfDayJob
|===
.BATCH_JOB_EXECUTION_PARAMS
|===
|JOB_EXECUTION_ID|TYPE_CD|KEY_NAME|DATE_VAL|IDENTIFYING
|1
|DATE
|schedule.Date
|2017-01-01
|TRUE
|===
.BATCH_JOB_EXECUTION
|===
|JOB_EXEC_ID|JOB_INST_ID|START_TIME|END_TIME|STATUS
|1
|1
|2017-01-01 21:00
|2017-01-01 21:30
|FAILED
|===
NOTE: Column names may have been abbreviated or removed for the sake of clarity and
formatting.
Now that the job has failed, assume that it took the entire night for the problem to be
determined, so that the 'batch window' is now closed. Further assuming that the window
starts at 9:00 PM, the job is kicked off again for 01-01, starting where it left off and
completing successfully at 9:30. Because it is now the next day, the 01-02 job must be
run as well, and it is kicked off just afterwards at 9:31 and completes in its normal one
hour time at 10:30. There is no requirement that one `JobInstance` be kicked off after
another, unless there is potential for the two jobs to attempt to access the same data,
causing issues with locking at the database level. It is entirely up to the scheduler to
determine when a `Job` should be run. Since they are separate `JobInstances`, Spring
Batch makes no attempt to stop them from being run concurrently. (Attempting to run the
same `JobInstance` while another is already running results in a
`JobExecutionAlreadyRunningException` being thrown). There should now be an extra entry
in both the `JobInstance` and `JobParameters` tables and two extra entries in the
`JobExecution` table, as shown in the following tables:
.BATCH_JOB_INSTANCE
|===
|JOB_INST_ID |JOB_NAME
|1
|EndOfDayJob
|2
|EndOfDayJob
|===
.BATCH_JOB_EXECUTION_PARAMS
|===
|JOB_EXECUTION_ID|TYPE_CD|KEY_NAME|DATE_VAL|IDENTIFYING
|1
|DATE
|schedule.Date
|2017-01-01 00:00:00
|TRUE
|2
|DATE
|schedule.Date
|2017-01-01 00:00:00
|TRUE
|3
|DATE
|schedule.Date
|2017-01-02 00:00:00
|TRUE
|===
.BATCH_JOB_EXECUTION
|===
|JOB_EXEC_ID|JOB_INST_ID|START_TIME|END_TIME|STATUS
|1
|1
|2017-01-01 21:00
|2017-01-01 21:30
|FAILED
|2
|1
|2017-01-02 21:00
|2017-01-02 21:30
|COMPLETED
|3
|2
|2017-01-02 21:31
|2017-01-02 22:29
|COMPLETED
|===
NOTE: Column names may have been abbreviated or removed for the sake of clarity and
formatting.
=== Step
A `Step` is a domain object that encapsulates an independent, sequential phase of a batch
job. Therefore, every Job is composed entirely of one or more steps. A `Step` contains
all of the information necessary to define and control the actual batch processing. This
is a necessarily vague description because the contents of any given `Step` are at the
discretion of the developer writing a `Job`. A `Step` can be as simple or complex as the
developer desires. A simple `Step` might load data from a file into the database,
requiring little or no code (depending upon the implementations used). A more complex
`Step` may have complicated business rules that are applied as part of the processing. As
with a `Job`, a `Step` has an individual `StepExecution` that correlates with a unique
`JobExecution`, as shown in the following image:
.Job Hierarchy With Steps
image::{batch-asciidoc}images/jobHeirarchyWithSteps.png[Figure 2.1: Job Hierarchy With Steps, scaledwidth="60%"]
==== StepExecution
A `StepExecution` represents a single attempt to execute a `Step`. A new `StepExecution`
is created each time a `Step` is run, similar to `JobExecution`. However, if a step fails
to execute because the step before it fails, no execution is persisted for it. A
`StepExecution` is created only when its `Step` is actually started.
`Step` executions are represented by objects of the `StepExecution` class. Each execution
contains a reference to its corresponding step and `JobExecution` and transaction related
data, such as commit and rollback counts and start and end times. Additionally, each step
execution contains an `ExecutionContext`, which contains any data a developer needs to
have persisted across batch runs, such as statistics or state information needed to
restart. The following table lists the properties for `StepExecution`:
.StepExecution Properties
|===
|Property|Definition
|Status
|A `BatchStatus` object that indicates the status of the execution. While running, the
status is `BatchStatus.STARTED`. If it fails, the status is `BatchStatus.FAILED`. If it
finishes successfully, the status is `BatchStatus.COMPLETED`.
|startTime
|A `java.util.Date` representing the current system time when the execution was started.
This field is empty if the step has yet to start.
|endTime
|A `java.util.Date` representing the current system time when the execution finished,
regardless of whether or not it was successful. This field is empty if the step has yet to
exit.
|exitStatus
|The `ExitStatus` indicating the result of the execution. It is most important, because
it contains an exit code that is returned to the caller. See chapter 5 for more details.
This field is empty if the job has yet to exit.
|executionContext
|The "property bag" containing any user data that needs to be persisted between
executions.
|readCount
|The number of items that have been successfully read.
|writeCount
|The number of items that have been successfully written.
|commitCount
|The number of transactions that have been committed for this execution.
|rollbackCount
|The number of times the business transaction controlled by the `Step` has been rolled
back.
|readSkipCount
|The number of times `read` has failed, resulting in a skipped item.
|processSkipCount
|The number of times `process` has failed, resulting in a skipped item.
|filterCount
|The number of items that have been 'filtered' by the `ItemProcessor`.
|writeSkipCount
|The number of times `write` has failed, resulting in a skipped item.
|===
=== ExecutionContext
An `ExecutionContext` represents a collection of key/value pairs that are persisted and
controlled by the framework in order to allow developers a place to store persistent
state that is scoped to a `StepExecution` object or a `JobExecution` object. For those
familiar with Quartz, it is very similar to JobDataMap. The best usage example is to
facilitate restart. Using flat file input as an example, while processing individual
lines, the framework periodically persists the `ExecutionContext` at commit points. Doing
so allows the `ItemReader` to store its state in case a fatal error occurs during the run
or even if the power goes out. All that is needed is to put the current number of lines
read into the context, as shown in the following example, and the framework will do the
rest:
[source, java]
----
executionContext.putLong(getKey(LINES_READ_COUNT), reader.getPosition());
----
Using the EndOfDay example from the `Job` Stereotypes section as an example, assume there
is one step, 'loadData', that loads a file into the database. After the first failed run,
the metadata tables would look like the following example:
.BATCH_JOB_INSTANCE
|===
|JOB_INST_ID|JOB_NAME
|1
|EndOfDayJob
|===
.BATCH_JOB_EXECUTION_PARAMS
|===
|JOB_INST_ID|TYPE_CD|KEY_NAME|DATE_VAL
|1
|DATE
|schedule.Date
|2017-01-01
|===
.BATCH_JOB_EXECUTION
|===
|JOB_EXEC_ID|JOB_INST_ID|START_TIME|END_TIME|STATUS
|1
|1
|2017-01-01 21:00
|2017-01-01 21:30
|FAILED
|===
.BATCH_STEP_EXECUTION
|===
|STEP_EXEC_ID|JOB_EXEC_ID|STEP_NAME|START_TIME|END_TIME|STATUS
|1
|1
|loadData
|2017-01-01 21:00
|2017-01-01 21:30
|FAILED
|===
.BATCH_STEP_EXECUTION_CONTEXT
|===
|STEP_EXEC_ID|SHORT_CONTEXT
|1
|{piece.count=40321}
|===
In the preceding case, the `Step` ran for 30 minutes and processed 40,321 'pieces', which
would represent lines in a file in this scenario. This value is updated just before each
commit by the framework and can contain multiple rows corresponding to entries within the
`ExecutionContext`. Being notified before a commit requires one of the various
`StepListener` implementations (or an `ItemStream`), which are discussed in more detail
later in this guide. As with the previous example, it is assumed that the `Job` is
restarted the next day. When it is restarted, the values from the `ExecutionContext` of
the last run are reconstituted from the database. When the `ItemReader` is opened, it can
check to see if it has any stored state in the context and initialize itself from there,
as shown in the following example:
[source, java]
----
if (executionContext.containsKey(getKey(LINES_READ_COUNT))) {
log.debug("Initializing for restart. Restart data is: " + executionContext);
long lineCount = executionContext.getLong(getKey(LINES_READ_COUNT));
LineReader reader = getReader();
Object record = "";
while (reader.getPosition() < lineCount && record != null) {
record = readLine();
}
}
----
In this case, after the above code runs, the current line is 40,322, allowing the `Step`
to start again from where it left off. The `ExecutionContext` can also be used for
statistics that need to be persisted about the run itself. For example, if a flat file
contains orders for processing that exist across multiple lines, it may be necessary to
store how many orders have been processed (which is much different from the number of
lines read), so that an email can be sent at the end of the `Step` with the total number
of orders processed in the body. The framework handles storing this for the developer, in
order to correctly scope it with an individual `JobInstance`. It can be very difficult to
know whether an existing `ExecutionContext` should be used or not. For example, using the
'EndOfDay' example from above, when the 01-01 run starts again for the second time, the
framework recognizes that it is the same `JobInstance` and on an individual `Step` basis,
pulls the `ExecutionContext` out of the database, and hands it (as part of the
`StepExecution`) to the `Step` itself. Conversely, for the 01-02 run, the framework
recognizes that it is a different instance, so an empty context must be handed to the
`Step`. There are many of these types of determinations that the framework makes for the
developer, to ensure the state is given to them at the correct time. It is also important
to note that exactly one `ExecutionContext` exists per `StepExecution` at any given time.
Clients of the `ExecutionContext` should be careful, because this creates a shared
keyspace. As a result, care should be taken when putting values in to ensure no data is
overwritten. However, the `Step` stores absolutely no data in the context, so there is no
way to adversely affect the framework.
It is also important to note that there is at least one `ExecutionContext` per
`JobExecution` and one for every `StepExecution`. For example, consider the following
code snippet:
[source, java]
----
ExecutionContext ecStep = stepExecution.getExecutionContext();
ExecutionContext ecJob = jobExecution.getExecutionContext();
//ecStep does not equal ecJob
----
As noted in the comment, `ecStep` does not equal `ecJob`. They are two different
`ExecutionContexts`. The one scoped to the `Step` is saved at every commit point in the
`Step`, whereas the one scoped to the Job is saved in between every `Step` execution.
=== JobRepository
`JobRepository` is the persistence mechanism for all of the Stereotypes mentioned above.
It provides CRUD operations for `JobLauncher`, `Job`, and `Step` implementations. When a
`Job` is first launched, a `JobExecution` is obtained from the repository, and, during
the course of execution, `StepExecution` and `JobExecution` implementations are persisted
by passing them to the repository.
[role="xmlContent"]
The batch namespace provides support for configuring a `JobRepository` instance with the
`<job-repository>` tag, as shown in the following example:
[source, xml, role="xmlContent"]
----
<job-repository id="jobRepository"/>
----
[role="javaContent"]
When using java configuration, `@EnableBatchProcessing` annotation provides a
`JobRepository` as one of the components automatically configured out of the box.
=== JobLauncher
`JobLauncher` represents a simple interface for launching a `Job` with a given set of
`JobParameters`, as shown in the following example:
[source, java]
----
public interface JobLauncher {
public JobExecution run(Job job, JobParameters jobParameters)
throws JobExecutionAlreadyRunningException, JobRestartException,
JobInstanceAlreadyCompleteException, JobParametersInvalidException;
}
----
It is expected that implementations obtain a valid `JobExecution` from the
`JobRepository` and execute the `Job`.
=== Item Reader
`ItemReader` is an abstraction that represents the retrieval of input for a `Step`, one
item at a time. When the `ItemReader` has exhausted the items it can provide, it
indicates this by returning `null`. More details about the `ItemReader` interface and its
various implementations can be found in
<<readersAndWriters.adoc#readersAndWriters,Readers And Writers>>.
=== Item Writer
`ItemWriter` is an abstraction that represents the output of a `Step`, one batch or chunk
of items at a time. Generally, an `ItemWriter` has no knowledge of the input it should
receive next and knows only the item that was passed in its current invocation. More
details about the `ItemWriter` interface and its various implementations can be found in
<<readersAndWriters.adoc#readersAndWriters,Readers And Writers>>.
=== Item Processor
`ItemProcessor` is an abstraction that represents the business processing of an item.
While the `ItemReader` reads one item, and the `ItemWriter` writes them, the
`ItemProcessor` provides an access point to transform or apply other business processing.
If, while processing the item, it is determined that the item is not valid, returning
`null` indicates that the item should not be written out. More details about the
`ItemProcessor` interface can be found in
<<readersAndWriters.adoc#readersAndWriters,Readers And Writers>>.
[role="xmlContent"]
=== Batch Namespace
Many of the domain concepts listed previously need to be configured in a Spring
`ApplicationContext`. While there are implementations of the interfaces above that can be
used in a standard bean definition, a namespace has been provided for ease of
configuration, as shown in the following example:
[source, xml, role="xmlContent"]
----
<beans:beans xmlns="http://www.springframework.org/schema/batch"
xmlns:beans="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="
http://www.springframework.org/schema/beans
https://www.springframework.org/schema/beans/spring-beans.xsd
http://www.springframework.org/schema/batch
https://www.springframework.org/schema/batch/spring-batch.xsd">
<job id="ioSampleJob">
<step id="step1">
<tasklet>
<chunk reader="itemReader" writer="itemWriter" commit-interval="2"/>
</tasklet>
</step>
</job>
</beans:beans>
----
[role="xmlContent"]
As long as the batch namespace has been declared, any of its elements can be used. More
information on configuring a Job can be found in <<job.adoc#configureJob,Configuring and
Running a Job>>. More information on configuring a `Step` can be found in
<<step.adoc#configureStep,Configuring a Step>>.

12
docs/asciidoc/footer/index-footer.adoc Normal file
View File

@ -0,0 +1,12 @@
'''
文档的中文版本由 CWIKI.US 进行翻译和整理。
YuCheng Hu
Copyright © 2009 - 2019 North Tecom, LLC. All Rights
Reserved.
Copies of this document may be made for your own use and for
distribution to others, provided that you do not charge any fee for such
copies and further provided that each copy contains this Copyright
Notice, whether distributed in print or electronically.

103
docs/asciidoc/glossary.adoc Normal file
View File

@ -0,0 +1,103 @@
[[glossary]]
[appendix]
== 术语表
[glossary]
=== Spring Batch 术语表
Batch::
An accumulation of business transactions over time.
Batch Application Style::
Term used to designate batch as an application style in its own right, similar to
online, Web, or SOA. It has standard elements of input, validation, transformation of
information to business model, business processing, and output. In addition, it
requires monitoring at a macro level.
Batch Processing::
The handling of a batch of many business transactions that have accumulated over a
period of time (such as an hour, a day, a week, a month, or a year). It is the
application of a process or set of processes to many data entities or objects in a
repetitive and predictable fashion with either no manual element or a separate manual
element for error processing.
Batch Window::
The time frame within which a batch job must complete. This can be constrained by other
systems coming online, other dependent jobs needing to execute, or other factors
specific to the batch environment.
Step::
The main batch task or unit of work. It initializes the business logic and controls the
transaction environment, based on commit interval setting and other factors.
Tasklet::
A component created by an application developer to process the business logic for a
Step.
Batch Job Type::
Job types describe application of jobs for particular types of processing. Common areas
are interface processing (typically flat files), forms processing (either for online
PDF generation or print formats), and report processing.
Driving Query::
A driving query identifies the set of work for a job to do. The job then breaks that
work into individual units of work. For instance, a driving query might be to identify
all financial transactions that have a status of "pending transmission" and send them
to a partner system. The driving query returns a set of record IDs to process. Each
record ID then becomes a unit of work. A driving query may involve a join (if the
criteria for selection falls across two or more tables) or it may work with a single
table.
Item::
An item represents the smallest amount of complete data for processing. In the simplest
terms, this might be a line in a file, a row in a database table, or a particular
element in an XML file.
Logicial Unit of Work (LUW)::
A batch job iterates through a driving query (or other input source, such as a file) to
perform the set of work that the job must accomplish. Each iteration of work performed
is a unit of work.
Commit Interval::
A set of LUWs processed within a single transaction.
Partitioning::
Splitting a job into multiple threads where each thread is responsible for a subset of
the overall data to be processed. The threads of execution may be within the same JVM
or they may span JVMs in a clustered environment that supports workload balancing.
Staging Table::
A table that holds temporary data while it is being processed.
Restartable::
A job that can be executed again and assumes the same identity as when run initially.
In other words, it is has the same job instance ID.
Rerunnable::
A job that is restartable and manages its own state in terms of the previous run's
record processing. An example of a rerunnable step is one based on a driving query. If
the driving query can be formed so that it limits the processed rows when the job is
restarted, then it is re-runnable. This is managed by the application logic. Often, a
condition is added to the `where` statement to limit the rows returned by the driving
query with logic resembling "and processedFlag!= true".
Repeat::
One of the most basic units of batch processing, it defines by repeatability calling a
portion of code until it is finished and while there is no error. Typically, a batch
process would be repeatable as long as there is input.
Retry::
Simplifies the execution of operations with retry semantics most frequently associated
with handling transactional output exceptions. Retry is slightly different from repeat,
rather than continually calling a block of code, retry is stateful and continually
calls the same block of code with the same input, until it either succeeds or some type
of retry limit has been exceeded. It is only generally useful when a subsequent
invocation of the operation might succeed because something in the environment has
improved.
Recover::
Recover operations handle an exception in such a way that a repeat process is able to
continue.
Skip::
Skip is a recovery strategy often used on file input sources as the strategy for
ignoring bad input records that failed validation.

BIN
docs/asciidoc/images/1-1-step.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 13 KiB

45
docs/asciidoc/index-single.adoc Normal file
View File

@ -0,0 +1,45 @@
:doctype: book
:toc: left
:toclevels: 4
:sectnums:
:onlyonetoggle: true
include::header/index-header.adoc[]
include::toggle.adoc[]
include::spring-batch-intro.adoc[]
include::whatsnew.adoc[]
include::domain.adoc[]
include::job.adoc[]
include::step.adoc[]
include::readersAndWriters.adoc[]
include::scalability.adoc[]
include::repeat.adoc[]
include::retry.adoc[]
include::testing.adoc[]
include::common-patterns.adoc[]
include::jsr-352.adoc[]
include::spring-batch-integration.adoc[]
include::monitoring-and-metrics.adoc[]
include::appendix.adoc[]
include::schema-appendix.adoc[]
include::transaction-appendix.adoc[]
include::glossary.adoc[]

34
docs/asciidoc/index.adoc Normal file
View File

@ -0,0 +1,34 @@
include::header/index-header.adoc[]
// ======================================================================================
欢迎来到 Spring 批量（Spring Batch）文档！本文档的内容同时还提供单一 link:index-single.html[html] 和 link:../pdf/spring-batch-reference.pdf[pdf] 文档。
本参考文档被分列为下面的部分：
[horizontal]
<<spring-batch-intro.adoc#spring-batch-intro,Spring Batch Introduction>> :: 背景，使用场景和一些常用使用指引。
<<whatsnew.adoc#whatsNew,What's new in Spring Batch 4.2>> :: 4.2 版本中的新特性。
<<domain.adoc#domainLanguageOfBatch,The Domain Language of Batch>> :: 核心概念和有关批处理域语言（Batch Domain Language）的抽象。
<<job.adoc#configureJob,Configuring and Running a Job>> :: 作业（Job）配置，执行和管理。
<<step.adoc#configureStep,Configuring a Step>> :: 步骤（Step）配置，不同类型的步骤，控制步骤流。
<<readersAndWriters.adoc#readersAndWriters,ItemReaders and ItemWriters>> :: 条目（Item）读和写接口以及如何使用它们。
<<scalability.adoc#scalability,Scaling and Parallel Processing>> :: 多线程步骤（Steps），并行步骤，远程分块（Chunking）和分区（Partitioning）。
<<repeat.adoc#repeat,Repeat>> :: 重复行为（Repetitive Actions）的完成策略和异常处理。
<<retry.adoc#retry,Retry>> :: 可重试操作的重做和退出策略。
<<testing.adoc#testing,Unit Testing>> :: 作业和步骤的测试策略和 APIs。
<<common-patterns.adoc#commonPatterns, Common Patterns>> :: 通用批量处理模式和使用指引。
<<jsr-352.adoc#jsr-352,JSR-352 Support>> :: JSR-352 支持，与 Spring Batch 相似和不同之处。
<<spring-batch-integration.adoc#springBatchIntegration,Spring Batch Integration>> :: Spring Batch 与 Spring 其他项目的整合。
<<monitoring-and-metrics.adoc#monitoring-and-metrics,Monitoring and metrics>> :: 批量作业的监控和指标
有下列有关的可用附录：
[horizontal]
<<appendix.adoc#listOfReadersAndWriters,List of ItemReaders and ItemWriters>> :: 提供的开箱即用（out-of-the box）项目读和写的完整列表。
<<schema-appendix.adoc#metaDataSchema,Meta-Data Schema>> :: 批量域模式使用的核心表。
<<transaction-appendix.adoc#transactions,Batch Processing and Transactions>> :: Spring Batch 中使用的事务边界（Transaction Boundaries），
事务传播（Transaction propagation）和事务隔离级别（Transaction Isolation Levels）。
<<glossary.adoc#glossary,Glossary>> :: 批量域中使用的常用术语，概念和词汇。
include::footer/index-footer.adoc[]

1780
docs/asciidoc/job.adoc Normal file
View File

File diff suppressed because it is too large Load Diff

64
docs/asciidoc/js/DocumentToggle.js Normal file
View File

@ -0,0 +1,64 @@
$(document).ready(function(){
setJava();
// Initial cookie handler. This part remembers the reader's choice and sets the toggle
// accordingly.
var docToggleCookieString = Cookies.get("docToggle");
if (docToggleCookieString != null) {
if (docToggleCookieString === "xml") {
$("#xmlButton").prop("checked", true);
setXml();
} else if (docToggleCookieString === "java") {
$("#javaButton").prop("checked", true);
setJava();
} else if (docToggleCookieString === "both") {
$("#bothButton").prop("checked", true);
setBoth();
}
}
// Click handlers
$("#xmlButton").on("click", function() {
setXml();
});
$("#javaButton").on("click", function() {
setJava();
});
$("#bothButton").on("click", function() {
setBoth();
});
// Functions to do the work of handling the reader's choice, whether through a click
// or through a cookie. 3652 days is 10 years, give or take a leap day.
function setXml() {
$("*.xmlContent").show();
$("*.javaContent").hide();
$("*.javaContent > *").addClass("js-toc-ignore");
$("*.xmlContent > *").removeClass("js-toc-ignore");
window.dispatchEvent(new Event("tocRefresh"));
tocbot.refresh();
Cookies.set('docToggle', 'xml', { expires: 3652 });
};
function setJava() {
$("*.javaContent").show();
$("*.xmlContent").hide();
$("*.xmlContent > *").addClass("js-toc-ignore");
$("*.javaContent > *").removeClass("js-toc-ignore");
window.dispatchEvent(new Event("tocRefresh"));
tocbot.refresh();
Cookies.set('docToggle', 'java', { expires: 3652 });
};
function setBoth() {
$("*.javaContent").show();
$("*.xmlContent").show();
$("*.javaContent > *").removeClass("js-toc-ignore");
$("*.xmlContent > *").removeClass("js-toc-ignore");
window.dispatchEvent(new Event("tocRefresh"));
tocbot.refresh();
Cookies.set('docToggle', 'both', { expires: 3652 });
};
});

4
docs/asciidoc/js/jquery-3.2.1.min.js vendored Normal file
View File

File diff suppressed because one or more lines are too long

9807
docs/asciidoc/js/jquery.js vendored Normal file
View File

File diff suppressed because it is too large Load Diff

165
docs/asciidoc/js/js.cookie.js Normal file
View File

@ -0,0 +1,165 @@
/*!
* JavaScript Cookie v2.1.4
* https://github.com/js-cookie/js-cookie
*
* Copyright 2006, 2015 Klaus Hartl & Fagner Brack
* Released under the MIT license
*/
;(function (factory) {
var registeredInModuleLoader = false;
if (typeof define === 'function' && define.amd) {
define(factory);
registeredInModuleLoader = true;
}
if (typeof exports === 'object') {
module.exports = factory();
registeredInModuleLoader = true;
}
if (!registeredInModuleLoader) {
var OldCookies = window.Cookies;
var api = window.Cookies = factory();
api.noConflict = function () {
window.Cookies = OldCookies;
return api;
};
}
}(function () {
function extend () {
var i = 0;
var result = {};
for (; i < arguments.length; i++) {
var attributes = arguments[ i ];
for (var key in attributes) {
result[key] = attributes[key];
}
}
return result;
}
function init (converter) {
function api (key, value, attributes) {
var result;
if (typeof document === 'undefined') {
return;
}
// Write
if (arguments.length > 1) {
attributes = extend({
path: '/'
}, api.defaults, attributes);
if (typeof attributes.expires === 'number') {
var expires = new Date();
expires.setMilliseconds(expires.getMilliseconds() + attributes.expires * 864e+5);
attributes.expires = expires;
}
// We're using "expires" because "max-age" is not supported by IE
attributes.expires = attributes.expires ? attributes.expires.toUTCString() : '';
try {
result = JSON.stringify(value);
if (/^[\{\[]/.test(result)) {
value = result;
}
} catch (e) {}
if (!converter.write) {
value = encodeURIComponent(String(value))
.replace(/%(23|24|26|2B|3A|3C|3E|3D|2F|3F|40|5B|5D|5E|60|7B|7D|7C)/g, decodeURIComponent);
} else {
value = converter.write(value, key);
}
key = encodeURIComponent(String(key));
key = key.replace(/%(23|24|26|2B|5E|60|7C)/g, decodeURIComponent);
key = key.replace(/[\(\)]/g, escape);
var stringifiedAttributes = '';
for (var attributeName in attributes) {
if (!attributes[attributeName]) {
continue;
}
stringifiedAttributes += '; ' + attributeName;
if (attributes[attributeName] === true) {
continue;
}
stringifiedAttributes += '=' + attributes[attributeName];
}
return (document.cookie = key + '=' + value + stringifiedAttributes);
}
// Read
if (!key) {
result = {};
}
// To prevent the for loop in the first place assign an empty array
// in case there are no cookies at all. Also prevents odd result when
// calling "get()"
var cookies = document.cookie ? document.cookie.split('; ') : [];
var rdecode = /(%[0-9A-Z]{2})+/g;
var i = 0;
for (; i < cookies.length; i++) {
var parts = cookies[i].split('=');
var cookie = parts.slice(1).join('=');
if (cookie.charAt(0) === '"') {
cookie = cookie.slice(1, -1);
}
try {
var name = parts[0].replace(rdecode, decodeURIComponent);
cookie = converter.read ?
converter.read(cookie, name) : converter(cookie, name) ||
cookie.replace(rdecode, decodeURIComponent);
if (this.json) {
try {
cookie = JSON.parse(cookie);
} catch (e) {}
}
if (key === name) {
result = cookie;
break;
}
if (!key) {
result[name] = cookie;
}
} catch (e) {}
}
return result;
}
api.set = api;
api.get = function (key) {
return api.call(api, key);
};
api.getJSON = function () {
return api.apply({
json: true
}, [].slice.call(arguments));
};
api.defaults = {};
api.remove = function (key, attributes) {
api(key, '', extend(attributes, {
expires: -1
}));
};
api.withConverter = init;
return api;
}
return init(function () {});
}));

653
docs/asciidoc/jsr-352.adoc Normal file
View File

@ -0,0 +1,653 @@
:batch-asciidoc: ./
:toc: left
:toclevels: 4
[[jsr-352]]
== JSR-352 Support
ifndef::onlyonetoggle[]
include::toggle.adoc[]
endif::onlyonetoggle[]
As of Spring Batch 3.0 support for JSR-352 has been fully implemented. This section is not a replacement for
the spec itself and instead, intends to explain how the JSR-352 specific concepts apply to Spring Batch.
Additional information on JSR-352 can be found via the
JCP here: link:$$https://jcp.org/en/jsr/detail?id=352$$[https://jcp.org/en/jsr/detail?id=352]
[[jsrGeneralNotes]]
=== General Notes about Spring Batch and JSR-352
Spring Batch and JSR-352 are structurally the same. They both have jobs that are made up of steps. They
both have readers, processors, writers, and listeners. However, their interactions are subtly different.
For example, the `org.springframework.batch.core.SkipListener#onSkipInWrite(S item, Throwable t)`
within Spring Batch receives two parameters: the item that was skipped and the Exception that caused the
skip. The JSR-352 version of the same method
(`javax.batch.api.chunk.listener.SkipWriteListener#onSkipWriteItem(List&lt;Object&gt; items, Exception ex)`)
also receives two parameters. However the first one is a `List` of all the items
within the current chunk with the second being the `Exception` that caused the skip.
Because of these differences, it is important to note that there are two paths to execute a job within
Spring Batch: either a traditional Spring Batch job or a JSR-352 based job. While the use of Spring Batch
artifacts (readers, writers, etc) will work within a job configured via JSR-352's JSL and executed via the
`JsrJobOperator`, they will behave according to the rules of JSR-352. It is also
important to note that batch artifacts that have been developed against the JSR-352 interfaces will not work
within a traditional Spring Batch job.
[[jsrSetup]]
=== Setup
[[jsrSetupContexts]]
==== Application Contexts
All JSR-352 based jobs within Spring Batch consist of two application contexts. A parent context, that
contains beans related to the infrastructure of Spring Batch such as the `JobRepository`,
`PlatformTransactionManager`, etc and a child context that consists of the configuration
of the job to be run. The parent context is defined via the `jsrBaseContext.xml` provided
by the framework. This context may be overridden via the `JSR-352-BASE-CONTEXT` system
property.
[NOTE]
====
The base context is not processed by the JSR-352 processors for things like property injection so
no components requiring that additional processing should be configured there.
====
[[jsrSetupLaunching]]
==== Launching a JSR-352 based job
JSR-352 requires a very simple path to executing a batch job. The following code is all that is needed to
execute your first batch job:
[source, java]
----
JobOperator operator = BatchRuntime.getJobOperator();
jobOperator.start("myJob", new Properties());
----
While that is convenient for developers, the devil is in the details. Spring Batch bootstraps a bit of
infrastructure behind the scenes that a developer may want to override. The following is bootstrapped the
first time `BatchRuntime.getJobOperator()` is called:
|===============
|__Bean Name__|__Default Configuration__|__Notes__
|
dataSource
|
Apache DBCP BasicDataSource with configured values.
|
By default, HSQLDB is bootstrapped.
|`transactionManager`|`org.springframework.jdbc.datasource.DataSourceTransactionManager`|
References the dataSource bean defined above.
|
A Datasource initializer
||
This is configured to execute the scripts configured via the
`batch.drop.script` and `batch.schema.script` properties. By
default, the schema scripts for HSQLDB are executed. This behavior can be disabled via
`batch.data.source.init` property.
|
jobRepository
|
A JDBC based `SimpleJobRepository`.
|
This `JobRepository` uses the previously mentioned data source and transaction
manager. The schema's table prefix is configurable (defaults to BATCH_) via the
`batch.table.prefix` property.
|
jobLauncher
|`org.springframework.batch.core.launch.support.SimpleJobLauncher`|
Used to launch jobs.
|
batchJobOperator
|`org.springframework.batch.core.launch.support.SimpleJobOperator`|
The `JsrJobOperator` wraps this to provide most of it's functionality.
|
jobExplorer
|`org.springframework.batch.core.explore.support.JobExplorerFactoryBean`|
Used to address lookup functionality provided by the `JsrJobOperator`.
|
jobParametersConverter
|`org.springframework.batch.core.jsr.JsrJobParametersConverter`|
JSR-352 specific implementation of the `JobParametersConverter`.
|
jobRegistry
|`org.springframework.batch.core.configuration.support.MapJobRegistry`|
Used by the `SimpleJobOperator`.
|
placeholderProperties
|`org.springframework.beans.factory.config.PropertyPlaceholderConfigure`|
Loads the properties file `batch-${ENVIRONMENT:hsql}.properties` to configure
the properties mentioned above. ENVIRONMENT is a System property (defaults to hsql)
that can be used to specify any of the supported databases Spring Batch currently
supports.
|===============
[NOTE]
====
None of the above beans are optional for executing JSR-352 based jobs. All may be overriden to
provide customized functionality as needed.
====
[[dependencyInjection]]
=== Dependency Injection
JSR-352 is based heavily on the Spring Batch programming model. As such, while not explicitly requiring a
formal dependency injection implementation, DI of some kind implied. Spring Batch supports all three
methods for loading batch artifacts defined by JSR-352:
* Implementation Specific Loader - Spring Batch is built upon Spring and so supports Spring
dependency injection within JSR-352 batch jobs.
* Archive Loader - JSR-352 defines the existing of a batch.xml file that provides mappings between a
logical name and a class name. This file must be found within the /META-INF/ directory if it is
used.
* Thread Context Class Loader - JSR-352 allows configurations to specify batch artifact
implementations in their JSL by providing the fully qualified class name inline. Spring Batch
supports this as well in JSR-352 configured jobs.
To use Spring dependency injection within a JSR-352 based batch job consists of configuring batch
artifacts using a Spring application context as beans. Once the beans have been defined, a job can refer to
them as it would any bean defined within the batch.xml.
.XML Configuration
[source, xml, role="xmlContent"]
----
<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://www.springframework.org/schema/beans
https://www.springframework.org/schema/beans/spring-beans.xsd
http://xmlns.jcp.org/xml/ns/javaee
https://xmlns.jcp.org/xml/ns/javaee/jobXML_1_0.xsd">
<!-- javax.batch.api.Batchlet implementation -->
<bean id="fooBatchlet" class="io.spring.FooBatchlet">
<property name="prop" value="bar"/>
</bean>
<!-- Job is defined using the JSL schema provided in JSR-352 -->
<job id="fooJob" xmlns="http://xmlns.jcp.org/xml/ns/javaee" version="1.0">
<step id="step1">
<batchlet ref="fooBatchlet"/>
</step>
</job>
</beans>
----
.Java Configuration
[source, java, role="javaContent"]
----
@Configuration
public class BatchConfiguration {
@Bean
public Batchlet fooBatchlet() {
FooBatchlet batchlet = new FooBatchlet();
batchlet.setProp("bar");
return batchlet;
}
}
<?xml version="1.0" encoding="UTF-8"?>
<job id="fooJob" xmlns="http://xmlns.jcp.org/xml/ns/javaee" version="1.0">
<step id="step1" >
<batchlet ref="fooBatchlet" />
</step>
</job>
----
The assembly of Spring contexts (imports, etc) works with JSR-352 jobs just as it would with any other
Spring based application. The only difference with a JSR-352 based job is that the entry point for the
context definition will be the job definition found in /META-INF/batch-jobs/.
To use the thread context class loader approach, all you need to do is provide the fully qualified class
name as the ref. It is important to note that when using this approach or the batch.xml approach, the class
referenced requires a no argument constructor which will be used to create the bean.
[source, xml]
----
<?xml version="1.0" encoding="UTF-8"?>
<job id="fooJob" xmlns="http://xmlns.jcp.org/xml/ns/javaee" version="1.0">
<step id="step1" >
<batchlet ref="io.spring.FooBatchlet" />
</step>
</job>
----
[[jsrJobProperties]]
=== Batch Properties
[[jsrPropertySupport]]
==== Property Support
JSR-352 allows for properties to be defined at the Job, Step and batch artifact level by way of
configuration in the JSL. Batch properties are configured at each level in the following way:
[source, xml]
----
<properties>
<property name="propertyName1" value="propertyValue1"/>
<property name="propertyName2" value="propertyValue2"/>
</properties>
----
`Properties` may be configured on any batch artifact.
[[jsrBatchPropertyAnnotation]]
==== @BatchProperty annotation
`Properties` are referenced in batch artifacts by annotating class fields with the
`@BatchProperty` and `@Inject` annotations (both annotations
are required by the spec). As defined by JSR-352, fields for properties must be String typed. Any type
conversion is up to the implementing developer to perform.
An `javax.batch.api.chunk.ItemReader` artifact could be configured with a
properties block such as the one described above and accessed as such:
[source, java]
----
public class MyItemReader extends AbstractItemReader {
@Inject
@BatchProperty
private String propertyName1;
...
}
----
The value of the field "propertyName1" will be "propertyValue1"
[[jsrPropertySubstitution]]
==== Property Substitution
Property substitution is provided by way of operators and simple conditional expressions. The general
usage is `#{operator['key']}`.
Supported operators:
* jobParameters - access job parameter values that the job was started/restarted with.
* jobProperties - access properties configured at the job level of the JSL.
* systemProperties - access named system properties.
* partitionPlan - access named property from the partition plan of a partitioned step.
----
#{jobParameters['unresolving.prop']}?:#{systemProperties['file.separator']}
----
The left hand side of the assignment is the expected value, the right hand side is the default value. In
this example, the result will resolve to a value of the system property `file.separator` as
`#{jobParameters['unresolving.prop']}` is assumed to not be resolvable. If neither expressions can be
resolved, an empty String will be returned. Multiple conditions can be used, which are separated by a
';'.
[[jsrProcessingModels]]
=== Processing Models
JSR-352 provides the same two basic processing models that Spring Batch does:
* Item based processing - Using an `javax.batch.api.chunk.ItemReader`, an
optional `javax.batch.api.chunk.ItemProcessor`, and an
`javax.batch.api.chunk.ItemWriter`.
* Task based processing - Using a `javax.batch.api.Batchlet`
implementation. This processing model is the same as the
`org.springframework.batch.core.step.tasklet.Tasklet` based processing
currently available.
==== Item based processing
Item based processing in this context is a chunk size being set by the number of items read by an
`ItemReader`. To configure a step this way, specify the
`item-count` (which defaults to 10) and optionally configure the
`checkpoint-policy` as item (this is the default).
[source, xml]
----
...
<step id="step1">
<chunk checkpoint-policy="item" item-count="3">
<reader ref="fooReader"/>
<processor ref="fooProcessor"/>
<writer ref="fooWriter"/>
</chunk>
</step>
...
----
If item based checkpointing is chosen, an additional attribute `time-limit` is
supported. This sets a time limit for how long the number of items specified has to be processed. If
the timeout is reached, the chunk will complete with however many items have been read by then
regardless of what the `item-count` is configured to be.
==== Custom checkpointing
JSR-352 calls the process around the commit interval within a step "checkpointing". Item based
checkpointing is one approach as mentioned above. However, this will not be robust enough in many
cases. Because of this, the spec allows for the implementation of a custom checkpointing algorithm by
implementing the `javax.batch.api.chunk.CheckpointAlgorithm` interface. This
functionality is functionally the same as Spring Batch's custom completion policy. To use an
implementation of `CheckpointAlgorithm`, configure your step with the custom
`checkpoint-policy` as shown below where `fooCheckpointer` refers to an
implementation of `CheckpointAlgorithm`.
[source, xml]
----
...
<step id="step1">
<chunk checkpoint-policy="custom">
<checkpoint-algorithm ref="fooCheckpointer"/>
<reader ref="fooReader"/>
<processor ref="fooProcessor"/>
<writer ref="fooWriter"/>
</chunk>
</step>
...
----
[[jsrRunningAJob]]
=== Running a job
The entrance to executing a JSR-352 based job is through the
`javax.batch.operations.JobOperator`. Spring Batch provides its own implementation of
this interface (`org.springframework.batch.core.jsr.launch.JsrJobOperator`). This
implementation is loaded via the `javax.batch.runtime.BatchRuntime`. Launching a
JSR-352 based batch job is implemented as follows:
[source, java]
----
JobOperator jobOperator = BatchRuntime.getJobOperator();
long jobExecutionId = jobOperator.start("fooJob", new Properties());
----
The above code does the following:
* Bootstraps a base `ApplicationContext` - In order to provide batch functionality, the framework
needs some infrastructure bootstrapped. This occurs once per JVM. The components that are
bootstrapped are similar to those provided by `@EnableBatchProcessing`.
Specific details can be found in the javadoc for the `JsrJobOperator`.
* Loads an `ApplicationContext` for the job requested - In the example
above, the framework will look in /META-INF/batch-jobs for a file named fooJob.xml and load a
context that is a child of the shared context mentioned previously.
* Launch the job - The job defined within the context will be executed asynchronously. The
`JobExecution's` id will be returned.
[NOTE]
====
All JSR-352 based batch jobs are executed asynchronously.
====
When `JobOperator#start` is called using `SimpleJobOperator`,
Spring Batch determines if the call is an initial run or a retry of a previously executed run. Using the
JSR-352 based `JobOperator#start(String jobXMLName, Properties jobParameters)`, the
framework will always create a new JobInstance (JSR-352 job parameters are
non-identifying). In order to restart a job, a call to
`JobOperator#restart(long executionId, Properties restartParameters)` is required.
[[jsrContexts]]
=== Contexts
JSR-352 defines two context objects that are used to interact with the meta-data of a job or step from
within a batch artifact: `javax.batch.runtime.context.JobContext` and
`javax.batch.runtime.context.StepContext`. Both of these are available in any step
level artifact (`Batchlet`, `ItemReader`, etc) with the
`JobContext` being available to job level artifacts as well
(`JobListener` for example).
To obtain a reference to the `JobContext` or `StepContext`
within the current scope, simply use the `@Inject` annotation:
[source, java]
----
@Inject
JobContext jobContext;
----
[NOTE]
.@Autowire for JSR-352 contexts
====
Using Spring's @Autowire is not supported for the injection of these contexts.
====
In Spring Batch, the `JobContext` and `StepContext` wrap their
corresponding execution objects (`JobExecution` and
`StepExecution` respectively). Data stored via
`StepContext#setPersistentUserData(Serializable data)` is stored in the
Spring Batch `StepExecution#executionContext`.
[[jsrStepFlow]]
=== Step Flow
Within a JSR-352 based job, the flow of steps works similarly as it does within Spring Batch.
However, there are a few subtle differences:
* Decision's are steps - In a regular Spring Batch job, a decision is a state that does not
have an independent `StepExecution` or any of the rights and
responsibilities that go along with being a full step.. However, with JSR-352, a decision
is a step just like any other and will behave just as any other steps (transactionality,
it gets a `StepExecution`, etc). This means that they are treated the
same as any other step on restarts as well.
* `next` attribute and step transitions - In a regular job, these are
allowed to appear together in the same step. JSR-352 allows them to both be used in the
same step with the next attribute taking precedence in evaluation.
* Transition element ordering - In a standard Spring Batch job, transition elements are
sorted from most specific to least specific and evaluated in that order. JSR-352 jobs
evaluate transition elements in the order they are specified in the XML.
[[jsrScaling]]
=== Scaling a JSR-352 batch job
Traditional Spring Batch jobs have four ways of scaling (the last two capable of being executed across
multiple JVMs):
* Split - Running multiple steps in parallel.
* Multiple threads - Executing a single step via multiple threads.
* Partitioning - Dividing the data up for parallel processing (manager/worker).
* Remote Chunking - Executing the processor piece of logic remotely.
JSR-352 provides two options for scaling batch jobs. Both options support only a single JVM:
* Split - Same as Spring Batch
* Partitioning - Conceptually the same as Spring Batch however implemented slightly different.
[[jsrPartitioning]]
==== Partitioning
Conceptually, partitioning in JSR-352 is the same as it is in Spring Batch. Meta-data is provided
to each worker to identify the input to be processed, with the workers reporting back to the manager the
results upon completion. However, there are some important differences:
* Partitioned `Batchlet` - This will run multiple instances of the
configured `Batchlet` on multiple threads. Each instance will have
it's own set of properties as provided by the JSL or the
`PartitionPlan`
* `PartitionPlan` - With Spring Batch's partitioning, an
`ExecutionContext` is provided for each partition. With JSR-352, a
single `javax.batch.api.partition.PartitionPlan` is provided with an
array of `Properties` providing the meta-data for each partition.
* `PartitionMapper` - JSR-352 provides two ways to generate partition
meta-data. One is via the JSL (partition properties). The second is via an implementation
of the `javax.batch.api.partition.PartitionMapper` interface.
Functionally, this interface is similar to the
`org.springframework.batch.core.partition.support.Partitioner`
interface provided by Spring Batch in that it provides a way to programmatically generate
meta-data for partitioning.
* `StepExecutions` - In Spring Batch, partitioned steps are run as
manager/worker. Within JSR-352, the same configuration occurs. However, the worker steps do
not get official `StepExecutions`. Because of that, calls to
`JsrJobOperator#getStepExecutions(long jobExecutionId)` will only
return the `StepExecution` for the manager.
[NOTE]
====
The child `StepExecutions` still exist in the job repository and are available
via the `JobExplorer` and Spring Batch Admin.
====
* Compensating logic - Since Spring Batch implements the manager/worker logic of
partitioning using steps, `StepExecutionListeners` can be used to
handle compensating logic if something goes wrong. However, since the workers JSR-352
provides a collection of other components for the ability to provide compensating logic when
errors occur and to dynamically set the exit status. These components include the following:
|===============
|__Artifact Interface__|__Description__
|`javax.batch.api.partition.PartitionCollector`|Provides a way for worker steps to send information back to the
manager. There is one instance per worker thread.
|`javax.batch.api.partition.PartitionAnalyzer`|End point that receives the information collected by the
`PartitionCollector` as well as the resulting
statuses from a completed partition.
|`javax.batch.api.partition.PartitionReducer`|Provides the ability to provide compensating logic for a partitioned
step.
|===============
[[jsrTesting]]
=== Testing
Since all JSR-352 based jobs are executed asynchronously, it can be difficult to determine when a job has
completed. To help with testing, Spring Batch provides the
`org.springframework.batch.test.JsrTestUtils`. This utility class provides the
ability to start a job and restart a job and wait for it to complete. Once the job completes, the
associated `JobExecution` is returned.

69
docs/asciidoc/monitoring-and-metrics.adoc Normal file
View File

@ -0,0 +1,69 @@
:batch-asciidoc: ./
:toc: left
:toclevels: 4
[[monitoring-and-metrics]]
== Monitoring and metrics
Since version 4.2, Spring Batch provides support for batch monitoring and metrics
based on link:$$https://micrometer.io/$$[Micrometer]. This section describes
which metrics are provided out-of-the-box and how to contribute custom metrics.
[[built-in-metrics]]
=== Built-in metrics
Metrics collection does not require any specific configuration. All metrics provided
by the framework are registered in
link:$$https://micrometer.io/docs/concepts#_global_registry$$[Micrometer's global registry]
under the `spring.batch` prefix. The following table explains all the metrics in details:
|===============
|__Metric Name__|__Type__|__Description__
|`spring.batch.job`|`TIMER`|Duration of job execution
|`spring.batch.job.active`|`LONG_TASK_TIMER`|Currently active jobs
|`spring.batch.step`|`TIMER`|Duration of step execution
|`spring.batch.item.read`|`TIMER`|Duration of item reading
|`spring.batch.item.process`|`TIMER`|Duration of item processing
|`spring.batch.chunk.write`|`TIMER`|Duration of chunk writing
|===============
[[custom-metrics]]
=== Custom metrics
If you want to use your own metrics in your custom components, we recommend using
Micrometer APIs directly. The following is an example of how to time a `Tasklet`:
[source, java]
----
import io.micrometer.core.instrument.Metrics;
import io.micrometer.core.instrument.Timer;
import org.springframework.batch.core.StepContribution;
import org.springframework.batch.core.scope.context.ChunkContext;
import org.springframework.batch.core.step.tasklet.Tasklet;
import org.springframework.batch.repeat.RepeatStatus;
public class MyTimedTasklet implements Tasklet {
@Override
public RepeatStatus execute(StepContribution contribution, ChunkContext chunkContext) {
Timer.Sample sample = Timer.start(Metrics.globalRegistry);
String status = "success";
try {
// do some work
} catch (Exception e) {
// handle exception
status = "failure";
} finally {
sample.stop(Timer.builder("my.tasklet.timer")
.description("Duration of MyTimedTasklet")
.tag("status", status)
.register(Metrics.globalRegistry));
}
return RepeatStatus.FINISHED;
}
}
----

3259
docs/asciidoc/readersAndWriters.adoc Normal file
View File

File diff suppressed because it is too large Load Diff

274
docs/asciidoc/repeat.adoc Normal file
View File

@ -0,0 +1,274 @@
:batch-asciidoc: ./
:toc: left
:toclevels: 4
[[repeat]]
== Repeat
ifndef::onlyonetoggle[]
include::toggle.adoc[]
endif::onlyonetoggle[]
[[repeatTemplate]]
=== RepeatTemplate
Batch processing is about repetitive actions, either as a simple optimization or as part
of a job. To strategize and generalize the repetition and to provide what amounts to an
iterator framework, Spring Batch has the `RepeatOperations` interface. The
`RepeatOperations` interface has the following definition:
[source, java]
----
public interface RepeatOperations {
RepeatStatus iterate(RepeatCallback callback) throws RepeatException;
}
----
The callback is an interface, shown in the following definition, that lets you insert
some business logic to be repeated:
[source, java]
----
public interface RepeatCallback {
RepeatStatus doInIteration(RepeatContext context) throws Exception;
}
----
The callback is executed repeatedly until the implementation determines that the
iteration should end. The return value in these interfaces is an enumeration that can
either be `RepeatStatus.CONTINUABLE` or `RepeatStatus.FINISHED`. A `RepeatStatus`
enumeration conveys information to the caller of the repeat operations about whether
there is any more work to do. Generally speaking, implementations of `RepeatOperations`
should inspect the `RepeatStatus` and use it as part of the decision to end the
iteration. Any callback that wishes to signal to the caller that there is no more work to
do can return `RepeatStatus.FINISHED`.
The simplest general purpose implementation of `RepeatOperations` is `RepeatTemplate`, as
shown in the following example:
[source, java]
----
RepeatTemplate template = new RepeatTemplate();
template.setCompletionPolicy(new SimpleCompletionPolicy(2));
template.iterate(new RepeatCallback() {
public RepeatStatus doInIteration(RepeatContext context) {
// Do stuff in batch...
return RepeatStatus.CONTINUABLE;
}
});
----
In the preceding example, we return `RepeatStatus.CONTINUABLE`, to show that there is
more work to do. The callback can also return `RepeatStatus.FINISHED`, to signal to the
caller that there is no more work to do. Some iterations can be terminated by
considerations intrinsic to the work being done in the callback. Others are effectively
infinite loops as far as the callback is concerned and the completion decision is
delegated to an external policy, as in the case shown in the preceding example.
[[repeatContext]]
==== RepeatContext
The method parameter for the `RepeatCallback` is a `RepeatContext`. Many callbacks ignore
the context. However, if necessary, it can be used as an attribute bag to store transient
data for the duration of the iteration. After the `iterate` method returns, the context
no longer exists.
If there is a nested iteration in progress, a `RepeatContext` has a parent context. The
parent context is occasionally useful for storing data that need to be shared between
calls to `iterate`. This is the case, for instance, if you want to count the number of
occurrences of an event in the iteration and remember it across subsequent calls.
[[repeatStatus]]
==== RepeatStatus
`RepeatStatus` is an enumeration used by Spring Batch to indicate whether processing has
finished. It has two possible `RepeatStatus` values, described in the following table:
.RepeatStatus Properties
|===============
|__Value__|__Description__
|CONTINUABLE|There is more work to do.
|FINISHED|No more repetitions should take place.
|===============
`RepeatStatus` values can also be combined with a logical AND operation by using the
`and()` method in `RepeatStatus`. The effect of this is to do a logical AND on the
continuable flag. In other words, if either status is `FINISHED`, then the result is
`FINISHED`.
[[completionPolicies]]
=== Completion Policies
Inside a `RepeatTemplate`, the termination of the loop in the `iterate` method is
determined by a `CompletionPolicy`, which is also a factory for the `RepeatContext`. The
`RepeatTemplate` has the responsibility to use the current policy to create a
`RepeatContext` and pass that in to the `RepeatCallback` at every stage in the iteration.
After a callback completes its `doInIteration`, the `RepeatTemplate` has to make a call
to the `CompletionPolicy` to ask it to update its state (which will be stored in the
`RepeatContext`). Then it asks the policy if the iteration is complete.
Spring Batch provides some simple general purpose implementations of `CompletionPolicy`.
`SimpleCompletionPolicy` allows execution up to a fixed number of times (with
`RepeatStatus.FINISHED` forcing early completion at any time).
Users might need to implement their own completion policies for more complicated
decisions. For example, a batch processing window that prevents batch jobs from executing
once the online systems are in use would require a custom policy.
[[repeatExceptionHandling]]
=== Exception Handling
If there is an exception thrown inside a `RepeatCallback`, the `RepeatTemplate` consults
an `ExceptionHandler`, which can decide whether or not to re-throw the exception.
The following listing shows the `ExceptionHandler` interface definition:
[source, java]
----
public interface ExceptionHandler {
void handleException(RepeatContext context, Throwable throwable)
throws Throwable;
}
----
A common use case is to count the number of exceptions of a given type and fail when a
limit is reached. For this purpose, Spring Batch provides the
`SimpleLimitExceptionHandler` and a slightly more flexible
`RethrowOnThresholdExceptionHandler`. The `SimpleLimitExceptionHandler` has a limit
property and an exception type that should be compared with the current exception. All
subclasses of the provided type are also counted. Exceptions of the given type are
ignored until the limit is reached, and then they are rethrown. Exceptions of other types
are always rethrown.
An important optional property of the `SimpleLimitExceptionHandler` is the boolean flag
called `useParent`. It is `false` by default, so the limit is only accounted for in the
current `RepeatContext`. When set to `true`, the limit is kept across sibling contexts in
a nested iteration (such as a set of chunks inside a step).
[[repeatListeners]]
=== Listeners
Often, it is useful to be able to receive additional callbacks for cross-cutting concerns
across a number of different iterations. For this purpose, Spring Batch provides the
`RepeatListener` interface. The `RepeatTemplate` lets users register `RepeatListener`
implementations, and they are given callbacks with the `RepeatContext` and `RepeatStatus`
where available during the iteration.
The `RepeatListener` interface has the following definition:
[source, java]
----
public interface RepeatListener {
void before(RepeatContext context);
void after(RepeatContext context, RepeatStatus result);
void open(RepeatContext context);
void onError(RepeatContext context, Throwable e);
void close(RepeatContext context);
}
----
The `open` and `close` callbacks come before and after the entire iteration. `before`,
`after`, and `onError` apply to the individual `RepeatCallback` calls.
Note that, when there is more than one listener, they are in a list, so there is an
order. In this case, `open` and `before` are called in the same order while `after`,
`onError`, and `close` are called in reverse order.
[[repeatParallelProcessing]]
=== Parallel Processing
Implementations of `RepeatOperations` are not restricted to executing the callback
sequentially. It is quite important that some implementations are able to execute their
callbacks in parallel. To this end, Spring Batch provides the
`TaskExecutorRepeatTemplate`, which uses the Spring `TaskExecutor` strategy to run the
`RepeatCallback`. The default is to use a `SynchronousTaskExecutor`, which has the effect
of executing the whole iteration in the same thread (the same as a normal
`RepeatTemplate`).
[[declarativeIteration]]
=== Declarative Iteration
Sometimes there is some business processing that you know you want to repeat every time
it happens. The classic example of this is the optimization of a message pipeline. It is
more efficient to process a batch of messages, if they are arriving frequently, than to
bear the cost of a separate transaction for every message. Spring Batch provides an AOP
interceptor that wraps a method call in a `RepeatOperations` object for just this
purpose. The `RepeatOperationsInterceptor` executes the intercepted method and repeats
according to the `CompletionPolicy` in the provided `RepeatTemplate`.
[role="xmlContent"]
The following example shows declarative iteration using the Spring AOP namespace to
repeat a service call to a method called `processMessage` (for more detail on how to
configure AOP interceptors, see the Spring User Guide):
[source, xml, role="xmlContent"]
----
<aop:config>
<aop:pointcut id="transactional"
expression="execution(* com..*Service.processMessage(..))" />
<aop:advisor pointcut-ref="transactional"
advice-ref="retryAdvice" order="-1"/>
</aop:config>
<bean id="retryAdvice" class="org.spr...RepeatOperationsInterceptor"/>
----
[role="javaContent"]
The following example demonstrates using java configuration to
repeat a service call to a method called `processMessage` (for more detail on how to
configure AOP interceptors, see the Spring User Guide):
[source, java, role="javaContent"]
----
@Bean
public MyService myService() {
ProxyFactory factory = new ProxyFactory(RepeatOperations.class.getClassLoader());
factory.setInterfaces(MyService.class);
factory.setTarget(new MyService());
MyService service = (MyService) factory.getProxy();
JdkRegexpMethodPointcut pointcut = new JdkRegexpMethodPointcut();
pointcut.setPatterns(".*processMessage.*");
RepeatOperationsInterceptor interceptor = new RepeatOperationsInterceptor();
((Advised) service).addAdvisor(new DefaultPointcutAdvisor(pointcut, interceptor));
return service;
}
----
The preceding example uses a default `RepeatTemplate` inside the interceptor. To change
the policies, listeners, and other details, you can inject an instance of
`RepeatTemplate` into the interceptor.
If the intercepted method returns `void`, then the interceptor always returns
`RepeatStatus.CONTINUABLE` (so there is a danger of an infinite loop if the
`CompletionPolicy` does not have a finite end point). Otherwise, it returns
`RepeatStatus.CONTINUABLE` until the return value from the intercepted method is `null`,
at which point it returns `RepeatStatus.FINISHED`. Consequently, the business logic
inside the target method can signal that there is no more work to do by returning `null`
or by throwing an exception that is re-thrown by the `ExceptionHandler` in the provided
`RepeatTemplate`.

456
docs/asciidoc/retry.adoc Normal file
View File

@ -0,0 +1,456 @@
:batch-asciidoc: ./
:toc: left
:toclevels: 4
[[retry]]
== Retry
ifndef::onlyonetoggle[]
include::toggle.adoc[]
endif::onlyonetoggle[]
To make processing more robust and less prone to failure, it sometimes
helps to automatically retry a failed operation in case it might
succeed on a subsequent attempt. Errors that are susceptible to intermittent failure
are often transient in nature. Examples include remote calls to a web
service that fails because of a network glitch or a
`DeadlockLoserDataAccessException` in a database update.
[[retryTemplate]]
=== `RetryTemplate`
[NOTE]
====
The retry functionality was pulled out of Spring Batch as of 2.2.0.
It is now part of a new library, https://github.com/spring-projects/spring-retry[Spring Retry].
====
To automate retry
operations Spring Batch has the `RetryOperations`
strategy. The following interface definition for `RetryOperations`:
[source, java]
----
public interface RetryOperations {
<T, E extends Throwable> T execute(RetryCallback<T, E> retryCallback) throws E;
<T, E extends Throwable> T execute(RetryCallback<T, E> retryCallback, RecoveryCallback<T> recoveryCallback)
throws E;
<T, E extends Throwable> T execute(RetryCallback<T, E> retryCallback, RetryState retryState)
throws E, ExhaustedRetryException;
<T, E extends Throwable> T execute(RetryCallback<T, E> retryCallback, RecoveryCallback<T> recoveryCallback,
RetryState retryState) throws E;
}
----
The basic callback is a simple interface that lets you
insert some business logic to be retried, as shown in the following interface definition:
[source, java]
----
public interface RetryCallback<T, E extends Throwable> {
T doWithRetry(RetryContext context) throws E;
}
----
The callback runs and, if it fails (by throwing an
`Exception`), it is retried until either it is
successful or the implementation aborts. There are a number of
overloaded `execute` methods in the
`RetryOperations` interface. Those methods deal with various use
cases for recovery when all retry attempts are exhausted and deal with
retry state, which allows clients and implementations to store information
between calls (we cover this in more detail later in the chapter).
The simplest general purpose implementation of
`RetryOperations` is
`RetryTemplate`. It can be used as follows:
[source, java]
----
RetryTemplate template = new RetryTemplate();
TimeoutRetryPolicy policy = new TimeoutRetryPolicy();
policy.setTimeout(30000L);
template.setRetryPolicy(policy);
Foo result = template.execute(new RetryCallback<Foo>() {
public Foo doWithRetry(RetryContext context) {
// Do stuff that might fail, e.g. webservice operation
return result;
}
});
----
In the preceding example, we make a web service call and return the result
to the user. If that call fails, then it is retried until a timeout is
reached.
[[retryContext]]
==== `RetryContext`
The method parameter for the `RetryCallback`
is a `RetryContext`. Many callbacks
ignore the context, but, if necessary, it can be used as an attribute bag
to store data for the duration of the iteration.
A `RetryContext` has a parent context
if there is a nested retry in progress in the same thread. The parent
context is occasionally useful for storing data that need to be shared
between calls to `execute`.
[[recoveryCallback]]
==== `RecoveryCallback`
When a retry is exhausted, the
`RetryOperations` can pass control to a different
callback, called the `RecoveryCallback`. To use this
feature, clients pass in the callbacks together to the same method,
as shown in the following example:
[source, java]
----
Foo foo = template.execute(new RetryCallback<Foo>() {
public Foo doWithRetry(RetryContext context) {
// business logic here
},
new RecoveryCallback<Foo>() {
Foo recover(RetryContext context) throws Exception {
// recover logic here
}
});
----
If the business logic does not succeed before the template
decides to abort, then the client is given the chance to do some
alternate processing through the recovery callback.
[[statelessRetry]]
==== Stateless Retry
In the simplest case, a retry is just a while loop. The
`RetryTemplate` can just keep trying until it
either succeeds or fails. The `RetryContext`
contains some state to determine whether to retry or abort, but this
state is on the stack and there is no need to store it anywhere
globally, so we call this stateless retry. The distinction between
stateless and stateful retry is contained in the implementation of the
`RetryPolicy` (the
`RetryTemplate` can handle both). In a stateless
retry, the retry callback is always executed in the same thread it was on
when it failed.
[[statefulRetry]]
==== Stateful Retry
Where the failure has caused a transactional resource to become
invalid, there are some special considerations. This does not apply to a
simple remote call because there is no transactional resource (usually),
but it does sometimes apply to a database update, especially when using
Hibernate. In this case it only makes sense to re-throw the exception
that called the failure immediately, so that the transaction can roll
back and we can start a new, valid transaction.
In cases involving transactions, a stateless retry is not good enough, because the
re-throw and roll back necessarily involve leaving the
`RetryOperations.execute()` method and potentially losing the
context that was on the stack. To avoid losing it we have to introduce a
storage strategy to lift it off the stack and put it (at a minimum) in
heap storage. For this purpose, Spring Batch provides a storage strategy called
`RetryContextCache`, which can be injected into the
`RetryTemplate`. The default implementation of the
`RetryContextCache` is in memory, using a simple
`Map`. Advanced usage with multiple processes in a
clustered environment might also consider implementing the
`RetryContextCache` with a cluster cache of some
sort (however, even in a clustered environment, this might be
overkill).
Part of the responsibility of the
`RetryOperations` is to recognize the failed
operations when they come back in a new execution (and usually wrapped
in a new transaction). To facilitate this, Spring Batch provides the
`RetryState` abstraction. This works in conjunction
with a special `execute` methods in the
`RetryOperations` interface.
The way the failed operations are recognized is by identifying the
state across multiple invocations of the retry. To identify the state,
the user can provide a `RetryState` object that is
responsible for returning a unique key identifying the item. The
identifier is used as a key in the
`RetryContextCache` interface.
[WARNING]
====
Be very careful with the implementation of
`Object.equals()` and `Object.hashCode()` in the
key returned by `RetryState`. The best advice is
to use a business key to identify the items. In the case of a JMS
message, the message ID can be used.
====
When the retry is exhausted, there is also the option to handle the
failed item in a different way, instead of calling the
`RetryCallback` (which is now presumed to be likely
to fail). Just like in the stateless case, this option is provided by
the `RecoveryCallback`, which can be provided by
passing it in to the `execute` method of
`RetryOperations`.
The decision to retry or not is actually delegated to a regular
`RetryPolicy`, so the usual concerns about limits
and timeouts can be injected there (described later in this chapter).
[[retryPolicies]]
=== Retry Policies
Inside a `RetryTemplate`, the decision to retry
or fail in the `execute` method is determined by a
`RetryPolicy`, which is also a factory for the
`RetryContext`. The
`RetryTemplate` has the responsibility to use the
current policy to create a `RetryContext` and pass
that in to the `RetryCallback` at every attempt.
After a callback fails, the `RetryTemplate` has to
make a call to the `RetryPolicy` to ask it to update
its state (which is stored in the
`RetryContext`) and then asks the policy if
another attempt can be made. If another attempt cannot be made (such as when a
limit is reached or a timeout is detected) then the policy is also
responsible for handling the exhausted state. Simple implementations
throw `RetryExhaustedException`, which causes
any enclosing transaction to be rolled back. More sophisticated
implementations might attempt to take some recovery action, in which case
the transaction can remain intact.
[TIP]
====
Failures are inherently either retryable or not. If the same
exception is always going to be thrown from the business logic, it
does no good to retry it. So do not retry on all exception types. Rather, try to
focus on only those exceptions that you expect to be retryable. It is not
usually harmful to the business logic to retry more aggressively, but
it is wasteful, because, if a failure is deterministic, you spend time
retrying something that you know in advance is fatal.
====
Spring Batch provides some simple general purpose implementations of
stateless `RetryPolicy`, such as
`SimpleRetryPolicy` and
`TimeoutRetryPolicy` (used in the preceding example).
The `SimpleRetryPolicy` allows a retry on
any of a named list of exception types, up to a fixed number of times. It
also has a list of "fatal" exceptions that should never be retried, and
this list overrides the retryable list so that it can be used to give
finer control over the retry behavior, as shown in the following example:
[source, java]
----
SimpleRetryPolicy policy = new SimpleRetryPolicy();
// Set the max retry attempts
policy.setMaxAttempts(5);
// Retry on all exceptions (this is the default)
policy.setRetryableExceptions(new Class[] {Exception.class});
// ... but never retry IllegalStateException
policy.setFatalExceptions(new Class[] {IllegalStateException.class});
// Use the policy...
RetryTemplate template = new RetryTemplate();
template.setRetryPolicy(policy);
template.execute(new RetryCallback<Foo>() {
public Foo doWithRetry(RetryContext context) {
// business logic here
}
});
----
There is also a more flexible implementation called
`ExceptionClassifierRetryPolicy`, which allows the
user to configure different retry behavior for an arbitrary set of
exception types though the `ExceptionClassifier`
abstraction. The policy works by calling on the classifier to convert an
exception into a delegate `RetryPolicy`. For
example, one exception type can be retried more times before failure than
another by mapping it to a different policy.
Users might need to implement their own retry policies for more
customized decisions. For instance, a custom retry policy makes sense when there is a well-known,
solution-specific classification of exceptions into retryable and not
retryable.
[[backoffPolicies]]
=== Backoff Policies
When retrying after a transient failure, it often helps to wait a bit
before trying again, because usually the failure is caused by some problem
that can only be resolved by waiting. If a
`RetryCallback` fails, the
`RetryTemplate` can pause execution according to the
`BackoffPolicy`.
The following code shows the interface definition for the `BackOffPolicy` interface:
[source, java]
----
public interface BackoffPolicy {
BackOffContext start(RetryContext context);
void backOff(BackOffContext backOffContext)
throws BackOffInterruptedException;
}
----
A `BackoffPolicy` is free to implement
the backOff in any way it chooses. The policies provided by Spring Batch
out of the box all use `Object.wait()`. A common use case is to
backoff with an exponentially increasing wait period, to avoid two retries
getting into lock step and both failing (this is a lesson learned from
ethernet). For this purpose, Spring Batch provides the
`ExponentialBackoffPolicy`.
[[retryListeners]]
=== Listeners
Often, it is useful to be able to receive additional callbacks for
cross cutting concerns across a number of different retries. For this
purpose, Spring Batch provides the `RetryListener`
interface. The `RetryTemplate` lets users
register `RetryListeners`, and they are given
callbacks with `RetryContext` and
`Throwable` where available during the
iteration.
The following code shows the interface definition for `RetryListener`:
[source, java]
----
public interface RetryListener {
<T, E extends Throwable> boolean open(RetryContext context, RetryCallback<T, E> callback);
<T, E extends Throwable> void onError(RetryContext context, RetryCallback<T, E> callback, Throwable throwable);
<T, E extends Throwable> void close(RetryContext context, RetryCallback<T, E> callback, Throwable throwable);
}
----
The `open` and
`close` callbacks come before and after the entire
retry in the simplest case, and `onError` applies to
the individual `RetryCallback` calls. The
`close` method might also receive a
`Throwable`. If there has been an error, it is the
last one thrown by the `RetryCallback`.
Note that, when there is more than one listener, they are in a list,
so there is an order. In this case, `open` is
called in the same order while `onError` and
`close` are called in reverse order.
[[declarativeRetry]]
=== Declarative Retry
Sometimes, there is some business processing that you know you want
to retry every time it happens. The classic example of this is the remote
service call. Spring Batch provides an AOP interceptor that wraps a method
call in a `RetryOperations` implementation for just this purpose.
The `RetryOperationsInterceptor` executes the
intercepted method and retries on failure according to the
`RetryPolicy` in the provided
`RetryTemplate`.
[role="xmlContent"]
The following example shows a declarative retry that uses the Spring AOP
namespace to retry a service call to a method called
`remoteCall` (for more detail on how to configure
AOP interceptors, see the Spring User Guide):
[source, xml, role="xmlContent"]
----
<aop:config>
<aop:pointcut id="transactional"
expression="execution(* com..*Service.remoteCall(..))" />
<aop:advisor pointcut-ref="transactional"
advice-ref="retryAdvice" order="-1"/>
</aop:config>
<bean id="retryAdvice"
class="org.springframework.batch.retry.interceptor.RetryOperationsInterceptor"/>
----
[role="javaContent"]
The following example shows a declarative retry that uses java configuration
to retry a service call to a method called
`remoteCall` (for more detail on how to configure
AOP interceptors, see the Spring User Guide):
[source, java, role="javaContent"]
----
@Bean
public MyService myService() {
ProxyFactory factory = new ProxyFactory(RepeatOperations.class.getClassLoader());
factory.setInterfaces(MyService.class);
factory.setTarget(new MyService());
MyService service = (MyService) factory.getProxy();
JdkRegexpMethodPointcut pointcut = new JdkRegexpMethodPointcut();
pointcut.setPatterns(".*remoteCall.*");
RetryOperationsInterceptor interceptor = new RetryOperationsInterceptor();
((Advised) service).addAdvisor(new DefaultPointcutAdvisor(pointcut, interceptor));
return service;
}
----
The preceding example uses a default
`RetryTemplate` inside the interceptor. To change the
policies or listeners, you can inject an instance of
`RetryTemplate` into the interceptor.

496
docs/asciidoc/scalability.adoc Normal file
View File

@ -0,0 +1,496 @@
:batch-asciidoc: ./
:toc: left
:toclevels: 4
[[scalability]]
== Scaling and Parallel Processing
ifndef::onlyonetoggle[]
include::toggle.adoc[]
endif::onlyonetoggle[]
Many batch processing problems can be solved with single threaded, single process jobs,
so it is always a good idea to properly check if that meets your needs before thinking
about more complex implementations. Measure the performance of a realistic job and see if
the simplest implementation meets your needs first. You can read and write a file of
several hundred megabytes in well under a minute, even with standard hardware.
When you are ready to start implementing a job with some parallel processing, Spring
Batch offers a range of options, which are described in this chapter, although some
features are covered elsewhere. At a high level, there are two modes of parallel
processing:
* Single process, multi-threaded
* Multi-process
These break down into categories as well, as follows:
* Multi-threaded Step (single process)
* Parallel Steps (single process)
* Remote Chunking of Step (multi process)
* Partitioning a Step (single or multi process)
First, we review the single-process options. Then we review the multi-process options.
[[multithreadedStep]]
=== Multi-threaded Step
The simplest way to start parallel processing is to add a `TaskExecutor` to your Step
configuration.
[role="xmlContent"]
For example, you might add an attribute of the `tasklet`, as shown in the
following example:
[source, xml, role="xmlContent"]
----
<step id="loading">
<tasklet task-executor="taskExecutor">...</tasklet>
</step>
----
[role="javaContent"]
When using java configuration, a `TaskExecutor` can be added to the step
as shown in the following example:
.Java Configuration
[source, java, role="javaContent"]
----
@Bean
public TaskExecutor taskExecutor(){
return new SimpleAsyncTaskExecutor("spring_batch");
}
@Bean
public Step sampleStep(TaskExecutor taskExecutor) {
return this.stepBuilderFactory.get("sampleStep")
.<String, String>chunk(10)
.reader(itemReader())
.writer(itemWriter())
.taskExecutor(taskExecutor)
.build();
}
----
In this example, the `taskExecutor` is a reference to another bean definition that
implements the `TaskExecutor` interface.
https://docs.spring.io/spring/docs/current/javadoc-api/org/springframework/core/task/TaskExecutor.html[`TaskExecutor`]
is a standard Spring interface, so consult the Spring User Guide for details of available
implementations. The simplest multi-threaded `TaskExecutor` is a
`SimpleAsyncTaskExecutor`.
The result of the above configuration is that the `Step` executes by reading, processing,
and writing each chunk of items (each commit interval) in a separate thread of execution.
Note that this means there is no fixed order for the items to be processed, and a chunk
might contain items that are non-consecutive compared to the single-threaded case. In
addition to any limits placed by the task executor (such as whether it is backed by a
thread pool), there is a throttle limit in the tasklet configuration which defaults to 4.
You may need to increase this to ensure that a thread pool is fully utilized.
[role="xmlContent"]
For example you might increase the throttle-limit, as shown in the following example:
[source, xml, role="xmlContent"]
----
<step id="loading"> <tasklet
task-executor="taskExecutor"
throttle-limit="20">...</tasklet>
</step>
----
[role="javaContent"]
When using java configuration, the builders provide access to the throttle limit:
.Java Configuration
[source, java, role="javaContent"]
----
@Bean
public Step sampleStep(TaskExecutor taskExecutor) {
return this.stepBuilderFactory.get("sampleStep")
.<String, String>chunk(10)
.reader(itemReader())
.writer(itemWriter())
.taskExecutor(taskExecutor)
.throttleLimit(20)
.build();
}
----
Note also that there may be limits placed on concurrency by any pooled resources used in
your step, such as a `DataSource`. Be sure to make the pool in those resources at least
as large as the desired number of concurrent threads in the step.
There are some practical limitations of using multi-threaded `Step` implementations for
some common batch use cases. Many participants in a `Step` (such as readers and writers)
are stateful. If the state is not segregated by thread, then those components are not
usable in a multi-threaded `Step`. In particular, most of the off-the-shelf readers and
writers from Spring Batch are not designed for multi-threaded use. It is, however,
possible to work with stateless or thread safe readers and writers, and there is a sample
(called `parallelJob`) in the
https://github.com/spring-projects/spring-batch/tree/master/spring-batch-samples[Spring
Batch Samples] that shows the use of a process indicator (see
<<readersAndWriters.adoc#process-indicator,Preventing State Persistence>>) to keep track
of items that have been processed in a database input table.
Spring Batch provides some implementations of `ItemWriter` and `ItemReader`. Usually,
they say in the Javadoc if they are thread safe or not or what you have to do to avoid
problems in a concurrent environment. If there is no information in the Javadoc, you can
check the implementation to see if there is any state. If a reader is not thread safe,
you can decorate it with the provided `SynchronizedItemStreamReader` or use it in your own
synchronizing delegator. You can synchronize the call to `read()` and as long as the
processing and writing is the most expensive part of the chunk, your step may still
complete much faster than it would in a single threaded configuration.
[[scalabilityParallelSteps]]
=== Parallel Steps
As long as the application logic that needs to be parallelized can be split into distinct
responsibilities and assigned to individual steps, then it can be parallelized in a
single process. Parallel Step execution is easy to configure and use.
[role="xmlContent"]
For example, executing steps `(step1,step2)` in parallel with `step3` is straightforward,
as shown in the following example:
[source, xml, role="xmlContent"]
----
<job id="job1">
<split id="split1" task-executor="taskExecutor" next="step4">
<flow>
<step id="step1" parent="s1" next="step2"/>
<step id="step2" parent="s2"/>
</flow>
<flow>
<step id="step3" parent="s3"/>
</flow>
</split>
<step id="step4" parent="s4"/>
</job>
<beans:bean id="taskExecutor" class="org.spr...SimpleAsyncTaskExecutor"/>
----
[role="javaContent"]
When using java configuration, executing steps `(step1,step2)` in parallel with `step3`
is straightforward, as shown in the following example:
.Java Configuration
[source, java, role="javaContent"]
----
@Bean
public Job job() {
return jobBuilderFactory.get("job")
.start(splitFlow())
.next(step4())
.build() //builds FlowJobBuilder instance
.build(); //builds Job instance
}
@Bean
public Flow splitFlow() {
return new FlowBuilder<SimpleFlow>("splitFlow")
.split(taskExecutor())
.add(flow1(), flow2())
.build();
}
@Bean
public Flow flow1() {
return new FlowBuilder<SimpleFlow>("flow1")
.start(step1())
.next(step2())
.build();
}
@Bean
public Flow flow2() {
return new FlowBuilder<SimpleFlow>("flow2")
.start(step3())
.build();
}
@Bean
public TaskExecutor taskExecutor(){
return new SimpleAsyncTaskExecutor("spring_batch");
}
----
The configurable task executor is used to specify which `TaskExecutor`
implementation should be used to execute the individual flows. The default is
`SyncTaskExecutor`, but an asynchronous `TaskExecutor` is required to run the steps in
parallel. Note that the job ensures that every flow in the split completes before
aggregating the exit statuses and transitioning.
See the section on <<step.adoc#split-flows,Split Flows>> for more detail.
[[remoteChunking]]
=== Remote Chunking
In remote chunking, the `Step` processing is split across multiple processes,
communicating with each other through some middleware. The following image shows the
pattern:
.Remote Chunking
image::{batch-asciidoc}images/remote-chunking.png[Remote Chunking, scaledwidth="60%"]
The manager component is a single process, and the workers are multiple remote processes.
This pattern works best if the manager is not a bottleneck, so the processing must be more
expensive than the reading of items (as is often the case in practice).
The manager is an implementation of a Spring Batch `Step` with the `ItemWriter` replaced
by a generic version that knows how to send chunks of items to the middleware as
messages. The workers are standard listeners for whatever middleware is being used (for
example, with JMS, they would be `MesssageListener` implementations), and their role is
to process the chunks of items using a standard `ItemWriter` or `ItemProcessor` plus
`ItemWriter`, through the `ChunkProcessor` interface. One of the advantages of using this
pattern is that the reader, processor, and writer components are off-the-shelf (the same
as would be used for a local execution of the step). The items are divided up dynamically
and work is shared through the middleware, so that, if the listeners are all eager
consumers, then load balancing is automatic.
The middleware has to be durable, with guaranteed delivery and a single consumer for each
message. JMS is the obvious candidate, but other options (such as JavaSpaces) exist in
the grid computing and shared memory product space.
See the section on
<<spring-batch-integration.adoc#remote-chunking,Spring Batch Integration - Remote Chunking>>
for more detail.
[[partitioning]]
=== Partitioning
Spring Batch also provides an SPI for partitioning a `Step` execution and executing it
remotely. In this case, the remote participants are `Step` instances that could just as
easily have been configured and used for local processing. The following image shows the
pattern:
.Partitioning
image::{batch-asciidoc}images/partitioning-overview.png[Partitioning Overview, scaledwidth="60%"]
The `Job` runs on the left-hand side as a sequence of `Step` instances, and one of the
`Step` instances is labeled as a manager. The workers in this picture are all identical
instances of a `Step`, which could in fact take the place of the manager, resulting in the
same outcome for the `Job`. The workers are typically going to be remote services but
could also be local threads of execution. The messages sent by the manager to the workers
in this pattern do not need to be durable or have guaranteed delivery. Spring Batch
metadata in the `JobRepository` ensures that each worker is executed once and only once for
each `Job` execution.
The SPI in Spring Batch consists of a special implementation of `Step` (called the
`PartitionStep`) and two strategy interfaces that need to be implemented for the specific
environment. The strategy interfaces are `PartitionHandler` and `StepExecutionSplitter`,
and their role is shown in the following sequence diagram:
.Partitioning SPI
image::{batch-asciidoc}images/partitioning-spi.png[Partitioning SPI, scaledwidth="60%"]
The `Step` on the right in this case is the "`remote`" worker, so, potentially, there are
many objects and or processes playing this role, and the `PartitionStep` is shown driving
the execution.
[role="xmlContent"]
The following example shows the `PartitionStep` configuration:
[source, xml, role="xmlContent"]
----
<step id="step1.manager">
<partition step="step1" partitioner="partitioner">
<handler grid-size="10" task-executor="taskExecutor"/>
</partition>
</step>
----
[role="javaContent"]
The following example shows the `PartitionStep` configuration using java configuration:
.Java Configuration
[source, java, role="javaContent"]
----
@Bean
public Step step1Manager() {
return stepBuilderFactory.get("step1.manager")
.<String, String>partitioner("step1", partitioner())
.step(step1())
.gridSize(10)
.taskExecutor(taskExecutor())
.build();
}
----
Similar to the multi-threaded step's `throttle-limit` attribute, the `grid-size`
attribute prevents the task executor from being saturated with requests from a single
step.
There is a simple example that can be copied and extended in the unit test suite for
https://github.com/spring-projects/spring-batch/tree/master/spring-batch-samples/src/main/resources/jobs[Spring
Batch Samples] (see `Partition*Job.xml` configuration).
Spring Batch creates step executions for the partitions called "step1:partition0", and so
on. Many people prefer to call the manager step "step1:manager" for consistency. You can
use an alias for the step (by specifying the `name` attribute instead of the `id`
attribute).
[[partitionHandler]]
==== PartitionHandler
The `PartitionHandler` is the component that knows about the fabric of the remoting or
grid environment. It is able to send `StepExecution` requests to the remote `Step`
instances, wrapped in some fabric-specific format, like a DTO. It does not have to know
how to split the input data or how to aggregate the result of multiple `Step` executions.
Generally speaking, it probably also does not need to know about resilience or failover,
since those are features of the fabric in many cases. In any case, Spring Batch always
provides restartability independent of the fabric. A failed `Job` can always be restarted
and only the failed `Steps` are re-executed.
The `PartitionHandler` interface can have specialized implementations for a variety of
fabric types, including simple RMI remoting, EJB remoting, custom web service, JMS, Java
Spaces, shared memory grids (like Terracotta or Coherence), and grid execution fabrics
(like GridGain). Spring Batch does not contain implementations for any proprietary grid
or remoting fabrics.
Spring Batch does, however, provide a useful implementation of `PartitionHandler` that
executes `Step` instances locally in separate threads of execution, using the
`TaskExecutor` strategy from Spring. The implementation is called
`TaskExecutorPartitionHandler`.
[role="xmlContent"]
The `TaskExecutorPartitionHandler` is the default for a step configured with the XML
namespace shown previously. It can also be configured explicitly, as shown in the
following example:
[source, xml, role="xmlContent"]
----
<step id="step1.manager">
<partition step="step1" handler="handler"/>
</step>
<bean class="org.spr...TaskExecutorPartitionHandler">
<property name="taskExecutor" ref="taskExecutor"/>
<property name="step" ref="step1" />
<property name="gridSize" value="10" />
</bean>
----
[role="javaContent"]
The `TaskExecutorPartitionHandler` can be configured explicitly within java configuration,
as shown in the following example:
.Java Configuration
[source, java, role="javaContent"]
----
@Bean
public Step step1Manager() {
return stepBuilderFactory.get("step1.manager")
.partitioner("step1", partitioner())
.partitionHandler(partitionHandler())
.build();
}
@Bean
public PartitionHandler partitionHandler() {
TaskExecutorPartitionHandler retVal = new TaskExecutorPartitionHandler();
retVal.setTaskExecutor(taskExecutor());
retVal.setStep(step1());
retVal.setGridSize(10);
return retVal;
}
----
The `gridSize` attribute determines the number of separate step executions to create, so
it can be matched to the size of the thread pool in the `TaskExecutor`. Alternatively, it
can be set to be larger than the number of threads available, which makes the blocks of
work smaller.
The `TaskExecutorPartitionHandler` is useful for IO-intensive `Step` instances, such as
copying large numbers of files or replicating filesystems into content management
systems. It can also be used for remote execution by providing a `Step` implementation
that is a proxy for a remote invocation (such as using Spring Remoting).
[[partitioner]]
==== Partitioner
The `Partitioner` has a simpler responsibility: to generate execution contexts as input
parameters for new step executions only (no need to worry about restarts). It has a
single method, as shown in the following interface definition:
[source, java]
----
public interface Partitioner {
Map<String, ExecutionContext> partition(int gridSize);
}
----
The return value from this method associates a unique name for each step execution (the
`String`) with input parameters in the form of an `ExecutionContext`. The names show up
later in the Batch metadata as the step name in the partitioned `StepExecutions`. The
`ExecutionContext` is just a bag of name-value pairs, so it might contain a range of
primary keys, line numbers, or the location of an input file. The remote `Step` then
normally binds to the context input using `#{...}` placeholders (late binding in step
scope), as illustrated in the next section.
The names of the step executions (the keys in the `Map` returned by `Partitioner`) need
to be unique amongst the step executions of a `Job` but do not have any other specific
requirements. The easiest way to do this (and to make the names meaningful for users) is
to use a prefix+suffix naming convention, where the prefix is the name of the step that
is being executed (which itself is unique in the `Job`), and the suffix is just a
counter. There is a `SimplePartitioner` in the framework that uses this convention.
An optional interface called `PartitionNameProvider` can be used to provide the partition
names separately from the partitions themselves. If a `Partitioner` implements this
interface, then, on a restart, only the names are queried. If partitioning is expensive,
this can be a useful optimization. The names provided by the `PartitionNameProvider` must
match those provided by the `Partitioner`.
[[bindingInputDataToSteps]]
==== Binding Input Data to Steps
It is very efficient for the steps that are executed by the `PartitionHandler` to have
identical configuration and for their input parameters to be bound at runtime from the
`ExecutionContext`. This is easy to do with the StepScope feature of Spring Batch
(covered in more detail in the section on <<step.adoc#late-binding,Late Binding>>). For
example, if the `Partitioner` creates `ExecutionContext` instances with an attribute key
called `fileName`, pointing to a different file (or directory) for each step invocation,
the `Partitioner` output might resemble the content of the following table:
.Example step execution name to execution context provided by `Partitioner` targeting directory processing
|===============
|__Step Execution Name (key)__|__ExecutionContext (value)__
|filecopy:partition0|fileName=/home/data/one
|filecopy:partition1|fileName=/home/data/two
|filecopy:partition2|fileName=/home/data/three
|===============
Then the file name can be bound to a step using late binding to the execution context, as
shown in the following example:
.XML Configuration
[source, xml, role="xmlContent"]
----
<bean id="itemReader" scope="step"
class="org.spr...MultiResourceItemReader">
<property name="resources" value="#{stepExecutionContext[fileName]}/*"/>
</bean>
----
.Java Configuration
[source, java, role="javaContent"]
----
@Bean
public MultiResourceItemReader itemReader(
@Value("#{stepExecutionContext['fileName']}/*") Resource [] resources) {
return new MultiResourceItemReaderBuilder<String>()
.delegate(fileReader())
.name("itemReader")
.resources(resources)
.build();
}
----

404
docs/asciidoc/schema-appendix.adoc Normal file
View File

@ -0,0 +1,404 @@
:batch-asciidoc: ./
:toc: left
:toclevels: 4
[[metaDataSchema]]
[appendix]
== Meta-Data Schema
[[metaDataSchemaOverview]]
=== Overview
The Spring Batch Metadata tables closely match the Domain objects that represent them in
Java. For example, `JobInstance`, `JobExecution`, `JobParameters`, and `StepExecution`
map to `BATCH_JOB_INSTANCE`, `BATCH_JOB_EXECUTION`, `BATCH_JOB_EXECUTION_PARAMS`, and
`BATCH_STEP_EXECUTION`, respectively. `ExecutionContext` maps to both
`BATCH_JOB_EXECUTION_CONTEXT` and `BATCH_STEP_EXECUTION_CONTEXT`. The `JobRepository` is
responsible for saving and storing each Java object into its correct table. This appendix
describes the metadata tables in detail, along with many of the design decisions that
were made when creating them. When viewing the various table creation statements below,
it is important to realize that the data types used are as generic as possible. Spring
Batch provides many schemas as examples, all of which have varying data types, due to
variations in how individual database vendors handle data types. The following image
shows an ERD model of all 6 tables and their relationships to one another:
.Spring Batch Meta-Data ERD
image::{batch-asciidoc}images/meta-data-erd.png[Spring Batch Meta-Data ERD, scaledwidth="60%"]
[[exampleDDLScripts]]
==== Example DDL Scripts
The Spring Batch Core JAR file contains example scripts to create the relational tables
for a number of database platforms (which are, in turn, auto-detected by the job
repository factory bean or namespace equivalent). These scripts can be used as is or
modified with additional indexes and constraints as desired. The file names are in the
form `schema-\*.sql`, where "*" is the short name of the target database platform.
The scripts are in the package `org.springframework.batch.core`.
[[migrationDDLScripts]]
==== Migration DDL Scripts
Spring Batch provides migration DDL scripts that you need to execute when you upgrade versions.
These scripts can be found in the Core Jar file under `org/springframework/batch/core/migration`.
Migration scripts are organized into folders corresponding to version numbers in which they were introduced:
* `2.2`: contains scripts needed if you are migrating from a version before `2.2` to version `2.2`
* `4.1`: contains scripts needed if you are migrating from a version before `4.1` to version `4.1`
[[metaDataVersion]]
==== Version
Many of the database tables discussed in this appendix contain a version column. This
column is important because Spring Batch employs an optimistic locking strategy when
dealing with updates to the database. This means that each time a record is 'touched'
(updated) the value in the version column is incremented by one. When the repository goes
back to save the value, if the version number has changed it throws an
`OptimisticLockingFailureException`, indicating there has been an error with concurrent
access. This check is necessary, since, even though different batch jobs may be running
in different machines, they all use the same database tables.
[[metaDataIdentity]]
==== Identity
`BATCH_JOB_INSTANCE`, `BATCH_JOB_EXECUTION`, and `BATCH_STEP_EXECUTION` each contain
columns ending in `_ID`. These fields act as primary keys for their respective tables.
However, they are not database generated keys. Rather, they are generated by separate
sequences. This is necessary because, after inserting one of the domain objects into the
database, the key it is given needs to be set on the actual object so that they can be
uniquely identified in Java. Newer database drivers (JDBC 3.0 and up) support this
feature with database-generated keys. However, rather than require that feature,
sequences are used. Each variation of the schema contains some form of the following
statements:
[source, sql]
----
CREATE SEQUENCE BATCH_STEP_EXECUTION_SEQ;
CREATE SEQUENCE BATCH_JOB_EXECUTION_SEQ;
CREATE SEQUENCE BATCH_JOB_SEQ;
----
Many database vendors do not support sequences. In these cases, work-arounds are used,
such as the following statements for MySQL:
[source, sql]
----
CREATE TABLE BATCH_STEP_EXECUTION_SEQ (ID BIGINT NOT NULL) type=InnoDB;
INSERT INTO BATCH_STEP_EXECUTION_SEQ values(0);
CREATE TABLE BATCH_JOB_EXECUTION_SEQ (ID BIGINT NOT NULL) type=InnoDB;
INSERT INTO BATCH_JOB_EXECUTION_SEQ values(0);
CREATE TABLE BATCH_JOB_SEQ (ID BIGINT NOT NULL) type=InnoDB;
INSERT INTO BATCH_JOB_SEQ values(0);
----
In the preceding case, a table is used in place of each sequence. The Spring core class,
`MySQLMaxValueIncrementer`, then increments the one column in this sequence in order to
give similar functionality.
[[metaDataBatchJobInstance]]
=== `BATCH_JOB_INSTANCE`
The `BATCH_JOB_INSTANCE` table holds all information relevant to a `JobInstance`, and
serves as the top of the overall hierarchy. The following generic DDL statement is used
to create it:
[source, sql]
----
CREATE TABLE BATCH_JOB_INSTANCE (
JOB_INSTANCE_ID BIGINT PRIMARY KEY ,
VERSION BIGINT,
JOB_NAME VARCHAR(100) NOT NULL ,
JOB_KEY VARCHAR(2500)
);
----
The following list describes each column in the table:
* `JOB_INSTANCE_ID`: The unique ID that identifies the instance. It is also the primary
key. The value of this column should be obtainable by calling the `getId` method on
`JobInstance`.
* `VERSION`: See <<metaDataVersion>>.
* `JOB_NAME`: Name of the job obtained from the `Job` object. Because it is required to
identify the instance, it must not be null.
* `JOB_KEY`: A serialization of the `JobParameters` that uniquely identifies separate
instances of the same job from one another. (`JobInstances` with the same job name must
have different `JobParameters` and, thus, different `JOB_KEY` values).
[[metaDataBatchJobParams]]
=== `BATCH_JOB_EXECUTION_PARAMS`
The `BATCH_JOB_EXECUTION_PARAMS` table holds all information relevant to the
`JobParameters` object. It contains 0 or more key/value pairs passed to a `Job` and
serves as a record of the parameters with which a job was run. For each parameter that
contributes to the generation of a job's identity, the `IDENTIFYING` flag is set to true.
Note that the table has been denormalized. Rather than creating a separate table for each
type, there is one table with a column indicating the type, as shown in the following
listing:
[source, sql]
----
CREATE TABLE BATCH_JOB_EXECUTION_PARAMS (
JOB_EXECUTION_ID BIGINT NOT NULL ,
TYPE_CD VARCHAR(6) NOT NULL ,
KEY_NAME VARCHAR(100) NOT NULL ,
STRING_VAL VARCHAR(250) ,
DATE_VAL DATETIME DEFAULT NULL ,
LONG_VAL BIGINT ,
DOUBLE_VAL DOUBLE PRECISION ,
IDENTIFYING CHAR(1) NOT NULL ,
constraint JOB_EXEC_PARAMS_FK foreign key (JOB_EXECUTION_ID)
references BATCH_JOB_EXECUTION(JOB_EXECUTION_ID)
);
----
The following list describes each column:
* `JOB_EXECUTION_ID`: Foreign key from the `BATCH_JOB_EXECUTION` table that indicates the
job execution to which the parameter entry belongs. Note that multiple rows (that is,
key/value pairs) may exist for each execution.
* TYPE_CD: String representation of the type of value stored, which can be a string, a
date, a long, or a double. Because the type must be known, it cannot be null.
* KEY_NAME: The parameter key.
* STRING_VAL: Parameter value, if the type is string.
* DATE_VAL: Parameter value, if the type is date.
* LONG_VAL: Parameter value, if the type is long.
* DOUBLE_VAL: Parameter value, if the type is double.
* IDENTIFYING: Flag indicating whether the parameter contributed to the identity of the
related `JobInstance`.
Note that there is no primary key for this table. This is because the framework has no
use for one and, thus, does not require it. If need be, you can add a primary key may be
added with a database generated key without causing any issues to the framework itself.
[[metaDataBatchJobExecution]]
=== `BATCH_JOB_EXECUTION`
The `BATCH_JOB_EXECUTION` table holds all information relevant to the `JobExecution`
object. Every time a `Job` is run, there is always a new `JobExecution`, and a new row in
this table. The following listing shows the definition of the `BATCH_JOB_EXECUTION`
table:
[source, sql]
----
CREATE TABLE BATCH_JOB_EXECUTION (
JOB_EXECUTION_ID BIGINT PRIMARY KEY ,
VERSION BIGINT,
JOB_INSTANCE_ID BIGINT NOT NULL,
CREATE_TIME TIMESTAMP NOT NULL,
START_TIME TIMESTAMP DEFAULT NULL,
END_TIME TIMESTAMP DEFAULT NULL,
STATUS VARCHAR(10),
EXIT_CODE VARCHAR(20),
EXIT_MESSAGE VARCHAR(2500),
LAST_UPDATED TIMESTAMP,
JOB_CONFIGURATION_LOCATION VARCHAR(2500) NULL,
constraint JOB_INSTANCE_EXECUTION_FK foreign key (JOB_INSTANCE_ID)
references BATCH_JOB_INSTANCE(JOB_INSTANCE_ID)
) ;
----
The following list describes each column:
* `JOB_EXECUTION_ID`: Primary key that uniquely identifies this execution. The value of
this column is obtainable by calling the `getId` method of the `JobExecution` object.
* `VERSION`: See <<metaDataVersion>>.
* `JOB_INSTANCE_ID`: Foreign key from the `BATCH_JOB_INSTANCE` table. It indicates the
instance to which this execution belongs. There may be more than one execution per
instance.
* `CREATE_TIME`: Timestamp representing the time when the execution was created.
* `START_TIME`: Timestamp representing the time when the execution was started.
* `END_TIME`: Timestamp representing the time when the execution finished, regardless of
success or failure. An empty value in this column when the job is not currently running
indicates that there has been some type of error and the framework was unable to perform
a last save before failing.
* `STATUS`: Character string representing the status of the execution. This may be
`COMPLETED`, `STARTED`, and others. The object representation of this column is the
`BatchStatus` enumeration.
* `EXIT_CODE`: Character string representing the exit code of the execution. In the case
of a command-line job, this may be converted into a number.
* `EXIT_MESSAGE`: Character string representing a more detailed description of how the
job exited. In the case of failure, this might include as much of the stack trace as is
possible.
* `LAST_UPDATED`: Timestamp representing the last time this execution was persisted.
[[metaDataBatchStepExecution]]
=== `BATCH_STEP_EXECUTION`
The BATCH_STEP_EXECUTION table holds all information relevant to the `StepExecution`
object. This table is similar in many ways to the `BATCH_JOB_EXECUTION` table, and there
is always at least one entry per `Step` for each `JobExecution` created. The following
listing shows the definition of the `BATCH_STEP_EXECUTION` table:
[source, sql]
----
CREATE TABLE BATCH_STEP_EXECUTION (
STEP_EXECUTION_ID BIGINT PRIMARY KEY ,
VERSION BIGINT NOT NULL,
STEP_NAME VARCHAR(100) NOT NULL,
JOB_EXECUTION_ID BIGINT NOT NULL,
START_TIME TIMESTAMP NOT NULL ,
END_TIME TIMESTAMP DEFAULT NULL,
STATUS VARCHAR(10),
COMMIT_COUNT BIGINT ,
READ_COUNT BIGINT ,
FILTER_COUNT BIGINT ,
WRITE_COUNT BIGINT ,
READ_SKIP_COUNT BIGINT ,
WRITE_SKIP_COUNT BIGINT ,
PROCESS_SKIP_COUNT BIGINT ,
ROLLBACK_COUNT BIGINT ,
EXIT_CODE VARCHAR(20) ,
EXIT_MESSAGE VARCHAR(2500) ,
LAST_UPDATED TIMESTAMP,
constraint JOB_EXECUTION_STEP_FK foreign key (JOB_EXECUTION_ID)
references BATCH_JOB_EXECUTION(JOB_EXECUTION_ID)
) ;
----
The following list describes for each column:
* `STEP_EXECUTION_ID`: Primary key that uniquely identifies this execution. The value of
this column should be obtainable by calling the `getId` method of the `StepExecution`
object.
* `VERSION`: See <<metaDataVersion>>.
* `STEP_NAME`: The name of the step to which this execution belongs.
* `JOB_EXECUTION_ID`: Foreign key from the `BATCH_JOB_EXECUTION` table. It indicates the
`JobExecution` to which this `StepExecution` belongs. There may be only one
`StepExecution` for a given `JobExecution` for a given `Step` name.
* `START_TIME`: Timestamp representing the time when the execution was started.
* `END_TIME`: Timestamp representing the time the when execution was finished, regardless
of success or failure. An empty value in this column, even though the job is not
currently running, indicates that there has been some type of error and the framework was
unable to perform a last save before failing.
* `STATUS`: Character string representing the status of the execution. This may be
`COMPLETED`, `STARTED`, and others. The object representation of this column is the
`BatchStatus` enumeration.
* `COMMIT_COUNT`: The number of times in which the step has committed a transaction
during this execution.
* `READ_COUNT`: The number of items read during this execution.
* `FILTER_COUNT`: The number of items filtered out of this execution.
* `WRITE_COUNT`: The number of items written and committed during this execution.
* `READ_SKIP_COUNT`: The number of items skipped on read during this execution.
* `WRITE_SKIP_COUNT`: The number of items skipped on write during this execution.
* `PROCESS_SKIP_COUNT`: The number of items skipped during processing during this
execution.
* `ROLLBACK_COUNT`: The number of rollbacks during this execution. Note that this count
includes each time rollback occurs, including rollbacks for retry and those in the skip
recovery procedure.
* `EXIT_CODE`: Character string representing the exit code of the execution. In the case
of a command-line job, this may be converted into a number.
* `EXIT_MESSAGE`: Character string representing a more detailed description of how the
job exited. In the case of failure, this might include as much of the stack trace as is
possible.
* `LAST_UPDATED`: Timestamp representing the last time this execution was persisted.
[[metaDataBatchJobExecutionContext]]
=== `BATCH_JOB_EXECUTION_CONTEXT`
The `BATCH_JOB_EXECUTION_CONTEXT` table holds all information relevant to the
`ExecutionContext` of a `Job`. There is exactly one `Job` `ExecutionContext` per
`JobExecution`, and it contains all of the job-level data that is needed for a particular
job execution. This data typically represents the state that must be retrieved after a
failure, so that a `JobInstance` can "start from where it left off". The following
listing shows the definition of the `BATCH_JOB_EXECUTION_CONTEXT` table:
[source, sql]
----
CREATE TABLE BATCH_JOB_EXECUTION_CONTEXT (
JOB_EXECUTION_ID BIGINT PRIMARY KEY,
SHORT_CONTEXT VARCHAR(2500) NOT NULL,
SERIALIZED_CONTEXT CLOB,
constraint JOB_EXEC_CTX_FK foreign key (JOB_EXECUTION_ID)
references BATCH_JOB_EXECUTION(JOB_EXECUTION_ID)
) ;
----
The following list describes each column:
* `JOB_EXECUTION_ID`: Foreign key representing the `JobExecution` to which the context
belongs. There may be more than one row associated with a given execution.
* `SHORT_CONTEXT`: A string version of the `SERIALIZED_CONTEXT`.
* `SERIALIZED_CONTEXT`: The entire context, serialized.
[[metaDataBatchStepExecutionContext]]
=== `BATCH_STEP_EXECUTION_CONTEXT`
The `BATCH_STEP_EXECUTION_CONTEXT` table holds all information relevant to the
`ExecutionContext` of a `Step`. There is exactly one `ExecutionContext` per
`StepExecution`, and it contains all of the data that
needs to be persisted for a particular step execution. This data typically represents the
state that must be retrieved after a failure, so that a `JobInstance` can 'start from
where it left off'. The following listing shows the definition of the
`BATCH_STEP_EXECUTION_CONTEXT` table:
[source, sql]
----
CREATE TABLE BATCH_STEP_EXECUTION_CONTEXT (
STEP_EXECUTION_ID BIGINT PRIMARY KEY,
SHORT_CONTEXT VARCHAR(2500) NOT NULL,
SERIALIZED_CONTEXT CLOB,
constraint STEP_EXEC_CTX_FK foreign key (STEP_EXECUTION_ID)
references BATCH_STEP_EXECUTION(STEP_EXECUTION_ID)
) ;
----
The following list describes each column:
* `STEP_EXECUTION_ID`: Foreign key representing the `StepExecution` to which the context
belongs. There may be more than one row associated to a given execution.
* `SHORT_CONTEXT`: A string version of the `SERIALIZED_CONTEXT`.
* `SERIALIZED_CONTEXT`: The entire context, serialized.
[[metaDataArchiving]]
=== Archiving
Because there are entries in multiple tables every time a batch job is run, it is common
to create an archive strategy for the metadata tables. The tables themselves are designed
to show a record of what happened in the past and generally do not affect the run of any
job, with a few notable exceptions pertaining to restart:
* The framework uses the metadata tables to determine whether a particular `JobInstance`
has been run before. If it has been run and if the job is not restartable, then an
exception is thrown.
* If an entry for a `JobInstance` is removed without having completed successfully, the
framework thinks that the job is new rather than a restart.
* If a job is restarted, the framework uses any data that has been persisted to the
`ExecutionContext` to restore the `Job's` state. Therefore, removing any entries from
this table for jobs that have not completed successfully prevents them from starting at
the correct point if run again.
[[multiByteCharacters]]
=== International and Multi-byte Characters
If you are using multi-byte character sets (such as Chinese or Cyrillic) in your business
processing, then those characters might need to be persisted in the Spring Batch schema.
Many users find that simply changing the schema to double the length of the `VARCHAR`
columns is enough. Others prefer to configure the
<<job.adoc#configuringJobRepository,JobRepository>> with `max-varchar-length` half the
value of the `VARCHAR` column length. Some users have also reported that they use
`NVARCHAR` in place of `VARCHAR` in their schema definitions. The best result depends on
the database platform and the way the database server has been configured locally.
[[recommendationsForIndexingMetaDataTables]]
=== Recommendations for Indexing Meta Data Tables
Spring Batch provides DDL samples for the metadata tables in the core jar file for
several common database platforms. Index declarations are not included in that DDL,
because there are too many variations in how users may want to index, depending on their
precise platform, local conventions, and the business requirements of how the jobs are
operated. The following below provides some indication as to which columns are going to
be used in a `WHERE` clause by the DAO implementations provided by Spring Batch and how
frequently they might be used, so that individual projects can make up their own minds
about indexing:
.Where clauses in SQL statements (excluding primary keys) and their approximate frequency of use.
|===============
|Default Table Name|Where Clause|Frequency
|BATCH_JOB_INSTANCE|JOB_NAME = ? and JOB_KEY = ?|Every time a job is launched
|BATCH_JOB_EXECUTION|JOB_INSTANCE_ID = ?|Every time a job is restarted
|BATCH_EXECUTION_CONTEXT|EXECUTION_ID = ? and KEY_NAME = ?|On commit interval, a.k.a. chunk
|BATCH_STEP_EXECUTION|VERSION = ?|On commit interval, a.k.a. chunk (and at start and end of
step)
|BATCH_STEP_EXECUTION|STEP_NAME = ? and JOB_EXECUTION_ID = ?|Before each step execution
|===============

1352
docs/asciidoc/spring-batch-integration.adoc Normal file
View File

File diff suppressed because it is too large Load Diff

366
docs/asciidoc/spring-batch-intro.adoc Normal file
View File

@ -0,0 +1,366 @@
:batch-asciidoc: ./
:toc: left
:toclevels: 4
[[spring-batch-intro]]
== Spring Batch 介绍
在企业域环境中针对关键环境进行商业操作的时候，有许多应用程序需要进行批量处理。这些业务运营包括：
* 无需用户交互即可最有效地处理大量信息的自动化复杂处理。这些操作通常包括基于时间的事件 （例如，月末统计计算，通知或者消息通知）。
* 在非常大的数据集中重复处理复杂业务规则的定期应用（例如，保险利益确定或费率调整）。
* 整合从内部或者外部系统中收到的信息，这些信息通常要求格式，校验和事务范式处理到记录系统中。
批处理通常被用来针对企业每天产生超过亿万级别的数据量。
Spring Batch是一个轻量级的综合性批处理框架，可用于开发企业信息系统中那些至关重要的数据批量处理业务。
Spring Batch构建了人们期望的Spring Framework 特性（生产环境，基于 POJO 的开发和易于使用），
同时让开发者很容易的访问和使用企业级服务。Spring Batch 不是一个自动调度运行框架。在市面上已经有了很多企
业级和开源的自动运行框架（例如 Quartz，Tivoli， Control-M 等）。
Spring Batch 被设计与计划任务和调度程序一同协作完成任务，而不是替换调度程序。
Spring Batch 提供了可重用的功能，这些功能被用来对大量数据和记录进行处理，包括有日志/跟踪（logging/tracing），
事务管理（transaction management），作业处理状态（job processing statistics），作业重启（job restart），
作业跳过（job skip）和资源管理（resource management）。
此外还提供了许多高级服务和特性, 使之能够通过优化（optimization ） 和分片技术（partitioning techniques）
来实现极高容量和性能的批处理作业。
Spring Batch 能够简单（例如，将文件读入到数据库中或者运行一个存储过程）和复杂（例如，在数据库之间对海量数据进行移动或转换等）
情况下都能很好的工作。
可以在框架高度扩展的方式下执行大批量的作业来处理海量的信息。
[[springBatchBackground]]
=== 背景
在开源项目及其相关社区把大部分注意力集中在基于 web 和 微服务体系框架时框架中时，基于 Java 的批处理框架却无人问津，
尽管在企业 IT 环境中一直都有这种批处理的需求。但因为缺乏一个标准的、可重用的批处理框架导致在企业客户的 IT 系统中
存在着很多一次编写，一次使用的版本，以及很多不同的内部解决方案。
SpringSource (现被命名为 Pivotal) 和 Accenture（埃森哲）致力于通过合作来改善这种状况。
埃森哲在实现批处理架构上有着丰富的产业实践经验，SpringSource 有深入的技术开发积累，
背靠 Spring 框架提供的编程模型,意味着两者能够结合成为默契且强大的合作伙伴，创造出高质量的、市场认可的企业级 Java 解决方案，
填补这一重要的行业空白。两家公司都与许多通过开发基于Spring的批处理架构解决方案解决类似问题的客户合作。
这提供了一些有用的额外细节和实际约束，有助于确保解决方案可以应用于客户提出的现实问题。
埃森哲为Spring Batch项目贡献了以前专有的批处理体系结构框架，以及提供支持，增强功能和现有功能集的提交者资源。
埃森哲的贡献基于几十年来在使用最新几代平台构建批量架构方面的经验：COBOL / Mainframe，C / Unix以及现在的Java / Anywhere。
埃森哲与SpringSource之间的合作旨在促进软件处理方法，框架和工具的标准化，
在创建批处理应用程序时，企业用户可以始终如一地利用这些方法，框架和工具。希望为其企业IT环境提供标准的，
经过验证的解决方案的公司和政府机构可以从 Spring Batch 中受益。
[[springBatchUsageScenarios]]
=== 使用场景
一般的典型批处理程序：
* 从数据库，文件或队列中读取大量记录。
* 以某种方式处理数据。
* 以修改的形式写回数据。
Spring Batch自动执行此基本批处理迭代，提供处理类似事务的功能，通常在脱机环境中处理，无需任何用户交互。
批处理作业是大多数 IT 项目的一部分，Spring Batch 是唯一提供强大的企业级解决方案的开源框架。
业务场景
* 周期提交批处理任务
* 同时批处理进程：并非处理一个任务
* 分阶段的企业消息驱动处理
* 高并发批处理
* 失败后的手动或定时重启
* 按顺序处理任务依赖（使用工作流驱动的批处理插件）
* 部分处理：跳过记录（例如，回滚）
* 全批次事务：因为可能有小数据量的批处理或存在存储过程/脚本中
技术目标
* 批量的开发者使用 Spring 的编程模型：开发者能够更加专注于业务逻辑，让框架来解决基础的功能
* 在基础架构、批处理执行环境、批处理应用之间有明确的划分
* 以接口形式提供通用的核心服务，以便所有项目都能使用
* 提供简单的默认实现，以实现核心执行接口的“开箱即用”
* 通过在所有层中对 Spring 框架进行平衡配置，能够实现更加容易的配置，自定义和扩展服务。
* 所有存在的核心服务应该能够很容易的在不同系统架构层进行影响的情况进行替换或扩展。
* 提供一个简单的部署模块，使用 Maven 来进行编译的 JARs 架构，并与应用完全分离。
[[springBatchArchitecture]]
=== Spring Batch 体系结构
// TODO Make a separate document
Spring Batch 设计的时候充分考虑了可扩展性和各类最终用户。
下图显示了 Spring Batch 的架构层次示意图,这种架构层次为最终用户开发者提供了很好的扩展性与易用性。
.Spring 批量层级体系结构
image::{batch-asciidoc}images/spring-batch-layers.png[Figure 1.1: 批量层级体系结构, scaledwidth="60%"]
这个层级体系结构高亮显示了 Spring Batch 的 3 个主要组件：应用（Application），核心（Core）和 基础架构（Infrastructure）。
应用层包含了所有的批量作业和开发者使用 Spring Batch 写的所有自定义代码。批量核心层包含了所有运行和控制批量作业所需必要的运行时类。
同时还包括了有 `JobLauncher`, `Job`, 和 `Step`。应用层和核心层都构建在基础架构层之上。
基础架构层包含了有 读（readers）和 写（writers ）以及服务（services）。例如有针对服务使用， `RetryTemplate`。
基础架构层的这些东西，这些能够被应用层开发（readers 和 writers，
例如 `ItemReader` 和 `ItemWriter`）和批量核心框架（例如，retry，这个是核心层自己的库）所使用。
简单的来说，基础架构层为应用层和批量核心层提供了所需要的的基础内容，是整个 Spring Batch 的基础。我们针对 Spring Batch 的开发绝大部分情况是在应用层完成的。
[[batchArchitectureConsiderations]]
=== 一般批量处理的原则和使用指引
下面是一些关键的指导原则，可以在构批量处理解决方案可以参考。
* 请记住，通常批量处理体系结构将会影响在线应用的体系结构，同时反过来也是一样的。
在你为批量任务和在线应用进行设计架构和环境的时候请尽可能的使用公共的模块。
* 越简单越好，尽量在一个单独的批量应用中构建简单的批量处理，并避免复杂的逻辑结构。
* 尽量的保持存储的数据和进程存储在同一个地方（换句话说就是尽量将数据保存到你程序运行的地方）。
* 最小化系统资源的使用，尤其针对 I/O。尽量在内存中执行尽可能多的操作。
* 检查应用的 I/O（分析 SQL 语句）来避免不必要的的物理 I/O 使用。特别是以下四个常见的缺陷（flaws）需要避免：
** 在数据可以只读一次就可以缓存起来的情况下，针对每一个事务都来读取数据
** 多次读取/查询同一事务中已经读取过的数据
** 产生不必要的表格或者索引扫描
** 在 SQL 查询中不指定 WHERE 查询的值
* 在批量运行的时候不要将一件事情重复 2 次。例如，如果你需要针对你需要报表的数据汇总,请在处理每一条记录时使用增量来存储,
尽可能不要再去遍历一次同样的数据。
* 为批量进程在开始的时候就分配足够的内存，以避免在运行的时候再次分配内存。
* 总是将数据完整性假定为最坏情况。对数据进行适当的检查和数据校验以保持数据完整性（integrity）。
* 可能的话，请实现内部校验（checksums ）。例如，针对文本文件，应该有一条结尾记录，
这个记录将会说明文件中的总记录数和关键字段的集合（aggregate）。
* 尽可能早地在模拟生产环境下使用真实的数据量，以便于进行计划和执行压力测试。
* 在大数据量的批量中，数据备份可能会非常复杂和充满挑战，尤其是你的系统要求不间断（24 - 7）运行的系统。
数据库备份通常在设计时就考虑好了，但是文件备份也应该提升到同样的重要程度。如果系统依赖于文本文件，
文件备份程序不仅要正确设置和形成文档，还要定期进行测试。
[[batchProcessingStrategy]]
=== 批量处理策略
为了帮助设计和实现批量处理系统，基本的批量应用是通过块和模式来构建的，
同时也应该能够为程序开发人员和设计人员提供结构的样例和基础的批量处理程序。
当你开始设计一个批量作业任务的时候，商业逻辑应该被拆分一系列的步骤，而这些步骤又是可以通过下面的标准构件块来实现的：
* __转换应用程序（Conversion Applications）：__ 针对每一个从外部系统导出或者提供的各种类型的文件，
我们都需要创建一个转换应用程序来讲这些类型的文件和数据转换为处理所需要的标准格式。
这个类型的批量应用程序可以是正规转换工具模块中的一部分，也可以是整个的转换工具模块（请查看：基本的批量服务（Basic Batch Services））。
// TODO Add a link to "Basic Batch Services", once you discover where that content is.
* __校验应用程序（Validation Applications）：__ 校验应用程序能够保证所有的输入和输出记录都是正确和一致的。
校验通常是基于头和尾进行校验的，校验码和校验算法通常是针对记录的交叉验证。
* __提取应用（Extract Applications）：__ 这个应用程序通常被用来从数据库或者文本文件中读取一系列的记录，
并对记录的选择通常是基于预先确定的规则，然后将这些记录输出到输出文件中。
* __提取/更新应用（Extract/Update Applications）：__ 这个应用程序通常被用来从数据库或者文本文件中读取记录，
并将每一条读取的输入记录更新到数据库或者输出数据库中。
* __处理和更新应用（Processing and Updating Applications）：__ 这种程序对从提取或验证程序 传过来的输入事务记录进行处理。
这处理通常包括有读取数据库并且获得需要处理的数据，为输出处理更新数据库或创建记录。
* __输出和格式化应用（Output/Format Applications）：__ 一个应用通过读取一个输入文件，
对输入文件的结构重新格式化为需要的标准格式，然后创建一个打印的输出文件，或将数据传输到其他的程序或者系统中。
更多的，一个基本的应用外壳应该也能够被针对商业逻辑来提供，这个外壳通常不能通过上面介绍的这些标准模块来完成。
// TODO What is an example of such a system?
另外的一个主要的构建块，每一个引用通常可以使用下面的一个或者多个标准工具步骤，例如：
* 分类（Sort）: 一个程序可以读取输入文件后生成一个输出文件，在这个输出文件中可以对记录进行重新排序，
重新排序的是根据给定记录的关键字段进行重新排序的。分类通常使用标准的系统工具来执行。
* 拆分（Split）：一个程序可以读取输入文件后，根据需要的字段值，将输入的文件拆分为多个文件进行输出。拆分通常使用标准的系统工具来执行。
* 合并（Merge）：一个程序可以读取多个输入文件，然后将多个输入文件进行合并处理后生成为一个单一的输出文件。
合并可以自定义或者由参数驱动的（parameter-driven）系统实用程序来执行。
批量处理应用程序可以通过下面的输入数据类型来进行分类：
* 数据库驱动应用程序（Database-driven applications）可以通过从数据库中获得的行或值来进行驱动。
* 文件驱动应用程序（File-driven applications） 可以通过从文件中获得的数据来进行驱动。
* 消息驱动应用程序（Message-driven applications） 可以通过从消息队列中获得的数据来进行驱动。
所有批量处理系统的处理基础都是策略（strategy）。对处理策略进行选择产生影响的因素包括有：预估批量处理需要处理的数据量，
在线并发量，和另外一个批量处理系统的在线并发量，
可用的批量处理时间窗口（很多企业都希望系统是能够不间断运行的，基本上来说批量处理可能没有处理时间窗口）。
针对批量处理的标准处理选项包括有（按实现复杂度的递增顺序）：
* 在一个批处理窗口中执行常规离线批处理
* 并发批量 / 在线处理
* 并发处理很多不同的批量处理或者有很多批量作业在同一时间运行
* 分区（Partitioning），就是在同一时间有很多示例在运行相同的批量作业
* 混合上面的一些需求
上面的一些选项或者所有选项能够被商业的任务调度所支持。
在下面的部分，我们将会针对上面的处理选项来对细节进行更多的说明。需要特别注意的是，
批量处理程序使用提交和锁定策略将会根据批量处理的不同而有所不同。作为最佳实践，在线锁策略应该使用相同的原则。
因此，在设计批处理整体架构时不能简单地拍脑袋决定，需要进行详细的分析和论证。
锁定策略可以仅仅使用常见的数据库锁或者你也可以在系统架构中使用其他的自定义锁定服务。
这个锁服务将会跟踪数据库的锁（例如在一个专用的数据库表（db-table）中存储必要的信息），然后在应用程序请求数据库操作时授予权限或拒绝。
重试逻辑应该也需要在系统架构中实现，以避免批量作业中的因资源锁定而导致批量任务被终止。
*1. 批量处理作业窗口中的常规处理* 针对运行在一个单独批处理窗口中的简单批量处理，更新的数据对在线用户或其他批处理来说并没有实时性要求，
也没有并发问题，在批处理运行完成后执行单次提交即可。
大多数情况下，一种更健壮的方法会更合适。要记住的是，批处理系统会随着时间的流逝而增长，包括复杂度和需要处理的数据量。
如果没有合适的锁定策略，系统仍然依赖于一个单一的提交点，则修改批处理程序会是一件痛苦的事情。 因此，即使是最简单的批处理系统，
也应该为重启-恢复（restart-recovery）选项考虑提交逻辑。针对下面的情况，批量处理就更加复杂了。
*2. 并发批量 / 在线处理* 批处理程序处理的数据如果会同时被在线用户实时更新，就不应该锁定在线用户需要的所有任何数据（不管是数据库还是文件），
即使只需要锁定几秒钟的时间。同时还应该每处理一批事务就提交一次数据库。这减少了其他程序不可用的数据数据量，也压缩了数据不可用的时间。
另一个可以使用的方案就是使用逻辑行基本的锁定实现来替代物理锁定。
通过使用乐观锁（Optimistic Locking ）或悲观锁（Pessimistic Locking）模式。
* 乐观锁假设记录争用的可能性很低。这通常意味着并发批处理和在线处理所使用的每个数据表中都有一个时间戳列。
当程序读取一行进行处理时，同时也获得对应的时间戳。当程序处理完该行以后尝试更新时，在 update 操作的 WHERE 子句中使用原来的时间戳作为条件。
如果时间戳相匹配，则数据和时间戳都更新成功。如果时间戳不匹配，这表明在本程序上次获取和此次更新这段时间内已经有另一个程序修改了同一条记录,因此更新不会被执行。
* 悲观锁定策略假设记录争用的可能性很高，因此在检索时需要获得一个物理锁或逻辑锁。有一种悲观逻辑锁在数据表中使用一个专用的 lock-column 列。
当程序想要为更新目的而获取一行时，它在 lock column 上设置一个标志。如果为某一行设置了标志位，其他程序在试图获取同一行时将会逻辑上获取失败。
当设置标志的程序更新该行时，它也同时清除标志位，允许其他程序获取该行。
请注意，在初步获取和初次设置标志位这段时间内必须维护数据的完整性，比如使用数据库锁（例如，SELECT FOR UPDATE）。
还请注意，这种方法和物理锁都有相同的缺点，除了它在构建一个超时机制时比较容易管理。比如记录而用户去吃午餐了，则超时时间到了以后锁会被自动释放。
这些模式并不一定适用于批处理，但他们可以被用在并发批处理和在线处理的情况下（例如,数据库不支持行级锁）。作为一般规则，乐观锁更适合于在线应用，
而悲观锁更适合于批处理应用。只要使用了逻辑锁，那么所有访问逻辑锁保护的数据的程序都必须采用同样的方案。
请注意：这两种解决方案都只锁定（address locking）单条记录。但很多情况下我们需要锁定一组相关的记录。如果使用物理锁，你必须非常小心地管理这些以避免潜在的死锁。
如果使用逻辑锁，通常最好的解决办法是创建一个逻辑锁管理器，使管理器能理解你想要保护的逻辑记录分组（groups），并确保连贯和没有死锁（non-deadlocking）。
这种逻辑锁管理器通常使用其私有的表来进行锁管理、争用报告、超时机制 等等。
*3. 并行处理* 并行处理允许多个批量处理运行（run）/任务（job）同时并行地运行。以使批量处理总运行时间降到最低。
如果多个任务不使用相同的文件、数据表、索引空间时，批量处理这些不算什么问题。如果确实存在共享和竞争，那么这个服务就应该使用分区数据来实现。
另一种选择是使用控制表来构建一个架构模块以维护他们之间的相互依赖关系。控制表应该为每个共享资源分配一行记录，不管这些资源是否被某个程序所使用。
执行并行作业的批处理架构或程序随后将查询这个控制表，以确定是否可以访问所需的资源。
如果解决了数据访问的问题，并行处理就可以通过使用额外的线程来并行实现。在传统的大型主机环境中，并行作业类上通常被用来确保所有进程都有充足的 CPU 时间。
无论如何，解决方案必须足够强劲，以确保所有正在运行的进程都有足够的运行处理时间。
并行处理的其他关键问题还包括负载平衡以及一般系统资源的可用性（如文件、数据库缓冲池等）。请注意，控制表本身也可能很容易变成一个至关重要的资源（有可能发生严重竞争）。
*4. 分区* 分区技术允许多版本的大型批处理程序并发地（concurrently）运行。这样做的目的是减少超长批处理作业过程所需的时间。
可以成功分区的过程主要是那些可以拆分的输入文件 和/或 主要的数据库表被分区以允许程序使用不同的数据来运行。
此外,被分区的过程必须设计为只处理分配给他的数据集。分区架构与数据库设计和数据库分区策略是密切相关的。
请注意，数据库分区并不一定指数据库需要在物理上实现分区，尽管在大多数情况下这是明智的。
下面的图片展示了分区的方法：
.分区处理
image::{batch-asciidoc}images/partitioned.png[Figure 1.2: 分区处理, scaledwidth="60%"]
系统架构应该足够灵活，以允许动态配置分区的数量。自动控制和用户配置都应该纳入考虑范围。
自动配置可以根据参数来决定，例如输入文件大小 和/或 输入记录的数量。
*4.1 分区方案* 下面列出了一些可能的分区方案，至于具体选择哪种分区方案，要根据具体情况来确定：
_1. 固定和均衡拆分记录集_
这涉及到将输入的记录集合分解成均衡的部分（例如，拆分为 10 份，这样每部分是整个数据集的十分之一）。
每个拆分的部分稍后由一个批处理/提取程序实例来处理。
为了使用这种方案，需要在预处理时候就将记录集进行拆分。拆分的结果有一个最大值和最小值的位置，这两个值可以用作限制每个 批处理/提取程序处理部分的输入。
预处理可能有一个很大的开销，因为它必须计算并确定的每部分数据集的边界。
_2. 通过关键字段（Key Column）拆分_
这涉及到将输入记录按照某个关键字段来拆分，比如一个地区代码（location code），并将每个键分配给一个批处理实例。为了达到这个目标，也可以使用列值：
* 通过分区表来指派给一个批量处理实例。
* 通过值的一部分（例如 0000-0999,1000-1999等）分配给批处理实例。
在选项 1 下，添加新值意味着手动重新配置批处理/提取以确保将新值添加到特定实例。
在选项 2 下，这可确保通过批处理作业的实例覆盖所有值。但是，一个实例处理的值的数量取决于列值的分布（0000-0999范围内可能有大量位置，1000-1999范围内可能很少）。
在此选项下，数据范围应设计为考虑分区。
在这两个选项下，无法实现记录到批处理实例的最佳均匀分布。没有使用批处理实例数的动态配置。
_3. 通过视图（Views）_
这种方法基本上是根据键列来分解，但不同的是在数据库级进行分解。它涉及到将记录集分解成视图。这些视图将被批处理程序的各个实例在处理时使用。分解将通过数据分组来完成。
使用这个方法时，批处理的每个实例都必须为其配置一个特定的视图（而非主表）。当然,对于新添加的数据，这个新的数据分组必须被包含在某个视图中。
也没有自动配置功能，实例数量的变化将导致视图需要进行相应的改变。
_4. 附加的处理识别器_
这涉及到输入表一个附加的新列，它充当一个指示器。在预处理阶段，所有指示器都被标志为未处理。在批处理程序获取记录阶段，只会读取被标记为未处理的记录，
一旦他们被读取（并加锁），它们就被标记为正在处理状态。当记录处理完成，指示器将被更新为完成或错误。
批处理程序的多个实例不需要改变就可以开始，因为附加列确保每条纪录只被处理一次。
// TODO On completion, what is the record marked as? Same for on error. (I expected a
在“完成时，指标被标记为完成”的顺序中的一两句话。
使用该选项时，表上的I/O会动态地增长。在批量更新的程序中，这种影响被降低了，因为写操作是必定要进行的。
_5. 提取表到无格式文件_
这包括将表中的数据提取到一个文件中。然后可以将这个文件拆分成多个部分，作为批处理实例的输入。
使用这个选项时，将数据提取到文件中，并将文件拆分的额外开销，有可能抵消多分区处理（multi-partitioning）的效果。可以通过改变文件分割脚本来实现动态配置。
_6. 使用哈希列（Hashing Column）_
这个计划需要在数据库表中增加一个哈希列（key/index）来检索驱动（driver）记录。这个哈希列将有一个指示器来确定将由批处理程序的哪个实例处理某个特定的行。
例如，如果启动了三个批处理实例，那么 “A” 指示器将标记某行由实例 1 来处理，“B”将标记着将由实例 2 来处理，“C”将标记着将由实例 3 来处理，以此类推。
稍后用于检索记录的过程（procedure）程序，将有一个额外的 WHERE 子句来选择以一个特定指标标记的所有行。
这个表的插入（insert）需要附加的标记字段，默认值将是其中的某一个实例（例如“A”）。
一个简单的批处理程序将被用来更新不同实例之间的重新分配负载的指标。当添加足够多的新行时，这个批处理会被运行（在任何时间，除了在批处理窗口中）。
// TODO Why not in the batch window?
批处理应用程序的其他实例只需要像上面这样的批处理程序运行着以重新分配指标，以决定新实例的数量。
*4.2 数据库和应用设计原则*
如果一个支持多分区（multi-partitioned）的应用程序架构，基于数据库采用关键列（key column）分区方法拆成的多个表，则应该包含一个中心分区仓库来存储分区参数。
这种方式提供了灵活性，并保证了可维护性。这个中心仓库通常只由单个表组成，叫做分区表。
存储在分区表中的信息应该是是静态的，并且只能由 DBA 维护。每个多分区程序对应的单个分区有一行记录，组成这个表。
这个表应该包含这些列：程序 ID 编号，分区编号（分区的逻辑ID），一个分区对应的关键列（key column）的最小值，分区对应的关键列的最大值。
在程序启动时，应用程序架构（Control Processing Tasklet， 控制处理微线程）应该将程序 `id` 和分区号传递给该程序。
这些变量被用于读取分区表，来确定应用程序应该处理的数据范围（如果使用关键列的话）。另外分区号必须在整个处理过程中用来：
* 为了使合并程序正常工作，需要将分区号添加到输出文件/数据库更新
* 向框架的错误处理程序报告正常处理批处理日志和执行期间发生的所有错误
*4.3 死锁最小化*
当程序并行或分区运行时,会导致数据库资源的争用,还可能会发生死锁（Deadlocks）。其中的关键是数据库设计团队在进行数据库设计时必须考虑尽可能消除潜在的竞争情况。
还要确保设计数据库表的索引时考虑到性能以及死锁预防。
死锁或热点往往发生在管理或架构表上,如日志表、控制表、锁表（lock tables）。这些影响也应该纳入考虑。为了确定架构可能的瓶颈，一个真实的压力测试是至关重要的。
要最小化数据冲突的影响，架构应该提供一些服务，如附加到数据库或遇到死锁时的 等待-重试（wait-and-retry）间隔时间。
这意味着要有一个内置的机制来处理数据库返回码，而不是立即引发错误处理，需要等待一个预定的时间并重试执行数据库操作。
*4.4 参数处理和校验*
对程序开发人员来说，分区架构应该相对透明。框架以分区模式运行时应该执行的相关任务包括：
* 在程序启动之前获取分区参数
* 在程序启动之前验证分区参数
* 在启动时将参数传递给应用程序
验证（validation）要包含必要的检查，以确保：
* 应用程序已经足够涵盖整个数据的分区
* 在各个分区之间没有遗漏断代（gaps）
如果数据库是分区的，可能需要一些额外的验证来保证单个分区不会跨越数据库的片区。
体系架构应该考虑整合分区(partitions).包括以下关键问题:
* 在进入下一个任务步骤之前是否所有的分区都必须完成？
* 如果一个分区 Job 中止了要怎么处理？

2289
docs/asciidoc/step.adoc Normal file
View File

File diff suppressed because it is too large Load Diff

397
docs/asciidoc/testing.adoc Normal file
View File

@ -0,0 +1,397 @@
:batch-asciidoc: ./
:toc: left
:toclevels: 4
[[testing]]
== Unit Testing
ifndef::onlyonetoggle[]
include::toggle.adoc[]
endif::onlyonetoggle[]
As with other application styles, it is extremely important to
unit test any code written as part of a batch job. The Spring core
documentation covers how to unit and integration test with Spring in great
detail, so it is not be repeated here. It is important, however, to think
about how to 'end to end' test a batch job, which is what this chapter
covers. The spring-batch-test project includes classes that
facilitate this end-to-end test approach.
[[creatingUnitTestClass]]
=== Creating a Unit Test Class
In order for the unit test to run a batch job, the framework must
load the job's `ApplicationContext`. Two annotations are used to trigger
this behavior:
* `@RunWith(SpringRunner.class)`:
Indicates that the class should use Spring's JUnit facilities
* `@ContextConfiguration(...)`:
Indicates which resources to configure the `ApplicationContext` with.
Starting from v4.1, it is also possible to inject Spring Batch test utilities
like the `JobLauncherTestUtils` and `JobRepositoryTestUtils` in the test context
using the `@SpringBatchTest` annotation.
The following example shows the annotations in use:
.Using Java Configuration
[source, java, role="javaContent"]
----
@SpringBatchTest
@RunWith(SpringRunner.class)
@ContextConfiguration(classes=SkipSampleConfiguration.class)
public class SkipSampleFunctionalTests { ... }
----
.Using XML Configuration
[source, java, role="xmlContent"]
----
@SpringBatchTest
@RunWith(SpringRunner.class)
@ContextConfiguration(locations = { "/simple-job-launcher-context.xml",
"/jobs/skipSampleJob.xml" })
public class SkipSampleFunctionalTests { ... }
----
[[endToEndTesting]]
=== End-To-End Testing of Batch Jobs
'End To End' testing can be defined as testing the complete run of a
batch job from beginning to end. This allows for a test that sets up a
test condition, executes the job, and verifies the end result.
In the following example, the batch job reads from the database and
writes to a flat file. The test method begins by setting up the database
with test data. It clears the CUSTOMER table and then inserts 10 new
records. The test then launches the `Job` by using the
`launchJob()` method. The
`launchJob()` method is provided by the
`JobLauncherTestUtils` class. The `JobLauncherTestUtils` class also provides the
`launchJob(JobParameters)` method, which
allows the test to give particular parameters. The
`launchJob()` method returns the
`JobExecution` object, which is useful for asserting
particular information about the `Job` run. In the
following case, the test verifies that the `Job` ended
with status "COMPLETED":
.XML Based Configuration
[source, java, role="xmlContent"]
----
@SpringBatchTest
@RunWith(SpringRunner.class)
@ContextConfiguration(locations = { "/simple-job-launcher-context.xml",
"/jobs/skipSampleJob.xml" })
public class SkipSampleFunctionalTests {
@Autowired
private JobLauncherTestUtils jobLauncherTestUtils;
private SimpleJdbcTemplate simpleJdbcTemplate;
@Autowired
public void setDataSource(DataSource dataSource) {
this.simpleJdbcTemplate = new SimpleJdbcTemplate(dataSource);
}
@Test
public void testJob() throws Exception {
simpleJdbcTemplate.update("delete from CUSTOMER");
for (int i = 1; i <= 10; i++) {
simpleJdbcTemplate.update("insert into CUSTOMER values (?, 0, ?, 100000)",
i, "customer" + i);
}
JobExecution jobExecution = jobLauncherTestUtils.launchJob();
Assert.assertEquals("COMPLETED", jobExecution.getExitStatus().getExitCode());
}
}
----
.Java Based Configuration
[source, java, role="javaContent"]
----
@SpringBatchTest
@RunWith(SpringRunner.class)
@ContextConfiguration(classes=SkipSampleConfiguration.class)
public class SkipSampleFunctionalTests {
@Autowired
private JobLauncherTestUtils jobLauncherTestUtils;
private SimpleJdbcTemplate simpleJdbcTemplate;
@Autowired
public void setDataSource(DataSource dataSource) {
this.simpleJdbcTemplate = new SimpleJdbcTemplate(dataSource);
}
@Test
public void testJob() throws Exception {
simpleJdbcTemplate.update("delete from CUSTOMER");
for (int i = 1; i <= 10; i++) {
simpleJdbcTemplate.update("insert into CUSTOMER values (?, 0, ?, 100000)",
i, "customer" + i);
}
JobExecution jobExecution = jobLauncherTestUtils.launchJob();
Assert.assertEquals("COMPLETED", jobExecution.getExitStatus().getExitCode());
}
}
----
[[testingIndividualSteps]]
=== Testing Individual Steps
For complex batch jobs, test cases in the end-to-end testing
approach may become unmanageable. It these cases, it may be more useful to
have test cases to test individual steps on their own. The
`JobLauncherTestUtils` class contains a method called
`launchStep`, which takes a step name and runs just
that particular `Step`. This approach allows for more
targeted tests letting the test set up data for only that step and
to validate its results directly. The following example shows how to use the
`launchStep` method to load a `Step` by name:
[source, java]
----
JobExecution jobExecution = jobLauncherTestUtils.launchStep("loadFileStep");
----
=== Testing Step-Scoped Components
Often, the components that are configured for your steps at runtime
use step scope and late binding to inject context from the step or job
execution. These are tricky to test as standalone components, unless you
have a way to set the context as if they were in a step execution. That is
the goal of two components in Spring Batch:
`StepScopeTestExecutionListener` and
`StepScopeTestUtils`.
The listener is declared at the class level, and its job is to
create a step execution context for each test method, as shown in the following example:
[source, java]
----
@ContextConfiguration
@TestExecutionListeners( { DependencyInjectionTestExecutionListener.class,
StepScopeTestExecutionListener.class })
@RunWith(SpringRunner.class)
public class StepScopeTestExecutionListenerIntegrationTests {
// This component is defined step-scoped, so it cannot be injected unless
// a step is active...
@Autowired
private ItemReader<String> reader;
public StepExecution getStepExecution() {
StepExecution execution = MetaDataInstanceFactory.createStepExecution();
execution.getExecutionContext().putString("input.data", "foo,bar,spam");
return execution;
}
@Test
public void testReader() {
// The reader is initialized and bound to the input data
assertNotNull(reader.read());
}
}
----
There are two `TestExecutionListeners`. One
is the regular Spring Test framework, which handles dependency injection
from the configured application context to inject the reader. The
other is the Spring Batch
`StepScopeTestExecutionListener`. It works by looking
for a factory method in the test case for a
`StepExecution`, using that as the context for
the test method, as if that execution were active in a `Step` at runtime. The
factory method is detected by its signature (it must return a
`StepExecution`). If a factory method is not provided,
then a default `StepExecution` is created.
Starting from v4.1, the `StepScopeTestExecutionListener` and
`JobScopeTestExecutionListener` are imported as test execution listeners
if the test class is annotated with `@SpringBatchTest`. The preceding test
example can be configured as follows:
[source, java]
----
@SpringBatchTest
@RunWith(SpringRunner.class)
@ContextConfiguration
public class StepScopeTestExecutionListenerIntegrationTests {
// This component is defined step-scoped, so it cannot be injected unless
// a step is active...
@Autowired
private ItemReader<String> reader;
public StepExecution getStepExecution() {
StepExecution execution = MetaDataInstanceFactory.createStepExecution();
execution.getExecutionContext().putString("input.data", "foo,bar,spam");
return execution;
}
@Test
public void testReader() {
// The reader is initialized and bound to the input data
assertNotNull(reader.read());
}
}
----
The listener approach is convenient if you want the duration of the
step scope to be the execution of the test method. For a more flexible
but more invasive approach, you can use the
`StepScopeTestUtils`. The following example counts the
number of items available in the reader shown in the previous example:
[source, java]
----
int count = StepScopeTestUtils.doInStepScope(stepExecution,
new Callable<Integer>() {
public Integer call() throws Exception {
int count = 0;
while (reader.read() != null) {
count++;
}
return count;
}
});
----
[[validatingOutputFiles]]
=== Validating Output Files
When a batch job writes to the database, it is easy to query the
database to verify that the output is as expected. However, if the batch
job writes to a file, it is equally important that the output be verified.
Spring Batch provides a class called `AssertFile` to
facilitate the verification of output files. The method called
`assertFileEquals` takes two
`File` objects (or two
`Resource` objects) and asserts, line by line, that
the two files have the same content. Therefore, it is possible to create a
file with the expected output and to compare it to the actual
result, as shown in the following example:
[source, java]
----
private static final String EXPECTED_FILE = "src/main/resources/data/input.txt";
private static final String OUTPUT_FILE = "target/test-outputs/output.txt";
AssertFile.assertFileEquals(new FileSystemResource(EXPECTED_FILE),
new FileSystemResource(OUTPUT_FILE));
----
[[mockingDomainObjects]]
=== Mocking Domain Objects
Another common issue encountered while writing unit and integration
tests for Spring Batch components is how to mock domain objects. A good
example is a `StepExecutionListener`, as illustrated
in the following code snippet:
[source, java]
----
public class NoWorkFoundStepExecutionListener extends StepExecutionListenerSupport {
public ExitStatus afterStep(StepExecution stepExecution) {
if (stepExecution.getReadCount() == 0) {
return ExitStatus.FAILED;
}
return null;
}
}
----
The preceding listener example is provided by the framework and checks a
`StepExecution` for an empty read count, thus
signifying that no work was done. While this example is fairly simple, it
serves to illustrate the types of problems that may be encountered when
attempting to unit test classes that implement interfaces requiring Spring
Batch domain objects. Consider the following unit test for the listener's in the preceding example:
[source, java]
----
private NoWorkFoundStepExecutionListener tested = new NoWorkFoundStepExecutionListener();
@Test
public void noWork() {
StepExecution stepExecution = new StepExecution("NoProcessingStep",
new JobExecution(new JobInstance(1L, new JobParameters(),
"NoProcessingJob")));
stepExecution.setExitStatus(ExitStatus.COMPLETED);
stepExecution.setReadCount(0);
ExitStatus exitStatus = tested.afterStep(stepExecution);
assertEquals(ExitStatus.FAILED.getExitCode(), exitStatus.getExitCode());
}
----
Because the Spring Batch domain model follows good object-oriented
principles, the `StepExecution` requires a
`JobExecution`, which requires a
`JobInstance` and
`JobParameters`, to create a valid
`StepExecution`. While this is good in a solid domain
model, it does make creating stub objects for unit testing verbose. To
address this issue, the Spring Batch test module includes a factory for
creating domain objects: `MetaDataInstanceFactory`.
Given this factory, the unit test can be updated to be more
concise, as shown in the following example:
[source, java]
----
private NoWorkFoundStepExecutionListener tested = new NoWorkFoundStepExecutionListener();
@Test
public void testAfterStep() {
StepExecution stepExecution = MetaDataInstanceFactory.createStepExecution();
stepExecution.setExitStatus(ExitStatus.COMPLETED);
stepExecution.setReadCount(0);
ExitStatus exitStatus = tested.afterStep(stepExecution);
assertEquals(ExitStatus.FAILED.getExitCode(), exitStatus.getExitCode());
}
----
The preceding method for creating a simple
`StepExecution` is just one convenience method
available within the factory. A full method listing can be found in its
link:$$https://docs.spring.io/spring-batch/apidocs/org/springframework/batch/test/MetaDataInstanceFactory.html$$[Javadoc].

14
docs/asciidoc/toggle.adoc Normal file
View File

@ -0,0 +1,14 @@
ifdef::backend-html5[]
+++
<div>
<script type="text/javascript" src="js/jquery-3.2.1.min.js"></script>
<script type="text/javascript" src="js/js.cookie.js"></script>
<script type="text/javascript" src="js/DocumentToggle.js"></script>
<div class="docToggle-button">
<input id="xmlButton" type="radio" name="docToggle" value="XML"><label for="xmlButton">XML</label>
<input id="javaButton" type="radio" name="docToggle" value="Java" checked><label for="javaButton">Java</label>
<!-- <input id="bothButton" type="radio" name="docToggle" value="Both" checked><label for="bothButton">Both</label> -->
</div>
</div>
+++
endif::backend-html5[]

345
docs/asciidoc/transaction-appendix.adoc Normal file
View File

@ -0,0 +1,345 @@
:batch-asciidoc: ./
:toc: left
:toclevels: 4
[[transactions]]
[appendix]
== Batch Processing and Transactions
[[transactionsNoRetry]]
=== Simple Batching with No Retry
Consider the following simple example of a nested batch with no retries. It shows a
common scenario for batch processing: An input source is processed until exhausted, and
we commit periodically at the end of a "chunk" of processing.
----
1 | REPEAT(until=exhausted) {
|
2 | TX {
3 | REPEAT(size=5) {
3.1 | input;
3.2 | output;
| }
| }
|
| }
----
The input operation (3.1) could be a message-based receive (such as from JMS), or a
file-based read, but to recover and continue processing with a chance of completing the
whole job, it must be transactional. The same applies to the operation at 3.2. It must
be either transactional or idempotent.
If the chunk at `REPEAT` (3) fails because of a database exception at 3.2, then `TX` (2)
must roll back the whole chunk.
[[transactionStatelessRetry]]
=== Simple Stateless Retry
It is also useful to use a retry for an operation which is not transactional, such as a
call to a web-service or other remote resource, as shown in the following example:
----
0 | TX {
1 | input;
1.1 | output;
2 | RETRY {
2.1 | remote access;
| }
| }
----
This is actually one of the most useful applications of a retry, since a remote call is
much more likely to fail and be retryable than a database update. As long as the remote
access (2.1) eventually succeeds, the transaction, `TX` (0), commits. If the remote
access (2.1) eventually fails, then the transaction, `TX` (0), is guaranteed to roll
back.
[[repeatRetry]]
=== Typical Repeat-Retry Pattern
The most typical batch processing pattern is to add a retry to the inner block of the
chunk, as shown in the following example:
----
1 | REPEAT(until=exhausted, exception=not critical) {
|
2 | TX {
3 | REPEAT(size=5) {
|
4 | RETRY(stateful, exception=deadlock loser) {
4.1 | input;
5 | } PROCESS {
5.1 | output;
6 | } SKIP and RECOVER {
| notify;
| }
|
| }
| }
|
| }
----
The inner `RETRY` (4) block is marked as "stateful". See <<transactionsNoRetry,the
typical use case>> for a description of a stateful retry. This means that if the
retry `PROCESS` (5) block fails, the behavior of the `RETRY` (4) is as follows:
. Throw an exception, rolling back the transaction, `TX` (2), at the chunk level, and
allowing the item to be re-presented to the input queue.
. When the item re-appears, it might be retried depending on the retry policy in place,
executing `PROCESS` (5) again. The second and subsequent attempts might fail again and
re-throw the exception.
. Eventually, the item reappears for the final time. The retry policy disallows another
attempt, so `PROCESS` (5) is never executed. In this case, we follow the `RECOVER` (6)
path, effectively "skipping" the item that was received and is being processed.
Note that the notation used for the `RETRY` (4) in the plan above explicitly shows that
the input step (4.1) is part of the retry. It also makes clear that there are two
alternate paths for processing: the normal case, as denoted by `PROCESS` (5), and the
recovery path, as denoted in a separate block by `RECOVER` (6). The two alternate paths
are completely distinct. Only one is ever taken in normal circumstances.
In special cases (such as a special `TranscationValidException` type), the retry policy
might be able to determine that the `RECOVER` (6) path can be taken on the last attempt
after `PROCESS` (5) has just failed, instead of waiting for the item to be re-presented.
This is not the default behavior, because it requires detailed knowledge of what has
happened inside the `PROCESS` (5) block, which is not usually available. For example, if
the output included write access before the failure, then the exception should be
re-thrown to ensure transactional integrity.
The completion policy in the outer `REPEAT` (1) is crucial to the success of the above
plan. If the output (5.1) fails, it may throw an exception (it usually does, as
described), in which case the transaction, `TX` (2), fails, and the exception could
propagate up through the outer batch `REPEAT` (1). We do not want the whole batch to
stop, because the `RETRY` (4) might still be successful if we try again, so we add
`exception=not critical` to the outer `REPEAT` (1).
Note, however, that if the `TX` (2) fails and we __do__ try again, by virtue of the outer
completion policy, the item that is next processed in the inner `REPEAT` (3) is not
guaranteed to be the one that just failed. It might be, but it depends on the
implementation of the input (4.1). Thus, the output (5.1) might fail again on either a
new item or the old one. The client of the batch should not assume that each `RETRY` (4)
attempt is going to process the same items as the last one that failed. For example, if
the termination policy for `REPEAT` (1) is to fail after 10 attempts, it fails after 10
consecutive attempts but not necessarily at the same item. This is consistent with the
overall retry strategy. The inner `RETRY` (4) is aware of the history of each item and
can decide whether or not to have another attempt at it.
[[asyncChunkProcessing]]
=== Asynchronous Chunk Processing
The inner batches or chunks in the <<repeatRetry,typical example>> can be executed
concurrently by configuring the outer batch to use an `AsyncTaskExecutor`. The outer
batch waits for all the chunks to complete before completing. The following example shows
asynchronous chunk processing:
----
1 | REPEAT(until=exhausted, concurrent, exception=not critical) {
|
2 | TX {
3 | REPEAT(size=5) {
|
4 | RETRY(stateful, exception=deadlock loser) {
4.1 | input;
5 | } PROCESS {
| output;
6 | } RECOVER {
| recover;
| }
|
| }
| }
|
| }
----
[[asyncItemProcessing]]
=== Asynchronous Item Processing
The individual items in chunks in the <<repeatRetry,typical example>> can also, in
principle, be processed concurrently. In this case, the transaction boundary has to move
to the level of the individual item, so that each transaction is on a single thread, as
shown in the following example:
----
1 | REPEAT(until=exhausted, exception=not critical) {
|
2 | REPEAT(size=5, concurrent) {
|
3 | TX {
4 | RETRY(stateful, exception=deadlock loser) {
4.1 | input;
5 | } PROCESS {
| output;
6 | } RECOVER {
| recover;
| }
| }
|
| }
|
| }
----
This plan sacrifices the optimization benefit, which the simple plan had, of having all
the transactional resources chunked together. It is only useful if the cost of the
processing (5) is much higher than the cost of transaction management (3).
[[transactionPropagation]]
=== Interactions Between Batching and Transaction Propagation
There is a tighter coupling between batch-retry and transaction management than we would
ideally like. In particular, a stateless retry cannot be used to retry database
operations with a transaction manager that does not support NESTED propagation.
The following example uses retry without repeat:
----
1 | TX {
|
1.1 | input;
2.2 | database access;
2 | RETRY {
3 | TX {
3.1 | database access;
| }
| }
|
| }
----
Again, and for the same reason, the inner transaction, `TX` (3), can cause the outer
transaction, `TX` (1), to fail, even if the `RETRY` (2) is eventually successful.
Unfortunately, the same effect percolates from the retry block up to the surrounding
repeat batch if there is one, as shown in the following example:
----
1 | TX {
|
2 | REPEAT(size=5) {
2.1 | input;
2.2 | database access;
3 | RETRY {
4 | TX {
4.1 | database access;
| }
| }
| }
|
| }
----
Now, if TX (3) rolls back, it can pollute the whole batch at TX (1) and force it to roll
back at the end.
What about non-default propagation?
* In the preceding example, `PROPAGATION_REQUIRES_NEW` at `TX` (3) prevents the outer
`TX` (1) from being polluted if both transactions are eventually successful. But if `TX`
(3) commits and `TX` (1) rolls back, then `TX` (3) stays committed, so we violate the
transaction contract for `TX` (1). If `TX` (3) rolls back, `TX` (1) does not necessarily
(but it probably does in practice, because the retry throws a roll back exception).
* `PROPAGATION_NESTED` at `TX` (3) works as we require in the retry case (and for a
batch with skips): `TX` (3) can commit but subsequently be rolled back by the outer
transaction, `TX` (1). If `TX` (3) rolls back, `TX` (1) rolls back in practice. This
option is only available on some platforms, not including Hibernate or
JTA, but it is the only one that consistently works.
Consequently, the `NESTED` pattern is best if the retry block contains any database
access.
[[specialTransactionOrthonogonal]]
=== Special Case: Transactions with Orthogonal Resources
Default propagation is always OK for simple cases where there are no nested database
transactions. Consider the following example, where the `SESSION` and `TX` are not
global `XA` resources, so their resources are orthogonal:
----
0 | SESSION {
1 | input;
2 | RETRY {
3 | TX {
3.1 | database access;
| }
| }
| }
----
Here there is a transactional message `SESSION` (0), but it does not participate in other
transactions with `PlatformTransactionManager`, so it does not propagate when `TX` (3)
starts. There is no database access outside the `RETRY` (2) block. If `TX` (3) fails and
then eventually succeeds on a retry, `SESSION` (0) can commit (independently of a `TX`
block). This is similar to the vanilla "best-efforts-one-phase-commit" scenario. The
worst that can happen is a duplicate message when the `RETRY` (2) succeeds and the
`SESSION` (0) cannot commit (for example, because the message system is unavailable).
[[statelessRetryCannotRecover]]
=== Stateless Retry Cannot Recover
The distinction between a stateless and a stateful retry in the typical example above is
important. It is actually ultimately a transactional constraint that forces the
distinction, and this constraint also makes it obvious why the distinction exists.
We start with the observation that there is no way to skip an item that failed and
successfully commit the rest of the chunk unless we wrap the item processing in a
transaction. Consequently, we simplify the typical batch execution plan to be as
follows:
----
0 | REPEAT(until=exhausted) {
|
1 | TX {
2 | REPEAT(size=5) {
|
3 | RETRY(stateless) {
4 | TX {
4.1 | input;
4.2 | database access;
| }
5 | } RECOVER {
5.1 | skip;
| }
|
| }
| }
|
| }
----
The preceding example shows a stateless `RETRY` (3) with a `RECOVER` (5) path that kicks
in after the final attempt fails. The `stateless` label means that the block is repeated
without re-throwing any exception up to some limit. This only works if the transaction,
`TX` (4), has propagation NESTED.
If the inner `TX` (4) has default propagation properties and rolls back, it pollutes the
outer `TX` (1). The inner transaction is assumed by the transaction manager to have
corrupted the transactional resource, so it cannot be used again.
Support for NESTED propagation is sufficiently rare that we choose not to support
recovery with stateless retries in the current versions of Spring Batch. The same effect
can always be achieved (at the expense of repeating more processing) by using the
typical pattern above.

43
docs/asciidoc/whatsnew.adoc Normal file
View File

@ -0,0 +1,43 @@
:batch-asciidoc: ./
:toc: left
:toclevels: 4
[[whatsNew]]
== Spring Batch 4.2 新特性
Spring Batch 4.2 添加了下面的新特性：
* 使用 https://micrometer.io[Micrometer] 来支持批量指标（batch metrics）
* 支持从 https://kafka.apache.org[Apache Kafka] topics 读取/写入（reading/writing） 数据
* 支持从 https://avro.apache.org[Apache Avro] 资源中读取/写入（reading/writing） 数据
* 改进支持文档
[[whatsNewMetrics]]
=== 使用 Micrometer 的批量指标
本发行版本介绍了可以让你通过使用 Micrometer 来监控你的批量作业。在默认的情况下，Spring Batch 将会收集相关批量指标
（包括，作业时间，步骤的时间，读取和写入的项目，以及其他的相关信息），和将这些指标通过 `spring.batch` 前缀（prefix）注册到 Micrometer 的全局指标中。
这些指标可以发布到任何能够支持 Micrometer 的 https://micrometer.io/docs/concepts#_supported_monitoring_systems[监控系统（monitoring system）]中。
有关这个新特性的更多细节，请参考
<<monitoring-and-metrics.adoc#monitoring-and-metrics,Monitoring and metrics>> 章节中的内容。
[[whatsNewKafka]]
=== Apache Kafka item 读取/写入
本发行版本添加了一个新的 `KafkaItemReader` 和 `KafkaItemWriter`， 用来从 Kafka 的 topics 中读取和写入。
有关更多这个新组建的信息，请参考： https://docs.spring.io/spring-batch/4.2.x/api/index.html[Javadoc]。
[[whatsNewAvro]]
=== Apache Avro item 读取/写入
本发行版本添加了一个新的 `AvroItemReader` 和 `AvroItemWriter`， 用来从 Avro 资源中读取和写入。
有关更多这个新组建的信息，请参考： https://docs.spring.io/spring-batch/4.2.x/api/index.html[Javadoc].
[[whatsNewDocs]]
=== 文档更新
对参考的文档进行更新，以便于与其他 Spring 项目的文档风格保持一致。

BIN
gradle/wrapper/gradle-wrapper.jar vendored Normal file
View File

Binary file not shown.

5
gradle/wrapper/gradle-wrapper.properties vendored Normal file
View File

@ -0,0 +1,5 @@
distributionBase=GRADLE_USER_HOME
distributionPath=wrapper/dists
distributionUrl=https\://services.gradle.org/distributions/gradle-5.4-bin.zip
zipStoreBase=GRADLE_USER_HOME
zipStorePath=wrapper/dists

2
pom.xml
View File

@ -2,7 +2,7 @@
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>com.ossez.reoc</groupId>
<version>1.0-SNAPSHOT</version>
<version>0.1.0</version>
<artifactId>reoc-mls-client</artifactId>
<packaging>jar</packaging>

30
src/main/java/com/ossez/reoc/rets/client/BrokerCodeRequredException.java Normal file
View File

@ -0,0 +1,30 @@
/*
* cart: CRT's Awesome RETS Tool
*
* Author: David Terrell
* Copyright (c) 2003, The National Association of REALTORS
* Distributed under a BSD-style license. See LICENSE.TXT for details.
*/
package com.ossez.reoc.rets.client;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
/**
* dbt is lame and hasn't overridden the default
* javadoc string.
*/
public class BrokerCodeRequredException extends RetsException {
private final List mCodeList;
public BrokerCodeRequredException(Collection codes) {
this.mCodeList = Collections.unmodifiableList(new ArrayList(codes));
}
public List getCodeList(){
return this.mCodeList;
}
}

152
src/main/java/com/ossez/reoc/rets/client/CapabilityUrls.java Normal file
View File

@ -0,0 +1,152 @@
package com.ossez.reoc.rets.client;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.HashMap;
import java.util.Map;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
public class CapabilityUrls {
public static final String ACTION_URL = "Action";
public static final String CHANGE_PASSWORD_URL = "ChangePassword";
public static final String GET_OBJECT_URL = "GetObject";
public static final String LOGIN_URL = "Login";
public static final String LOGIN_COMPLETE_URL = "LoginComplete";
public static final String LOGOUT_URL = "Logout";
public static final String SEARCH_URL = "Search";
public static final String GET_METADATA_URL = "GetMetadata";
public static final String UPDATE_URL = "Update";
public static final String SERVER_INFO_URL = "ServerInformation";// for rets 1.7
private static final Log LOG = LogFactory.getLog(CapabilityUrls.class);
private final Map mCapabilityUrls;
private URL mUrl;
public CapabilityUrls() {
this(null);
}
public CapabilityUrls(URL baseurl) {
this.mUrl = baseurl;
this.mCapabilityUrls = new HashMap();
}
public void setCapabilityUrl(String capability, String url) {
if (this.mUrl != null) {
try {
String newurl = new URL(this.mUrl, url).toString();
if (!newurl.equals(url)) {
LOG.info("qualified " + capability + " URL different: "
+ url + " -> " + newurl);
url = newurl;
}
} catch (MalformedURLException e) {
LOG.warn("Couldn't normalize URL", e);
}
}
this.mCapabilityUrls.put(capability, url);
}
public String getCapabilityUrl(String capability) {
return (String) this.mCapabilityUrls.get(capability);
}
public void setActionUrl(String url) {
setCapabilityUrl(ACTION_URL, url);
}
public String getActionUrl() {
return getCapabilityUrl(ACTION_URL);
}
public void setChangePasswordUrl(String url) {
setCapabilityUrl(CHANGE_PASSWORD_URL, url);
}
public String getChangePasswordUrl() {
return getCapabilityUrl(CHANGE_PASSWORD_URL);
}
public void setGetObjectUrl(String url) {
setCapabilityUrl(GET_OBJECT_URL, url);
}
public String getGetObjectUrl() {
return getCapabilityUrl(GET_OBJECT_URL);
}
public void setLoginUrl(String url) {
if (this.mUrl == null) {
try {
this.mUrl = new URL(url);
} catch (MalformedURLException e) {
LOG.debug("java.net.URL can't parse login url: " + url);
this.mUrl = null;
}
}
setCapabilityUrl(LOGIN_URL, url);
}
public String getLoginUrl() {
return getCapabilityUrl(LOGIN_URL);
}
public void setLoginCompleteUrl(String url) {
setCapabilityUrl(LOGIN_COMPLETE_URL, url);
}
public String getLoginCompleteUrl() {
return getCapabilityUrl(LOGIN_COMPLETE_URL);
}
public void setLogoutUrl(String url) {
setCapabilityUrl(LOGOUT_URL, url);
}
public String getLogoutUrl() {
return getCapabilityUrl(LOGOUT_URL);
}
public void setSearchUrl(String url) {
setCapabilityUrl(SEARCH_URL, url);
}
public String getSearchUrl() {
return getCapabilityUrl(SEARCH_URL);
}
public void setGetMetadataUrl(String url) {
setCapabilityUrl(GET_METADATA_URL, url);
}
public String getGetMetadataUrl() {
return getCapabilityUrl(GET_METADATA_URL);
}
public void setUpdateUrl(String url) {
setCapabilityUrl(UPDATE_URL, url);
}
public String getUpdateUrl() {
return getCapabilityUrl(UPDATE_URL);
}
/**
* This is for RETS 1.7 and later and will return an empty string if it is not implemented.
* @param url
*/
public void setServerInfo(String url) {
setCapabilityUrl(SERVER_INFO_URL, url);
}
/**
* This is for RETS 1.7 and later and will return an empty string if it is not implemented.
* @return
*/
public String getServerInfo() {
return getCapabilityUrl(SERVER_INFO_URL);
}
}

40
src/main/java/com/ossez/reoc/rets/client/ChangePasswordRequest.java Normal file
View File

@ -0,0 +1,40 @@
package com.ossez.reoc.rets.client;
import java.security.GeneralSecurityException;
import java.security.MessageDigest;
import javax.crypto.Cipher;
import javax.crypto.SecretKey;
import javax.crypto.SecretKeyFactory;
import javax.crypto.spec.DESKeySpec;
import org.apache.commons.codec.binary.Base64;
public class ChangePasswordRequest extends VersionInsensitiveRequest {
public ChangePasswordRequest(String username, String oldPassword, String newPassword) throws RetsException {
try {
MessageDigest md5 = MessageDigest.getInstance("MD5");
md5.update(username.toUpperCase().getBytes());
md5.update(oldPassword.toUpperCase().getBytes());
byte[] digest = md5.digest();
DESKeySpec keyspec = new DESKeySpec(digest);
SecretKey key = SecretKeyFactory.getInstance("DES").generateSecret(keyspec);
Cipher cipher = Cipher.getInstance("DES/ECB/PKCS5Padding");
cipher.init(Cipher.ENCRYPT_MODE, key);
cipher.update(newPassword.getBytes());
cipher.update(":".getBytes());
cipher.update(username.getBytes());
md5.reset();
md5.update(cipher.doFinal());
byte[] output = md5.digest();
byte[] param = Base64.encodeBase64(output);
setQueryParameter("PWD", new String(param));
} catch (GeneralSecurityException e) {
throw new RetsException(e);
}
}
@Override
public void setUrl(CapabilityUrls urls) {
this.setUrl(urls.getChangePasswordUrl());
}
}

35
src/main/java/com/ossez/reoc/rets/client/ChangePasswordResponse.java Normal file
View File

@ -0,0 +1,35 @@
package com.ossez.reoc.rets.client;
import java.io.InputStream;
import org.jdom.Document;
import org.jdom.Element;
import org.jdom.input.SAXBuilder;
/**
* dbt is lame and hasn't overridden the default
* javadoc string.
*/
public class ChangePasswordResponse {
public ChangePasswordResponse(InputStream stream) throws RetsException {
SAXBuilder builder = new SAXBuilder();
Document document = null;
try {
document = builder.build(stream);
} catch (Exception e) {
throw new RetsException(e);
}
Element rets = document.getRootElement();
if (!rets.getName().equals("RETS")) {
throw new RetsException("Invalid Change Password Response");
}
int replyCode = Integer.parseInt(rets.getAttributeValue("ReplyCode"));
if (replyCode != 0) {
InvalidReplyCodeException exception;
exception = new InvalidReplyCodeException(replyCode);
exception.setRemoteMessage(rets.getAttributeValue("ReplyText"));
throw exception;
}
}
}

40
src/main/java/com/ossez/reoc/rets/client/CollectionOfCollectionsIterator.java Normal file
View File

@ -0,0 +1,40 @@
package com.ossez.reoc.rets.client;
import java.util.Collection;
import java.util.Iterator;
import java.util.NoSuchElementException;
public class CollectionOfCollectionsIterator implements Iterator {
private Iterator mOuter;
private Iterator mInner;
public CollectionOfCollectionsIterator(Collection c) {
this.mOuter = c.iterator();
hasNext();
}
public boolean hasNext() {
if( this.mInner != null && this.mInner.hasNext() ) {
return true;
}
while( this.mOuter.hasNext() ){
this.mInner = ((Collection) this.mOuter.next()).iterator();
if( this.mInner.hasNext() ){
return true;
}
}
return false;
}
public Object next() {
if ( this.hasNext() )
return this.mInner.next();
throw new NoSuchElementException();
}
public void remove() throws UnsupportedOperationException {
throw new UnsupportedOperationException();
}
}

180
src/main/java/com/ossez/reoc/rets/client/CommonsHttpClient.java Normal file
View File

@ -0,0 +1,180 @@
package com.ossez.reoc.rets.client;
import java.io.UnsupportedEncodingException;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import com.ossez.reoc.rets.common.util.CaseInsensitiveTreeMap;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.http.Header;
import org.apache.http.HttpResponse;
import org.apache.http.HttpStatus;
import org.apache.http.StatusLine;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.methods.HttpRequestBase;
import org.apache.http.client.params.ClientPNames;
import org.apache.http.client.params.CookiePolicy;
import org.apache.http.cookie.Cookie;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.impl.conn.tsccm.ThreadSafeClientConnManager;
import org.apache.http.params.BasicHttpParams;
import org.apache.http.params.HttpConnectionParams;
public class CommonsHttpClient extends RetsHttpClient {
private static final int DEFAULT_TIMEOUT = 300000;
private static final String RETS_VERSION = "RETS-Version";
private static final String RETS_SESSION_ID = "RETS-Session-ID";
private static final String RETS_REQUEST_ID = "RETS-Request-ID";
private static final String USER_AGENT = "User-Agent";
private static final String RETS_UA_AUTH_HEADER = "RETS-UA-Authorization";
private static final String ACCEPT_ENCODING = "Accept-Encoding";
public static final String CONTENT_ENCODING = "Content-Encoding";
public static final String DEFLATE_ENCODINGS = "gzip,deflate";
public static final String CONTENT_TYPE = "Content-Type";
public static BasicHttpParams defaultParams(int timeout) {
BasicHttpParams httpClientParams = new BasicHttpParams();
// connection to server timeouts
HttpConnectionParams.setConnectionTimeout(httpClientParams, timeout);
HttpConnectionParams.setSoTimeout(httpClientParams, timeout);
// set to rfc 2109 as it puts the ASP (IIS) cookie _FIRST_, this is critical for interealty
httpClientParams.setParameter(ClientPNames.COOKIE_POLICY, CookiePolicy.RFC_2109);
return httpClientParams;
}
public static ThreadSafeClientConnManager defaultConnectionManager(int maxConnectionsPerRoute, int maxConnectionsTotal) {
// allows for multi threaded requests from a single client
ThreadSafeClientConnManager connectionManager = new ThreadSafeClientConnManager();
connectionManager.setDefaultMaxPerRoute(maxConnectionsPerRoute);
connectionManager.setMaxTotal(maxConnectionsTotal);
return connectionManager;
}
private final ConcurrentHashMap<String, String> defaultHeaders;
private final DefaultHttpClient httpClient;
// method choice improvement
private final String userAgentPassword;
public CommonsHttpClient() {
this(new DefaultHttpClient(defaultConnectionManager(Integer.MAX_VALUE, Integer.MAX_VALUE), defaultParams(DEFAULT_TIMEOUT)), null, true);
}
public CommonsHttpClient(int timeout, String userAgentPassword, boolean gzip) {
this(new DefaultHttpClient(defaultConnectionManager(Integer.MAX_VALUE, Integer.MAX_VALUE), defaultParams(timeout)), userAgentPassword, gzip);
}
public CommonsHttpClient(DefaultHttpClient client, String userAgentPassword, boolean gzip) {
this.defaultHeaders = new ConcurrentHashMap<String, String>();
this.userAgentPassword = userAgentPassword;
this.httpClient = client;
// ask the server if we can use gzip
if( gzip ) this.addDefaultHeader(ACCEPT_ENCODING, DEFLATE_ENCODINGS);
}
public DefaultHttpClient getHttpClient(){
return this.httpClient;
}
//----------------------method implementations
@Override
public void setUserCredentials(String userName, String password) {
UsernamePasswordCredentials creds = new UsernamePasswordCredentials(userName, password);
this.httpClient.getCredentialsProvider().setCredentials(AuthScope.ANY, creds);
}
@Override
public RetsHttpResponse doRequest(String httpMethod, RetsHttpRequest request) throws RetsException {
return "GET".equals(StringUtils.upperCase(httpMethod)) ? this.doGet(request) : this.doPost(request);
}
//----------------------method implementations
public RetsHttpResponse doGet(RetsHttpRequest request) throws RetsException {
String url = request.getUrl();
String args = request.getHttpParameters();
if (args != null) {
url = url + "?" + args;
}
HttpGet method = new HttpGet(url);
return execute(method, request.getHeaders());
}
public RetsHttpResponse doPost(RetsHttpRequest request) throws RetsException {
String url = request.getUrl();
String body = request.getHttpParameters();
if (body == null) body = ""; // commons-httpclient 3.0 refuses to accept null entity (body)
HttpPost method = new HttpPost(url);
try {
method.setEntity(new StringEntity(body, null, null));
} catch (UnsupportedEncodingException e) {
throw new RetsException(e);
}
//updated Content-Type, application/x-www-url-encoded no longer supported
method.setHeader("Content-Type", "application/x-www-form-urlencoded");
return execute(method, request.getHeaders());
}
protected RetsHttpResponse execute(final HttpRequestBase method, Map<String,String> headers) throws RetsException {
try {
// add the default headers
if (this.defaultHeaders != null) {
for (Map.Entry<String,String> entry : this.defaultHeaders.entrySet()) {
method.setHeader(entry.getKey(), entry.getValue());
}
}
// add our request headers from rets
if (headers != null) {
for (Map.Entry<String,String> entry : headers.entrySet()) {
method.setHeader(entry.getKey(), entry.getValue());
}
}
// optional ua-auth stuff here
if( this.userAgentPassword != null ){
method.setHeader(RETS_UA_AUTH_HEADER, calculateUaAuthHeader(method,getCookies()));
}
// try to execute the request
HttpResponse response = this.httpClient.execute(method);
StatusLine status = response.getStatusLine();
if (status.getStatusCode() != HttpStatus.SC_OK) {
throw new InvalidHttpStatusException(status);
}
return new CommonsHttpClientResponse(response, getCookies());
} catch (Exception e) {
throw new RetsException(e);
}
}
@Override
public synchronized void addDefaultHeader(String key, String value) {
this.defaultHeaders.put(key, value);
if( value == null ) this.defaultHeaders.remove(key);
}
protected Map<String,String> getCookies() {
Map<String,String> cookieMap = new CaseInsensitiveTreeMap();
for (Cookie cookie : this.httpClient.getCookieStore().getCookies()) {
cookieMap.put(cookie.getName(), cookie.getValue());
}
return cookieMap;
}
protected String calculateUaAuthHeader(HttpRequestBase method, Map<String, String> cookies ) {
final String userAgent = this.getHeaderValue(method, USER_AGENT);
final String requestId = this.getHeaderValue(method, RETS_REQUEST_ID);
final String sessionId = cookies.get(RETS_SESSION_ID);
final String retsVersion = this.getHeaderValue(method, RETS_VERSION);
String secretHash = DigestUtils.md5Hex(String.format("%s:%s",userAgent,this.userAgentPassword));
String pieces = String.format("%s:%s:%s:%s",secretHash,StringUtils.trimToEmpty(requestId),StringUtils.trimToEmpty(sessionId),retsVersion);
return String.format("Digest %s", DigestUtils.md5Hex(pieces));
}
protected String getHeaderValue(HttpRequestBase method, String key){
Header requestHeader = method.getFirstHeader(key);
if( requestHeader == null ) return null;
return requestHeader.getValue();
}
}

110
src/main/java/com/ossez/reoc/rets/client/CommonsHttpClientResponse.java Normal file
View File

@ -0,0 +1,110 @@
package com.ossez.reoc.rets.client;
import java.io.IOException;
import java.io.InputStream;
import java.util.Map;
import java.util.zip.GZIPInputStream;
import com.ossez.reoc.rets.common.util.CaseInsensitiveTreeMap;
import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.http.Header;
import org.apache.http.HttpResponse;
import com.google.common.io.Closeables;
public class CommonsHttpClientResponse implements RetsHttpResponse {
private HttpResponse response;
private Map<String,String> headers;
private Map<String,String> cookies;
public CommonsHttpClientResponse(HttpResponse response, Map<String,String> cookies) {
this.response = response;
this.cookies = new CaseInsensitiveTreeMap<String,String>(cookies);
this.headers = new CaseInsensitiveTreeMap<String,String>();
for (Header header : this.response.getAllHeaders()) {
this.headers.put(header.getName(), header.getValue());
}
}
public int getResponseCode() {
return this.response.getStatusLine().getStatusCode();
}
public Map<String,String> getHeaders() {
return this.headers;
}
public String getHeader(String header) {
return this.headers.get(header);
}
public Map<String,String> getCookies() throws RetsException {
return this.cookies;
}
public String getCookie(String cookie) throws RetsException {
return this.cookies.get(cookie);
}
public String getCharset() {
String contentType = StringUtils.trimToEmpty(this.getHeader(CommonsHttpClient.CONTENT_TYPE)).toLowerCase();
String[] split = StringUtils.split(contentType, ";");
if (split == null) return null;
for (String s : split) {
String sLower = s.toLowerCase().trim();
boolean b = sLower.startsWith("charset=");
if (b){
return StringUtils.substringAfter(s, "charset=");
}
}
return null;
}
/** using this mess to provide logging, gzipping and httpmethod closing */
public InputStream getInputStream() throws RetsException {
try {
// get our underlying stream
InputStream inputStream = this.response.getEntity().getContent();
// gzipped aware checks
String contentEncoding = StringUtils.trimToEmpty(this.getHeader(CommonsHttpClient.CONTENT_ENCODING)).toLowerCase();
boolean gzipped = ArrayUtils.contains(CommonsHttpClient.DEFLATE_ENCODINGS.split(","),contentEncoding);
if( gzipped ) inputStream = new GZIPInputStream(inputStream);
final InputStream in = inputStream;
// the http method close wrapper (necessary)
return new InputStream(){
public int read() throws IOException {
return in.read();
}
public int read(byte[] b) throws IOException {
return in.read(b);
}
public int read(byte[] b, int off, int len) throws IOException {
return in.read(b, off, len);
}
public void close() throws IOException {
// connection release _AFTER_ the input stream has been read
try {
Closeables.close(in, true);
} catch (IOException e) {
// ignore
}
}
};
} catch (IOException e) {
throw new RetsException(e);
}
}
}

56
src/main/java/com/ossez/reoc/rets/client/CompactRowPolicy.java Normal file
View File

@ -0,0 +1,56 @@
package com.ossez.reoc.rets.client;
import org.apache.commons.logging.LogFactory;
public interface CompactRowPolicy {
/** fail fast and furiously */
public static final CompactRowPolicy STRICT = new CompactRowPolicy(){
public boolean apply(int row, String[] columns, String[] values) {
if( values.length != columns.length )
throw new IllegalArgumentException(String.format("Invalid number of result columns: got %s, expected %s",values.length, columns.length));
return true;
}};
/** drop everything thats suspect */
public static final CompactRowPolicy DROP = new CompactRowPolicy(){
public boolean apply(int row, String[] columns, String[] values) {
if (values.length != columns.length) {
LogFactory.getLog(CompactRowPolicy.class).warn(String.format("Row %s: Invalid number of result columns: got %s, expected ",row, values.length, columns.length));
return false;
}
return true;
}};
/** fail fast on long rows */
public static final CompactRowPolicy DEFAULT = new CompactRowPolicy(){
public boolean apply(int row, String[] columns, String[] values) {
if (values.length > columns.length) {
throw new IllegalArgumentException(String.format("Invalid number of result columns: got %s, expected %s",values.length, columns.length));
}
if (values.length < columns.length) {
LogFactory.getLog(CompactRowPolicy.class).warn(String.format("Row %s: Invalid number of result columns: got %s, expected ",row, values.length, columns.length));
}
return true;
}};
/** drop and log long rows, try to keep short rows */
public static final CompactRowPolicy DROP_LONG = new CompactRowPolicy(){
public boolean apply(int row, String[] columns, String[] values) {
if (values.length > columns.length) {
LogFactory.getLog(CompactRowPolicy.class).warn(String.format("Row %s: Invalid number of result columns: got %s, expected ",row, values.length, columns.length));
return false;
}
if (values.length < columns.length) {
LogFactory.getLog(CompactRowPolicy.class).warn(String.format("Row %s: Invalid number of result columns: got %s, expected ",row, values.length, columns.length));
}
return true;
}};
public boolean apply(int row, String[] columns, String[] values);
}

36
src/main/java/com/ossez/reoc/rets/client/GenericHttpRequest.java Normal file
View File

@ -0,0 +1,36 @@
package com.ossez.reoc.rets.client;
/**
* on the off chance you need an ad hoc request object...
*/
public class GenericHttpRequest extends VersionInsensitiveRequest {
public GenericHttpRequest(){
// noop
}
public GenericHttpRequest(String url){
this.mUrl = url;
}
/**
* throws an exception. GenericHttpRequest can't have a
* CapabilityUrl
* @param urls the CapabilityUrls object that has nothing we can use
*/
@Override
public void setUrl(CapabilityUrls urls) {
// do nothing
return;
}
/**
* expose the queryParameter interface to build query arguments.
* @param name the parameter name
* @param value the parameter value
*/
@Override
public void setQueryParameter(String name, String value) {
super.setQueryParameter(name, value);
}
}

73
src/main/java/com/ossez/reoc/rets/client/GetMetadataRequest.java Normal file
View File

@ -0,0 +1,73 @@
package com.ossez.reoc.rets.client;
import org.apache.commons.lang.StringUtils;
public class GetMetadataRequest extends VersionInsensitiveRequest {
private static final int COMPACT_FORMAT = 0;
private static final int STANDARD_XML_FORMAT = 1;
public static final String KEY_TYPE = "Type";
public static final String KEY_ID = "ID";
public static final String KEY_FORMAT = "Format";
public static final String FORMAT_STANDARD = "STANDARD-XML";
public static final String FORMAT_STANDARD_PREFIX = "STANDARD-XML:";
public static final String FORMAT_COMPACT = "COMPACT";
private int format;
private String standardXmlVersion;
public GetMetadataRequest(String type, String id) throws RetsException {
this(type, new String[] { id });
}
public GetMetadataRequest(String type, String[] ids) throws RetsException {
assertValidIds(ids);
type = "METADATA-" + type;
if (type.equals("METADATA-SYSTEM") || type.equals("METADATA-RESOURCE")) {
assertIdZeroOrStar(ids);
}
setQueryParameter(KEY_TYPE, type);
setQueryParameter(KEY_ID, StringUtils.join(ids, ":"));
setQueryParameter(KEY_FORMAT, FORMAT_STANDARD);
this.format = STANDARD_XML_FORMAT;
}
@Override
public void setUrl(CapabilityUrls urls) {
setUrl(urls.getGetMetadataUrl());
}
private void assertValidIds(String[] ids) throws InvalidArgumentException {
if (ids.length == 0) {
throw new InvalidArgumentException("Expecting at least one ID");
}
}
private void assertIdZeroOrStar(String[] ids) throws InvalidArgumentException {
if (ids.length != 1) {
throw new InvalidArgumentException("Expecting 1 ID, but found, " + ids.length);
}
if (!ids[0].equals("0") && !ids[0].equals("*")) {
throw new InvalidArgumentException("Expecting ID of 0 or *, but was " + ids[0]);
}
}
public void setCompactFormat() {
setQueryParameter(KEY_FORMAT, FORMAT_COMPACT);
this.format = COMPACT_FORMAT;
this.standardXmlVersion = null;
}
public boolean isCompactFormat() {
return (this.format == COMPACT_FORMAT);
}
public boolean isStandardXmlFormat() {
return (this.format == STANDARD_XML_FORMAT);
}
public String getStandardXmlVersion() {
return this.standardXmlVersion;
}
}

83
src/main/java/com/ossez/reoc/rets/client/GetMetadataResponse.java Normal file
View File

@ -0,0 +1,83 @@
package com.ossez.reoc.rets.client;
import java.io.IOException;
import java.io.InputStream;
import java.util.List;
import com.ossez.reoc.rets.common.metadata.JDomCompactBuilder;
import com.ossez.reoc.rets.common.metadata.JDomStandardBuilder;
import com.ossez.reoc.rets.common.metadata.MetaObject;
import com.ossez.reoc.rets.common.metadata.MetadataException;
import org.apache.commons.lang.math.NumberUtils;
import org.jdom.Element;
import org.jdom.JDOMException;
import org.jdom.Document;
import org.jdom.input.SAXBuilder;
public class GetMetadataResponse {
private MetaObject[] mMetadataObjs;
public GetMetadataResponse(InputStream stream, boolean compact, boolean isStrict) throws RetsException {
try {
SAXBuilder builder = new SAXBuilder();
Document document = builder.build(stream);
Element retsElement = document.getRootElement();
if (!retsElement.getName().equals("RETS")) {
throw new RetsException("Expecting RETS");
}
int replyCode = NumberUtils.toInt(retsElement.getAttributeValue("ReplyCode"));
if (ReplyCode.SUCCESS.equals(replyCode)) {
if (compact) {
handleCompactMetadata(document, isStrict);
} else {
handleStandardMetadata(document, isStrict);
}
} else if (ReplyCode.NO_METADATA_FOUND.equals(replyCode)) {
// No metadata is not an exceptional case
handleNoMetadataFound(retsElement);
} else {
InvalidReplyCodeException e = new InvalidReplyCodeException(replyCode);
e.setRemoteMessage(retsElement.getAttributeValue(retsElement.getAttributeValue("ReplyText")));
throw e;
}
} catch (JDOMException e) {
throw new RetsException(e);
} catch (IOException e) {
throw new RetsException(e);
}
}
private void handleNoMetadataFound(Element retsElement) throws RetsException {
List children = retsElement.getChildren();
if (children.size() != 0) {
throw new RetsException("Expecting 0 children when results");
}
this.mMetadataObjs = new MetaObject[0];
}
private void handleCompactMetadata(Document document, boolean isStrict) throws RetsException {
try {
JDomCompactBuilder builder = new JDomCompactBuilder();
builder.setStrict(isStrict);
this.mMetadataObjs = builder.parse(document);
} catch (MetadataException e) {
throw new RetsException(e);
}
}
private void handleStandardMetadata(Document document, boolean isStrict) throws RetsException {
try {
JDomStandardBuilder builder = new JDomStandardBuilder();
builder.setStrict(isStrict);
this.mMetadataObjs = builder.parse(document);
} catch (MetadataException e) {
throw new RetsException(e);
}
}
public MetaObject[] getMetadata() {
return this.mMetadataObjs;
}
}

12
src/main/java/com/ossez/reoc/rets/client/GetObjectIterator.java Normal file
View File

@ -0,0 +1,12 @@
package com.ossez.reoc.rets.client;
import java.io.Closeable;
import java.util.Iterator;
/**
* Iterator for SingleResoponseObjects
* @param <G>
*/
public interface GetObjectIterator<G extends SingleObjectResponse> extends Closeable, Iterator<G>{
// noop
}

101
src/main/java/com/ossez/reoc/rets/client/GetObjectRequest.java Normal file
View File

@ -0,0 +1,101 @@
package com.ossez.reoc.rets.client;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.Set;
import java.util.Map;
import org.apache.commons.lang.ObjectUtils;
import org.apache.commons.lang.StringUtils;
public class GetObjectRequest extends VersionInsensitiveRequest {
public static final String KEY_RESOURCE = "Resource";
public static final String KEY_TYPE = "Type";
public static final String KEY_LOCATION = "Location";
public static final String KEY_ID = "ID";
private final Map mMap;
public GetObjectRequest(String resource, String type) {
this(resource, type, new String[] { "*/*" });
}
public GetObjectRequest(String resource, String type, String[] acceptMimeTypes) {
setQueryParameter(KEY_RESOURCE, resource);
setQueryParameter(KEY_TYPE, type);
this.mMap = new HashMap();
setHeader("Accept", StringUtils.join(acceptMimeTypes, ", "));
}
@Override
public void setUrl(CapabilityUrls urls) {
setUrl(urls.getGetObjectUrl());
}
public void setLocationOnly(boolean flag) {
if (flag) {
setQueryParameter(KEY_LOCATION, "1");
} else {
setQueryParameter(KEY_LOCATION, null);
}
}
public void addObject(String resourceEntity, String id) {
if (id == null)
throw new NullPointerException("Object id should not be null. "
+ "Cannot remove object already added to request.");
Object cur = this.mMap.get(resourceEntity);
if (id.equals("*")) {
this.mMap.put(resourceEntity, id);
} else if (cur == null) {
this.mMap.put(resourceEntity, id);
} else if (cur instanceof String) {
if (ObjectUtils.equals(cur, "*")) {
return;
}
if (ObjectUtils.equals(cur, id)) {
return;
}
Set s = new HashSet();
s.add(cur);
s.add(id);
this.mMap.put(resourceEntity, s);
} else if (cur instanceof Set) {
((Set) cur).add(id);
} else {
/* NOTREACHED */
throw new RuntimeException(resourceEntity + " has invalid value " + "of type " + cur.getClass().getName());
}
setQueryParameter(KEY_ID, makeIdStr());
}
private String makeIdStr() {
StringBuffer id = new StringBuffer();
Iterator iter = this.mMap.keySet().iterator();
while (iter.hasNext()) {
String key = (String) iter.next();
id.append(key);
Object cur = this.mMap.get(key);
if (cur instanceof String) {
id.append(":");
id.append(cur);
} else if (cur instanceof Set) {
Iterator iter2 = ((Set) cur).iterator();
while (iter2.hasNext()) {
String val = (String) iter2.next();
id.append(":");
id.append(val);
}
} else {
throw new RuntimeException(key + " has invalid value of " + "type " + cur.getClass().getName());
}
if (iter.hasNext()) {
id.append(",");
}
}
return id.toString();
}
}

190
src/main/java/com/ossez/reoc/rets/client/GetObjectResponse.java Normal file
View File

@ -0,0 +1,190 @@
package com.ossez.reoc.rets.client;
import java.io.IOException;
import java.io.InputStream;
import java.util.Map;
import java.util.NoSuchElementException;
import com.ossez.reoc.rets.common.util.CaseInsensitiveTreeMap;
import org.apache.commons.lang.math.NumberUtils;
import org.apache.http.HeaderElement;
import org.apache.http.NameValuePair;
import org.apache.http.message.BasicHeaderValueParser;
import org.jdom.Document;
import org.jdom.Element;
import org.jdom.JDOMException;
import org.jdom.input.SAXBuilder;
public class GetObjectResponse{
private static final int DEFAULT_BUFFER_SIZE = 8192;
private final static GetObjectIterator<SingleObjectResponse> EMPTY_OBJECT_RESPONSE_ITERATOR = new GetObjectIterator<SingleObjectResponse>() {
public boolean hasNext() {
return false;
}
public SingleObjectResponse next() {
throw new NoSuchElementException();
}
public void close() {
/* no op */
}
public void remove() {
/* no op */
}
};
private final Map headers;
private final InputStream inputStream;
private final boolean isMultipart;
/** Indicate whether the response was empty */
private boolean emptyResponse;
/** Indicate whether this GetObjectResponse is exhausted, i.e. has no objects */
private boolean exhausted;
public GetObjectResponse(Map headers, InputStream in) throws RetsException {
this.emptyResponse = false;
this.exhausted = false;
this.headers = new CaseInsensitiveTreeMap(headers);
this.isMultipart = getType().contains("multipart");
this.inputStream = in;
boolean isXml = getType().equals("text/xml");
boolean containsContentId = headers.containsKey(SingleObjectResponse.CONTENT_ID);
// non multipart request that returns text/xml and does NOT have a Context-ID header, must only be a non-zero response code
boolean nonMultiPart_xml_withoutContentId = !this.isMultipart && isXml && !containsContentId;
// multipart request that returns text/xml can only be a non-zero response code
boolean multiPart_xml = this.isMultipart && isXml;
if ( multiPart_xml || nonMultiPart_xml_withoutContentId ) {
int replyCode = 0;
try {
// GetObjectResponse is empty, because we have a Rets ReplyCode
this.emptyResponse = true;
SAXBuilder builder = new SAXBuilder();
Document mDocument = builder.build(in);
Element root = mDocument.getRootElement();
if (root.getName().equals("RETS")) {
replyCode = NumberUtils.toInt(root.getAttributeValue("ReplyCode"));
// success
if (ReplyCode.SUCCESS.equals(replyCode)) return;
// no object found - that's fine
if (ReplyCode.NO_OBJECT_FOUND.equals(replyCode)) return;
throw new InvalidReplyCodeException(replyCode);
}
// no other possibilities
throw new RetsException("Malformed response [multipart="+this.isMultipart+", content-type=text/xml]. " +
"Content id did not exist in response and response was not valid rets response.");
} catch (JDOMException e) {
throw new RetsException(e);
} catch (IOException e) {
throw new RetsException(e);
}
}
}
public String getType() {
return (String) this.headers.get("Content-Type");
}
public String getBoundary() {
String contentTypeValue = getType();
HeaderElement[] contentType = BasicHeaderValueParser.parseElements(contentTypeValue, new BasicHeaderValueParser());
if (contentType.length != 1)
throw new IllegalArgumentException("Multipart response appears to have a bad Content-Type: header value: "
+ contentTypeValue);
NameValuePair boundaryNV = contentType[0].getParameterByName("boundary");
if (boundaryNV == null)
return null;
return unescapeBoundary(boundaryNV.getValue());
}
private static String unescapeBoundary(String boundaryValue) {
if (boundaryValue.startsWith("\""))
boundaryValue = boundaryValue.substring(1);
if (boundaryValue.endsWith("\""))
boundaryValue = boundaryValue.substring(0, boundaryValue.length() - 1);
return boundaryValue;
}
/**
* @return GetObjectIterator, which iterates over SingleObjectResponse
* objects.
*
* @throws RetsException
*/
public <T extends SingleObjectResponse> GetObjectIterator<T> iterator() throws RetsException {
return iterator(DEFAULT_BUFFER_SIZE);
}
/**
* @return GetObjectIterator, which iterates over SingleObjectResponse
* objects.
*
* @param bufferSize How large a buffer should be used for underlying
* streams.
*
* @throws RetsException
*/
public <T extends SingleObjectResponse> GetObjectIterator<T> iterator(int bufferSize) throws RetsException {
if(this.exhausted )
throw new RetsException("response was exhausted - cannot request iterator a second time");
if( this.emptyResponse )
return (GetObjectIterator<T>) EMPTY_OBJECT_RESPONSE_ITERATOR;
if( this.isMultipart ){
try {
return GetObjectResponseIterator.createIterator(this, bufferSize);
} catch (Exception e) {
throw new RetsException("Error creating multipart GetObjectIterator", e);
}
}
// no other possibilities
return new NonMultipartGetObjectResponseIterator(this.headers, this.inputStream);
}
public InputStream getInputStream() {
return this.inputStream;
}
}
/** Used to implement GetObjectIterator for a non multipart response. */
final class NonMultipartGetObjectResponseIterator implements GetObjectIterator {
private boolean exhausted;
private final Map headers;
private final InputStream inputStream;
public NonMultipartGetObjectResponseIterator(Map headers, InputStream in){
this.exhausted = false;
this.headers = headers;
this.inputStream = in;
}
public void close() throws IOException {
this.inputStream.close();
}
public void remove() {
throw new UnsupportedOperationException();
}
public boolean hasNext() {
return !this.exhausted;
}
public SingleObjectResponse next() {
if( this.exhausted )
throw new NoSuchElementException("stream exhausted");
this.exhausted = true;
return new SingleObjectResponse(this.headers, this.inputStream);
}
}

140
src/main/java/com/ossez/reoc/rets/client/GetObjectResponseIterator.java Normal file
View File

@ -0,0 +1,140 @@
package com.ossez.reoc.rets.client;
import java.io.BufferedInputStream;
import java.io.IOException;
import java.io.PushbackInputStream;
import java.util.HashMap;
import java.util.Map;
import java.util.NoSuchElementException;
import org.apache.commons.lang.StringUtils;
public class GetObjectResponseIterator<T extends SingleObjectResponse> implements GetObjectIterator<T> {
public static final char CR = '\r';
public static final char LF = '\n';
public static final String EOL = CR+""+LF;
public static final String BS = "--";
private final PushbackInputStream multipartStream;
private final String boundary;
private Boolean hasNext;
public static <T extends SingleObjectResponse> GetObjectIterator<T> createIterator(final GetObjectResponse response, int streamBufferSize) throws Exception {
String boundary = response.getBoundary();
if (boundary != null)
return new GetObjectResponseIterator(response, boundary, streamBufferSize);
return new GetObjectIterator<T>() {
public void close() throws IOException{
response.getInputStream().close();
}
public boolean hasNext() {
return false;
}
public T next() {
throw new NoSuchElementException();
}
public void remove() {
throw new UnsupportedOperationException("");
}
};
}
private GetObjectResponseIterator(GetObjectResponse response, String boundary, int streamBufferSize) throws Exception {
this.boundary = boundary;
BufferedInputStream input = new BufferedInputStream(response.getInputStream(), streamBufferSize);
this.multipartStream = new PushbackInputStream(input, BS.length() + this.boundary.length() + EOL.length());
}
public boolean hasNext() {
if (this.hasNext != null)
return this.hasNext.booleanValue();
try {
this.hasNext = new Boolean(this.getHaveNext());
return this.hasNext.booleanValue();
} catch (IOException e) {
throw new RuntimeException(e);
}
}
public T next() {
if (!this.hasNext())
throw new NoSuchElementException();
this.hasNext = null;
try {
return getNext();
} catch (Exception e) {
throw new RuntimeException(e);
}
}
public void remove() {
throw new UnsupportedOperationException();
}
public void close() throws IOException {
this.multipartStream.close();
}
private boolean getHaveNext() throws IOException {
String line = null;
while ((line = this.readLine()) != null) {
if (line.equals(BS+this.boundary))
return true;
if (line.equals(BS+this.boundary+BS))
return false;
}
return false;
}
private T getNext() throws Exception {
Map headers = new HashMap();
String header = null;
while (StringUtils.isNotEmpty(header = this.readLine())) {
int nvSeperatorIndex = header.indexOf(':');
if (nvSeperatorIndex == -1){
headers.put(header, "");
} else {
String name = header.substring(0, nvSeperatorIndex);
String value = header.substring(nvSeperatorIndex + 1).trim();
headers.put(name, value);
}
}
return (T)new SingleObjectResponse(headers, new SinglePartInputStream(this.multipartStream, BS+this.boundary));
}
// TODO find existing library to do this
private String readLine() throws IOException {
boolean eolReached = false;
StringBuffer line = new StringBuffer();
int currentChar = -1;
while (!eolReached && (currentChar = this.multipartStream.read()) != -1) {
eolReached = (currentChar == CR || currentChar == LF);
if (!eolReached)
line.append((char) currentChar);
}
if (currentChar == -1 && line.length() == 0)
return null;
if (currentChar == CR) {
int nextChar = this.multipartStream.read();
if (nextChar != LF)
this.multipartStream.unread(new byte[] { (byte) nextChar });
}
return line.toString();
}
}

7
src/main/java/com/ossez/reoc/rets/client/InvalidArgumentException.java Normal file
View File

@ -0,0 +1,7 @@
package com.ossez.reoc.rets.client;
public class InvalidArgumentException extends RetsException {
public InvalidArgumentException(String message) {
super(message);
}
}

12
src/main/java/com/ossez/reoc/rets/client/InvalidHttpStatusException.java Normal file
View File

@ -0,0 +1,12 @@
package com.ossez.reoc.rets.client;
import org.apache.http.StatusLine;
public class InvalidHttpStatusException extends RetsException {
public InvalidHttpStatusException(StatusLine status) {
super("Status code (" + status.getStatusCode() + ") " + status.getReasonPhrase());
}
public InvalidHttpStatusException(StatusLine status, String message) {
super("Status code (" + status.getStatusCode() + ") " + status.getReasonPhrase() +" '"+message+"'");
}
}

44
src/main/java/com/ossez/reoc/rets/client/InvalidReplyCodeException.java Normal file
View File

@ -0,0 +1,44 @@
package com.ossez.reoc.rets.client;
import org.apache.commons.lang.SystemUtils;
/**
* Exception class for invalid reply codes from a Rets server
*/
public class InvalidReplyCodeException extends RetsException {
private final ReplyCode mReplyCode;
private String mMsg;
private String mReqinfo;
public InvalidReplyCodeException(int replyCodeValue) {
this.mReplyCode = ReplyCode.fromValue(replyCodeValue);
}
public InvalidReplyCodeException(ReplyCode replyCode) {
this.mReplyCode = replyCode;
}
@Override
public String getMessage() {
StringBuffer sb = new StringBuffer(this.mReplyCode.toString());
if (this.mMsg != null) {
sb.append(SystemUtils.LINE_SEPARATOR + this.mMsg);
}
if (this.mReqinfo != null) {
sb.append(SystemUtils.LINE_SEPARATOR + this.mReqinfo);
}
return sb.toString();
}
public int getReplyCodeValue() {
return this.mReplyCode.getValue();
}
public void setRemoteMessage(String msg) {
this.mMsg = msg;
}
public void setRequestInfo(String reqinfo) {
this.mReqinfo = reqinfo;
}
}

17
src/main/java/com/ossez/reoc/rets/client/InvalidReplyCodeHandler.java Normal file
View File

@ -0,0 +1,17 @@
package com.ossez.reoc.rets.client;
public interface InvalidReplyCodeHandler {
InvalidReplyCodeHandler FAIL = new InvalidReplyCodeHandler() {
public void invalidRetsReplyCode(int replyCode) throws InvalidReplyCodeException {
throw new InvalidReplyCodeException(replyCode);
}
public void invalidRetsStatusReplyCode(int replyCode) throws InvalidReplyCodeException {
throw new InvalidReplyCodeException(replyCode);
}
};
public void invalidRetsReplyCode(int replyCode) throws InvalidReplyCodeException;
public void invalidRetsStatusReplyCode(int replyCode) throws InvalidReplyCodeException;
}

112
src/main/java/com/ossez/reoc/rets/client/KeyValueResponse.java Normal file
View File

@ -0,0 +1,112 @@
package com.ossez.reoc.rets.client;
import java.util.List;
import java.util.StringTokenizer;
import java.io.InputStream;
import java.io.IOException;
import org.jdom.Element;
import org.jdom.JDOMException;
import org.jdom.Document;
import org.jdom.input.SAXBuilder;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang.math.NumberUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
abstract public class KeyValueResponse {
protected static final String CRLF = "\r\n";
private static final Log LOG = LogFactory.getLog(KeyValueResponse.class);
protected Document mDoc;
protected int mReplyCode;
protected boolean mStrict;
public KeyValueResponse() {
this.mStrict = false;
}
public void parse(InputStream stream, RetsVersion mVersion) throws RetsException {
try {
SAXBuilder builder = new SAXBuilder();
this.mDoc = builder.build(stream);
Element retsElement = this.mDoc.getRootElement();
if (!retsElement.getName().equals("RETS")) {
throw new RetsException("Expecting RETS");
}
int replyCode = NumberUtils.toInt(retsElement.getAttributeValue("ReplyCode"));
this.mReplyCode = replyCode;
if (!isValidReplyCode(replyCode)) {
throw new InvalidReplyCodeException(replyCode);
}
Element capabilityContainer;
if (RetsVersion.RETS_10.equals(mVersion)) {
capabilityContainer = retsElement;
} else {
List children = retsElement.getChildren();
if (children.size() != 1) {
throw new RetsException("Invalid number of children: " + children.size());
}
capabilityContainer = (Element) children.get(0);
if (!capabilityContainer.getName().equals("RETS-RESPONSE")) {
throw new RetsException("Expecting RETS-RESPONSE");
}
}
this.handleRetsResponse(capabilityContainer);
} catch (JDOMException e) {
throw new RetsException(e);
} catch (IOException e) {
throw new RetsException(e);
}
}
protected boolean isValidReplyCode(int replyCode) {
return (ReplyCode.SUCCESS.equals(replyCode));
}
private void handleRetsResponse(Element retsResponse) throws RetsException {
StringTokenizer tokenizer = new StringTokenizer(retsResponse.getText(), CRLF);
while (tokenizer.hasMoreTokens()) {
String line = tokenizer.nextToken();
String splits[] = StringUtils.split(line, "=");
String key = splits[0].trim();
// guard against a missing value in a KeyValueResponse
String value = splits.length > 1 ? splits[1].trim() : "";
if (LOG.isDebugEnabled()) {
LOG.debug("<" + key + "> -> <" + value + ">");
}
this.handleKeyValue(key, value);
}
}
protected abstract void handleKeyValue(String key, String value) throws RetsException;
public void setStrict(boolean strict) {
this.mStrict = strict;
}
public boolean isStrict() {
return this.mStrict;
}
protected boolean matchKey(String key, String value) {
if (this.mStrict)
return key.equals(value);
return key.equalsIgnoreCase(value);
}
protected void assertStrictWarning(Log log, String message) throws RetsException {
if (this.mStrict)
throw new RetsException(message);
log.warn(message);
}
}

23
src/main/java/com/ossez/reoc/rets/client/LoginRequest.java Normal file
View File

@ -0,0 +1,23 @@
package com.ossez.reoc.rets.client;
public class LoginRequest extends VersionInsensitiveRequest {
@Override
public void setUrl(CapabilityUrls urls) {
setUrl(urls.getLoginUrl());
}
public void setBrokerCode(String code, String branch) {
if (code == null) {
setQueryParameter(KEY_BROKERCODE, null);
} else {
if (branch == null) {
setQueryParameter(KEY_BROKERCODE, code);
} else {
setQueryParameter(KEY_BROKERCODE, code + "," + branch);
}
}
}
public static final String KEY_BROKERCODE = "BrokerCode";
}

197
src/main/java/com/ossez/reoc/rets/client/LoginResponse.java Normal file
View File

@ -0,0 +1,197 @@
package com.ossez.reoc.rets.client;
import java.io.InputStream;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.HashSet;
import java.util.Set;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang.math.NumberUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
public class LoginResponse extends KeyValueResponse {
private static final String BROKER_KEY = "Broker";
private static final String MEMBER_NAME_KEY = "MemberName";
private static final String METADATA_VER_KEY = "MetadataVersion";
private static final String MIN_METADATA_VER_KEY = "MinMetadataVersion";
private static final String USER_INFO_KEY = "User";
private static final String OFFICE_LIST_KEY = "OfficeList";
private static final String BALANCE_KEY = "Balance";
private static final String TIMEOUT_KEY = "TimeoutSeconds";
private static final String PWD_EXPIRE_KEY = "Expr";
private static final String METADATA_TIMESTAMP_KEY = "MetadataTimestamp";
private static final String MIN_METADATA_TIMESTAMP_KEY = "MinMetadataTimestamp";
private static final Log LOG = LogFactory.getLog(LoginResponse.class);
private String sessionId;
private String memberName;
private String userInformation;
private String broker;
private String metadataVersion;
private String minMetadataVersion;
private String metadataTimestamp;
private String minMetadataTimestamp;
private String officeList;
private String balance;
private int sessionTimeout;
private String passwordExpiration;
private CapabilityUrls capabilityUrls;
private Set brokerCodes;
public LoginResponse(String loginUrl) {
super();
this.brokerCodes = new HashSet();
URL url = null;
try {
url = new URL(loginUrl);
} catch (MalformedURLException e) {
LOG.warn("Bad URL: " + loginUrl);
}
this.capabilityUrls = new CapabilityUrls(url);
}
public LoginResponse() {
super();
this.capabilityUrls = new CapabilityUrls();
}
@Override
public void parse(InputStream stream, RetsVersion version) throws RetsException {
super.parse(stream, version);
if (ReplyCode.BROKER_CODE_REQUIRED.equals(this.mReplyCode)) {
throw new BrokerCodeRequredException(this.brokerCodes);
}
}
@Override
protected boolean isValidReplyCode(int replyCode) {
return (super.isValidReplyCode(replyCode) || ReplyCode.BROKER_CODE_REQUIRED.equals(replyCode));
}
@Override
protected void handleKeyValue(String key, String value) throws RetsException {
if (ReplyCode.BROKER_CODE_REQUIRED.equals(this.mReplyCode)) {
if (matchKey(key, BROKER_KEY)) {
String[] strings = StringUtils.split(value, ",");
if (strings.length > 0 && strings.length < 3) {
this.brokerCodes.add(strings);
} else {
throw new RetsException("Invalid broker/branch code: " + value);
}
}
}
if (matchKey(key, BROKER_KEY)) {
this.broker = value;
} else if (matchKey(key, MEMBER_NAME_KEY)) {
this.memberName = value;
} else if (matchKey(key, METADATA_VER_KEY)) {
this.metadataVersion = value;
} else if (matchKey(key, MIN_METADATA_VER_KEY)) {
this.minMetadataVersion = value;
} else if (matchKey(key, METADATA_TIMESTAMP_KEY)) {
this.metadataTimestamp = value;
} else if (matchKey(key, MIN_METADATA_TIMESTAMP_KEY)) {
this.minMetadataTimestamp = value;
} else if (matchKey(key, USER_INFO_KEY)) {
this.userInformation = value;
} else if (matchKey(key, OFFICE_LIST_KEY)) {
this.officeList = value;
} else if (matchKey(key, BALANCE_KEY)) {
this.balance = value;
} else if (matchKey(key, TIMEOUT_KEY)) {
this.sessionTimeout = NumberUtils.toInt(value);
} else if (matchKey(key, PWD_EXPIRE_KEY)) {
this.passwordExpiration = value;
} else if (matchKey(key, CapabilityUrls.ACTION_URL)) {
this.capabilityUrls.setActionUrl(value);
} else if (matchKey(key, CapabilityUrls.CHANGE_PASSWORD_URL)) {
this.capabilityUrls.setChangePasswordUrl(value);
} else if (matchKey(key, CapabilityUrls.GET_OBJECT_URL)) {
this.capabilityUrls.setGetObjectUrl(value);
} else if (matchKey(key, CapabilityUrls.LOGIN_URL)) {
this.capabilityUrls.setLoginUrl(value);
} else if (matchKey(key, CapabilityUrls.LOGIN_COMPLETE_URL)) {
this.capabilityUrls.setLoginCompleteUrl(value);
} else if (matchKey(key, CapabilityUrls.LOGOUT_URL)) {
this.capabilityUrls.setLogoutUrl(value);
} else if (matchKey(key, CapabilityUrls.SEARCH_URL)) {
this.capabilityUrls.setSearchUrl(value);
} else if (matchKey(key, CapabilityUrls.GET_METADATA_URL)) {
this.capabilityUrls.setGetMetadataUrl(value);
} else if (matchKey(key, CapabilityUrls.UPDATE_URL)) {
this.capabilityUrls.setUpdateUrl(value);
}else if (matchKey(key, CapabilityUrls.SERVER_INFO_URL)) {
this.capabilityUrls.setServerInfo(value);
LOG.warn("Depreciated: " + key + " -> " + value);
} else if (matchKey(key, "Get")) {
LOG.warn("Found bad key: Get -> " + value);
// FIX ME: Should not get this
} else {
if (key.substring(0, 2).equalsIgnoreCase("X-")) {
LOG.warn("Unknown experimental key: " + key + " -> " + value);
} else {
assertStrictWarning(LOG, "Invalid login response key: " + key + " -> " + value);
}
}
}
public String getMemberName() {
return this.memberName;
}
public String getUserInformation() {
return this.userInformation;
}
public String getBroker() {
return this.broker;
}
public String getMetadataVersion() {
return this.metadataVersion;
}
public String getMinMetadataVersion() {
return this.minMetadataVersion;
}
public String getMetadataTimestamp() {
return this.metadataTimestamp;
}
public String getMinMetadataTimestamp() {
return this.minMetadataTimestamp;
}
public String getOfficeList() {
return this.officeList;
}
public String getBalance() {
return this.balance;
}
public int getSessionTimeout() {
return this.sessionTimeout;
}
public String getPasswordExpiration() {
return this.passwordExpiration;
}
public CapabilityUrls getCapabilityUrls() {
return this.capabilityUrls;
}
public String getSessionId() {
return this.sessionId;
}
public void setSessionId(String sessionId) {
this.sessionId = sessionId;
}
}

9
src/main/java/com/ossez/reoc/rets/client/LogoutRequest.java Normal file
View File

@ -0,0 +1,9 @@
package com.ossez.reoc.rets.client;
public class LogoutRequest extends VersionInsensitiveRequest {
@Override
public void setUrl(CapabilityUrls urls) {
setUrl(urls.getLogoutUrl());
}
}

41
src/main/java/com/ossez/reoc/rets/client/LogoutResponse.java Normal file
View File

@ -0,0 +1,41 @@
package com.ossez.reoc.rets.client;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
public class LogoutResponse extends KeyValueResponse {
private static final Log LOG = LogFactory.getLog(LogoutResponse.class);
private static final String CONNECT_TIME_KEY = "ConnectTime";
private static final String BILLING_KEY = "Billing";
private static final String SIGNOFF_KEY = "SignOffMessage";
private String seconds;
private String billingInfo;
private String logoutMessage;
@Override
protected void handleKeyValue(String key, String value) throws RetsException {
if (matchKey(key, CONNECT_TIME_KEY)) {
this.seconds = value;
} else if (matchKey(key, BILLING_KEY)) {
this.billingInfo = value;
} else if (matchKey(key, SIGNOFF_KEY)) {
this.logoutMessage = value;
} else {
assertStrictWarning(LOG, "Invalid logout response key: " + key + " -> " + value);
}
}
public String getSeconds() {
return this.seconds;
}
public String getBillingInfo() {
return this.billingInfo;
}
public String getLogoutMessage() {
return this.logoutMessage;
}
}

61
src/main/java/com/ossez/reoc/rets/client/MetaCollectorAdapter.java Normal file
View File

@ -0,0 +1,61 @@
package com.ossez.reoc.rets.client;
import com.ossez.reoc.rets.common.metadata.MetaCollector;
import com.ossez.reoc.rets.common.metadata.MetaObject;
import com.ossez.reoc.rets.common.metadata.MetadataType;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
public abstract class MetaCollectorAdapter implements MetaCollector {
public MetaObject[] getMetadata(MetadataType type, String path) {
return getSome(type, path, "0");
}
public MetaObject[] getMetadataRecursive(MetadataType type, String path) {
return getSome(type, path, "*");
}
private MetaObject[] getSome(MetadataType type, String path, String sfx) {
boolean compact = Boolean.getBoolean("rets-client.metadata.compact");
try {
GetMetadataRequest req;
if (path == null || path.equals("")) {
req = new GetMetadataRequest(type.name(), sfx);
} else {
String[] ppath = StringUtils.split(path, ":");
String[] id = new String[ppath.length + 1];
System.arraycopy(ppath, 0, id, 0, ppath.length);
id[ppath.length] = sfx;
req = new GetMetadataRequest(type.name(), id);
}
if (compact) {
req.setCompactFormat();
}
GetMetadataResponse response;
response = doRequest(req);
return response.getMetadata();
} catch (RetsException e) {
LOG.error("bad metadata request", e);
return null;
}
}
/**
* Perform operation of turning a GetMetadataRequest into
* a GetMetadataResponse
*
* @param req Requested metadata
* @return parsed MetaObjects
*
* @throws RetsException if an error occurs
*/
protected abstract GetMetadataResponse doRequest(GetMetadataRequest req) throws RetsException;
private static final Log LOG = LogFactory.getLog(MetaCollectorAdapter.class);
}

22
src/main/java/com/ossez/reoc/rets/client/MetaCollectorImpl.java Normal file
View File

@ -0,0 +1,22 @@
/*
* cart: CRT's Awesome RETS Tool
*
* Author: David Terrell
* Copyright (c) 2003, The National Association of REALTORS
* Distributed under a BSD-style license. See LICENSE.TXT for details.
*/
package com.ossez.reoc.rets.client;
public class MetaCollectorImpl extends MetaCollectorAdapter {
private final RetsTransport mTransport;
public MetaCollectorImpl(RetsTransport transport) {
this.mTransport = transport;
}
@Override
protected GetMetadataResponse doRequest(GetMetadataRequest req) throws RetsException {
return this.mTransport.getMetadata(req);
}
}

23
src/main/java/com/ossez/reoc/rets/client/NetworkEventMonitor.java Normal file
View File

@ -0,0 +1,23 @@
package com.ossez.reoc.rets.client;
/**
* A client can register a monitor for network events
*/
public interface NetworkEventMonitor
{
/**
* inform the client app that an event has started.
* the client app can return an object, which will be passed
* to eventFinish().
*
* @param message a message describing the event
* @return an object to be passed to eventFinish, or null
*/
public Object eventStart(String message);
/**
* Inform the client app that the previous event has completed
*
* @param o the object returned from eventStart
*/
public void eventFinish(Object o);
}

13
src/main/java/com/ossez/reoc/rets/client/NullNetworkEventMonitor.java Normal file
View File

@ -0,0 +1,13 @@
package com.ossez.reoc.rets.client;
public class NullNetworkEventMonitor implements NetworkEventMonitor {
public Object eventStart(String message) {
return null;
}
public void eventFinish(Object o) {
//noop
}
}

100
src/main/java/com/ossez/reoc/rets/client/ReplyCode.java Normal file
View File

@ -0,0 +1,100 @@
package com.ossez.reoc.rets.client;
import java.util.Map;
import java.util.HashMap;
public class ReplyCode {
// static initialization loop.... this declaration _MUST_ come before the members
private static final Map<Integer,ReplyCode> CODES = new HashMap<Integer,ReplyCode>();
public static final ReplyCode SUCCESS = new ReplyCode(0, "Success");
public static final ReplyCode ZERO_BALANCE = new ReplyCode(20003, "Zero balance");
public static final ReplyCode BROKER_CODE_REQUIRED = new ReplyCode(20012, "Broker code required");
public static final ReplyCode BROKER_CODE_INVALID = new ReplyCode(20013, "Broker Code Invalid");
public static final ReplyCode ADDTIONAL_LOGIN_NOT_PREMITTED = new ReplyCode(20022, "Additional login not permitted");
public static final ReplyCode MISCELLANEOUS_LOGIN_ERROR = new ReplyCode(20036, "Miscellaneous server login error");
public static final ReplyCode CLIENT_PASSWORD_INVALID = new ReplyCode(20037, "Client passsword invalid");
public static final ReplyCode SERVER_TEMPORARILY_DISABLED = new ReplyCode(20050, "Server temporarily disabled");
public static final ReplyCode UNKNOWN_QUERY_FIELD = new ReplyCode(20200, "Unknown Query Field");
public static final ReplyCode NO_RECORDS_FOUND = new ReplyCode(20201, "No Records Found");
public static final ReplyCode INVALID_SELECT = new ReplyCode(20202, "Invalid select");
public static final ReplyCode MISCELLANOUS_SEARCH_ERROR = new ReplyCode(20203, "Miscellaneous search error");
public static final ReplyCode INVALID_QUERY_SYNTAX = new ReplyCode(20206, "Invalid query syntax");
public static final ReplyCode UNAUTHORIZED_QUERY = new ReplyCode(20207, "Unauthorized query");
public static final ReplyCode MAXIMUM_RECORDS_EXCEEDED = new ReplyCode(20208, "Maximum records exceeded");
public static final ReplyCode SEARCH_TIMED_OUT = new ReplyCode(20209, "Search timed out");
public static final ReplyCode TOO_MANY_OUTSTANDING_QUERIES = new ReplyCode(20210, "Too many outstanding queries");
public static final ReplyCode INVALID_RESOURCE_GETOBJECT = new ReplyCode(20400, "Invalid Resource");
public static final ReplyCode INVALID_TYPE_GETOBJECT = new ReplyCode(20401, "Invalid Type");
public static final ReplyCode INVALID_IDENTIFIER_GETOBJECT = new ReplyCode(20402, "Invalid Identifier");
public static final ReplyCode NO_OBJECT_FOUND = new ReplyCode(20403, "No Object Found");
public static final ReplyCode UNSUPPORTED_MIME_TYPE_GETOBJECT = new ReplyCode(20406, "Unsupported MIME Type");
public static final ReplyCode UNAUTHORIZED_RETRIEVAL_GETOBJECT = new ReplyCode(20407, "Unauthorized Retrieval");
public static final ReplyCode RESOURCE_UNAVAILABLE_GETOBJECT = new ReplyCode(20408, "Resource Unavailable");
public static final ReplyCode OBJECT_UNAVAILABLE = new ReplyCode(20409, "Object Unavailable");
public static final ReplyCode REQUEST_TOO_LARGE_GETOBJECT = new ReplyCode(20410, "Request Too Large");
public static final ReplyCode TIMEOUT_GETOBJECT = new ReplyCode(20411, "Timeout");
public static final ReplyCode TOO_MANY_OUTSTANDING_QUERIES_GETOBJECT = new ReplyCode(20412,"Too Many Outstanding Queries");
public static final ReplyCode MISCELLANEOUS_ERROR_GETOBJECT = new ReplyCode(20413, "Miscellaneous Error");
public static final ReplyCode INVALID_RESOURCE = new ReplyCode(20500, "Invalid resource");
public static final ReplyCode INVALID_TYPE = new ReplyCode(20501, "Invalid type");
public static final ReplyCode INVALID_IDENTIFIER = new ReplyCode(20502, "Invalid identifier");
public static final ReplyCode NO_METADATA_FOUND = new ReplyCode(20503, "No metadata found");
public static final ReplyCode UNSUPPORTED_MIME_TYPE = new ReplyCode(20506, "Unsupported MIME type");
public static final ReplyCode UNAUTHORIZED_RETRIEVAL = new ReplyCode(20507, "Unauthorized retrieval");
public static final ReplyCode RESOURCE_UNAVAILABLE = new ReplyCode(20508, "Resource unavailable");
public static final ReplyCode METADATA_UNAVAILABLE = new ReplyCode(20509, "Metadata unavailable");
public static final ReplyCode REQUEST_TOO_LARGE = new ReplyCode(20510, "Request too large");
public static final ReplyCode TIMEOUT = new ReplyCode(20511, "Timeout");
public static final ReplyCode TOO_MANY_OUSTANDING_REQUESTS = new ReplyCode(20512, "Too many outstanding requests");
public static final ReplyCode MISCELLANEOUS_ERROR = new ReplyCode(20513, "Miscellanous error");
public static final ReplyCode REQUESTED_DTD_UNAVAILABLE = new ReplyCode(20514, "Requested DTD unvailable");
private final int mValue;
private final String mMessage;
private ReplyCode(int value, String message) {
this.mValue = value;
this.mMessage = message;
if (CODES.containsValue(new Integer(value)))
throw new IllegalArgumentException(String.format("value already used: %s ( %s ) ",value,message));
CODES.put(new Integer(value), this);
}
@Override
public boolean equals(Object o) {
if (!(o instanceof ReplyCode)) {
return false;
}
ReplyCode rhs = (ReplyCode) o;
return (this.mValue == rhs.mValue);
}
public boolean equals(int value) {
return this.mValue == value;
}
@Override
public String toString() {
return String.format("%s (%s)",this.mValue,this.mMessage);
}
public int getValue() {
return this.mValue;
}
public String getMessage() {
return this.mMessage;
}
public static ReplyCode fromValue(int value) {
ReplyCode replyCode = CODES.get(new Integer(value));
if (replyCode != null)
return replyCode;
return new ReplyCode(value, "Unknown");
}
}

23
src/main/java/com/ossez/reoc/rets/client/ReplyCodeHandler.java Normal file
View File

@ -0,0 +1,23 @@
package com.ossez.reoc.rets.client;
/**
* @author jrayburn
*/
public interface ReplyCodeHandler {
/**
* ReplyCodeHandler can choose to handle reply codes
* that are non-zero reply codes in its own fashion.
*
* This is intended to be used to allow the SearchResultCollector
* to choose to throw InvalidReplyCodeException if the response is
* 20201 (Empty) or 20208 (MaxRowsExceeded).
*
* @param replyCode The RETS reply code
*
* @throws InvalidReplyCodeException Thrown if reply code is
* invalid for the SearchResultCollector.
*/
public void handleReplyCode(int replyCode) throws InvalidReplyCodeException;
}

21
src/main/java/com/ossez/reoc/rets/client/RetsException.java Normal file
View File

@ -0,0 +1,21 @@
package com.ossez.reoc.rets.client;
import org.apache.commons.lang.exception.NestableException;
public class RetsException extends NestableException {
public RetsException() {
super();
}
public RetsException(String message) {
super(message);
}
public RetsException(String message, Throwable cause) {
super(message, cause);
}
public RetsException(Throwable cause) {
super(cause);
}
}

36
src/main/java/com/ossez/reoc/rets/client/RetsHttpClient.java Normal file
View File

@ -0,0 +1,36 @@
package com.ossez.reoc.rets.client;
public abstract class RetsHttpClient {
public static final String SESSION_ID_COOKIE = "RETS-Session-ID";
public static final String LOGIN_SESSION_ID = "0";
public abstract void setUserCredentials(String userName, String password);
/**
* The protocol specific implementation happens here.
*
* @param httpMethod
* @param request
* @return
* @throws RetsException
*/
public abstract RetsHttpResponse doRequest(String httpMethod, RetsHttpRequest request) throws RetsException;
/**
*
*
* @param name header name, case should be preserved
* @param value static header value, if <tt>null</tt> then implementation should not include the header in requests
*/
/**
* Add an HTTP header that should be included by default in all requests
*
* @param name header name, case should be preserved
* @param value static header value, if <code>null</code> then implementation should not include the header in requests
*/
public abstract void addDefaultHeader(String name, String value);
}

81
src/main/java/com/ossez/reoc/rets/client/RetsHttpRequest.java Normal file
View File

@ -0,0 +1,81 @@
package com.ossez.reoc.rets.client;
import java.io.Serializable;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.SortedMap;
import java.util.TreeMap;
import com.ossez.reoc.rets.common.util.CaseInsensitiveTreeMap;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang.builder.ToStringBuilder;
/** Base Http Request object */
public abstract class RetsHttpRequest implements Serializable {
private final Map<String,String> mHeaders;
private final SortedMap<String,String> mQueryParameters;
protected String mUrl;
public RetsHttpRequest() {
this.mHeaders = new CaseInsensitiveTreeMap<String,String>();
this.mQueryParameters = new TreeMap<String,String>();
}
public abstract void setUrl(CapabilityUrls urls);
public void setUrl(String url) {
this.mUrl = url;
}
public String getUrl() {
return this.mUrl;
}
public void setHeader(String key, String value) {
this.mHeaders.put(key, value);
}
public Map<String,String> getHeaders() {
return this.mHeaders;
}
public String getHttpParameters() {
if (this.mQueryParameters.isEmpty())
return null;
List<String> params = new LinkedList<String>();
for(Map.Entry<String,String> param : this.mQueryParameters.entrySet()){
params.add(String.format("%s=%s",RetsUtil.urlEncode(param.getKey()),RetsUtil.urlEncode(param.getValue())));
}
return StringUtils.join(params.iterator(),"&");
}
protected void setQueryParameter(String name, String value) {
if (value == null) {
this.mQueryParameters.remove(name);
} else {
this.mQueryParameters.put(name, value);
}
}
@Override
public String toString() {
ToStringBuilder builder = new ToStringBuilder(this);
Iterator iterator = this.mQueryParameters.keySet().iterator();
while (iterator.hasNext()) {
String s = (String) iterator.next();
builder.append(s, this.mQueryParameters.get(s));
}
return builder.toString();
}
/**
* any request with version-specific handling should deal with this.
*
* @param version
*/
public abstract void setVersion(RetsVersion version);
}

27
src/main/java/com/ossez/reoc/rets/client/RetsHttpResponse.java Normal file
View File

@ -0,0 +1,27 @@
package com.ossez.reoc.rets.client;
import java.util.Map;
import java.io.InputStream;
/**
* Interface for retrieving useful header fields from a RETS HTTP response
*
*
*/
public interface RetsHttpResponse {
public int getResponseCode() throws RetsException;
public Map getHeaders() throws RetsException;
public String getHeader(String hdr) throws RetsException;
public String getCookie(String cookie) throws RetsException;
public String getCharset() throws RetsException;
public InputStream getInputStream() throws RetsException;
public Map getCookies() throws RetsException;
}

403
src/main/java/com/ossez/reoc/rets/client/RetsSession.java Normal file
View File

@ -0,0 +1,403 @@
package com.ossez.reoc.rets.client;
import java.util.Map;
import com.ossez.reoc.rets.common.metadata.Metadata;
import com.ossez.reoc.rets.common.metadata.MetadataException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
/**
* RetsSession is the core class of the rets.client package.
*/
public class RetsSession {
public static final String METADATA_TABLES = "metadata_tables.xml";
public static final String RETS_CLIENT_VERSION = "1.5";//change default version
private static final Log LOG = LogFactory.getLog(RetsSession.class);
private static String sUserAgent = "crt-rets-client/" + RETS_CLIENT_VERSION;
private CapabilityUrls capabilityUrls;
private RetsHttpClient httpClient;
private RetsTransport transport;
private String sessionId;
/**
* Creates a new <code>RetsSession</code> instance.
* You must call login(user, pass) before attempting any other
* transactions.
*
* Uses a default implementation of RetsHttpClient based on
* apache commons http client.
*
* Uses the RetsVersion.RETS_DEFAULT as the RetsVersion for
* this session.
*
* Uses sAgent at the User-Agent setting for this RetsSession.
*
* @param loginUrl URL of the Login transaction.
*/
public RetsSession(String loginUrl) {
this(loginUrl, new CommonsHttpClient());
}
/**
* Creates a new <code>RetsSession</code> instance.
* You must call login(user, pass) before attempting any other
* transactions.
*
* Uses the RetsVersion.RETS_DEFAULT as the RetsVersion for
* this session.
*
* Uses sAgent at the User-Agent setting for this RetsSession.
*
* @param loginUrl URL of the Login transaction
* @param httpClient a RetsHttpClient implementation. The default
* is CommonsHttpClient.
*/
public RetsSession(String loginUrl, RetsHttpClient httpClient) {
this(loginUrl, httpClient, RetsVersion.DEFAULT);
}
/**
* Creates a new <code>RetsSession</code> instance.
* You must call login(user, pass) before attempting any other
* transactions.
*
* Uses sAgent at the User-Agent setting for this RetsSession.
*
* @param loginUrl URL of the Login transaction
* @param httpClient a RetsHttpClient implementation. The default
* is CommonsHttpClient.
* @param retsVersion The RetsVersion used by this RetsSession.
*/
public RetsSession(String loginUrl, RetsHttpClient httpClient, RetsVersion retsVersion) {
this(loginUrl, httpClient, retsVersion, sUserAgent,false);
}
/**
* Creates a new <code>RetsSession</code> instance.
* You must call login(user, pass) before attempting any other
* transactions.
*
* @param loginUrl URL of the Login transaction
* @param httpClient a RetsHttpClient implementation. The default
* is CommonsHttpClient.
* @param retsVersion The RetsVersion used by this RetsSession.
* @param userAgent specific User-Agent to use for this session.
*/
public RetsSession(String loginUrl, RetsHttpClient httpClient, RetsVersion retsVersion, String userAgent, boolean strict) {
this.capabilityUrls = new CapabilityUrls();
this.capabilityUrls.setLoginUrl(loginUrl);
this.httpClient = httpClient;
this.transport = new RetsTransport(httpClient, this.capabilityUrls, retsVersion, strict);
this.httpClient.addDefaultHeader("User-Agent", userAgent);
}
/**
* Query the current RetsVersion being used in this session.
*
* Initially, this will be the value passed to the RetsTransport.
* However, if during auto-negotiation the RetsTransport changes
* the RetsSession, this value may change throughout the session.
*
* @return the current RetsVersion value being used by the
* RetsTransport.
*/
public RetsVersion getRetsVersion() {
return this.transport.getRetsVersion();
}
/**
* Get the current RETS Session ID
*
* @return the current RETS Session ID or null is the server has
* not specified one
*/
public String getSessionId() {
return this.sessionId;
}
public void setSessionId(String sessionId) {
LOG.debug("setting Session-ID to: " + sessionId);
this.sessionId = sessionId;
}
public void setMonitor(NetworkEventMonitor monitor) {
this.transport.setMonitor(monitor);
}
public void setStrict(boolean strict) {
this.transport.setStrict(strict);
}
public boolean isStrict() {
return this.transport.isStrict();
}
/**
* Sets the default User-Agent value for RetsSessions created without
* a specified User-Agent value.
*
* @param userAgent Default User-Agent value to use for all RetsSession
* objects created in the future.
*/
public static void setUserAgent(String userAgent) {
sUserAgent = userAgent;
}
public String getLoginUrl() {
return this.capabilityUrls.getLoginUrl();
}
public Metadata getIncrementalMetadata() throws RetsException {
try {
return new Metadata(new MetaCollectorImpl(this.transport));
} catch (MetadataException e) {
throw new RetsException(e);
}
}
/**
* Get the complete RETS metadata.
*
* @return The RETS metadata object for these credentials.
*
* @throws RetsException
*/
public Metadata getMetadata() throws RetsException {
return this.transport.getMetadata("null");
}
/**
* Ability to download the raw metadata to a location
* @param location
* @return
* @throws RetsException
*/
public Metadata getMetadata(String location) throws RetsException {
return this.transport.getMetadata(location);
}
/**
* Perform a low level GetMetadatRequest. To retrieve
* structured metadata,
*
* @see #getMetadata()
*
* @param req GetMetadataRequest
* @return GetMetadataResponse, containing all MetaObjects
* returned
*
* @throws RetsException if an error occurs
*/
public GetMetadataResponse getMetadata(GetMetadataRequest req) throws RetsException {
return this.transport.getMetadata(req);
}
/**
* Fetches the action (MOTD) from the server.
*
* @exception RetsException if an error occurs
*/
private void getAction() throws RetsException {
String actionUrl = this.capabilityUrls.getActionUrl();
if (actionUrl == null) {
LOG.warn("No Action-URL available, skipping");
return;
}
GenericHttpRequest actionRequest = new GenericHttpRequest(actionUrl){
@Override
public Map<String, String> getHeaders() {
return null;
}
};
RetsHttpResponse httpResponse = this.httpClient.doRequest("GET", actionRequest);
try {
httpResponse.getInputStream().close();
} catch (Exception e) {
LOG.error("Action URL weirdness", e);
}
}
/**
* Implementation that allow for single or multi-part
* GetObject requests.
*
* @param req
* @return
* @exception RetsException if an error occurs
*/
public GetObjectResponse getObject(GetObjectRequest req) throws RetsException {
return this.transport.getObject(req);
}
/**
*
* @param resource
* @param type
* @param entity
* @param id
* @return response
* @exception RetsException if an error occurs
*/
public GetObjectResponse getObject(String resource, String type, String entity, String id) throws RetsException {
GetObjectRequest req = new GetObjectRequest(resource, type);
req.addObject(entity, id);
return getObject(req);
}
/**
* Log into the RETS server (see RETS 1.5, section 4). No other
* transactions will work until you have logged in.
*
* @param userName Username to authenticate
* @param password Password to authenticate with
* @return LoginResponse if success.
* @exception RetsException if authentication was denied
*/
public LoginResponse login(String userName, String password) throws RetsException {
return login(userName, password, null, null);
}
/**
* Log into the RETS server (see RETS 1.5, section 4). No other
* transactions will work until you have logged in.
*
* @param userName username to authenticate
* @param password password to authenticate with
* @param brokerCode broker code if the same user belongs to multiple
* brokerages. May be null.
* @param brokerBranch branch code if the same user belongs to multiple
* branches. May be null. brokerCode is required if you want
* brokerBranch to work.
* @return LoginResponse if success.
* @exception RetsException if authentication was denied
*/
public LoginResponse login(String userName, String password, String brokerCode, String brokerBranch) throws RetsException {
this.httpClient.setUserCredentials(userName, password);
LoginRequest request = new LoginRequest();
request.setBrokerCode(brokerCode, brokerBranch);
LoginResponse response = this.transport.login(request);
this.capabilityUrls = response.getCapabilityUrls();
this.transport.setCapabilities(this.capabilityUrls);
this.setSessionId(response.getSessionId());
this.getAction();
return response;
}
/**
* Log out of the current session. Another login _may_ re-establish a new connection
* depending the the behavior of the {#link RetsHttpClient} and its' ability to
* maintain and restablish a connection.
*
* @return a LogoutResponse
* @throws RetsException if the logout transaction failed
*/
public LogoutResponse logout() throws RetsException {
try {
return this.transport.logout();
} finally {
this.setSessionId(null);
}
}
/**
* Will perform a search as requested and return a filled
* SearchResult object. This method caches all result information
* in memory in the SearchResult object.
*
* @param req Contains parameters on which to search.
* @return a completed SearchResult
* @exception RetsException if an error occurs
*/
public SearchResult search(SearchRequest req) throws RetsException {
SearchResultImpl res = new SearchResultImpl();
search(req, res);
return res;
}
/**
* Execute a RETS Search. The collector object will be filled
* when this method is returned. See RETS 1.52d, Section 5.
*
* @param req Contains parameters on which to search.
* @param collector SearchResult object which will be informed of the results
* as they come in. If you don't need live results, see the other
* search invocation.
* @exception RetsException if an error occurs
*/
public void search(SearchRequest req, SearchResultCollector collector) throws RetsException {
this.transport.search(req, collector);
}
/**
* Search and process the Search using a given SearchResultProcessor.
*
* @param req the search request
* @param processor the result object that will process the data
*/
public SearchResultSet search(SearchRequest req, SearchResultProcessor processor) throws RetsException {
return this.transport.search(req, processor);
}
/**
* The lowest level integration. This method is not recommened for general use.
*/
public RetsHttpResponse request(RetsHttpRequest request) throws RetsException{
return this.transport.doRequest(request);
}
/**
* switch to a specific HttpMethodName, POST/GET, where the
* method is supported. Where GET is not supported, POST
* will be used.
* @param method the HttpMethodName to use
*/
public void setMethod(String method) {
this.transport.setMethod(method);
}
/** Make sure GC'd sessions are logged out. */
@Override
protected void finalize() throws Throwable {
try {
if( this.sessionId != null ) this.logout();
} finally {
super.finalize();
}
}
/**
* Performs a search returning only the number of records resulting from a query.
*
* Convenience method to get number records from a query
*
* @param req the search request
* @return the number of records that returned from the search request
* @throws RetsException
*/
public int getQueryCount(SearchRequest req) throws RetsException {
req.setCountOnly();
SearchResult res = this.search(req);
return res.getCount();
}
/**
* Gives the URL's of an Object request instead of object themselves
*
* Convenience method to get the URL's of the requeseted object only
*
* @param req
* @return
* @throws RetsException
*/
public GetObjectResponse getObjectUrl(GetObjectRequest req) throws RetsException {
req.setLocationOnly(true);
GetObjectResponse res = this.getObject(req);
return res;
}
}

331
src/main/java/com/ossez/reoc/rets/client/RetsTransport.java Normal file
View File

@ -0,0 +1,331 @@
package com.ossez.reoc.rets.client;
import java.io.FileWriter;
import java.util.HashMap;
import java.util.Map;
import com.ossez.reoc.rets.common.metadata.JDomCompactBuilder;
import com.ossez.reoc.rets.common.metadata.JDomStandardBuilder;
import com.ossez.reoc.rets.common.metadata.Metadata;
import com.ossez.reoc.rets.common.metadata.MetadataBuilder;
import org.jdom.Document;
import org.jdom.input.SAXBuilder;
import org.jdom.output.XMLOutputter;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
/**
* Implements the basic transport mechanism. This class deals with the
* very basic parts of sending the request, returning a response object,
* and version negotiation.
*
*/
public class RetsTransport {
private static final String RETS_SESSION_ID_HEADER = "RETS-Session-ID"; // TODO spec says hyphen, Marketlinx uses an underscore
private RetsHttpClient client;
private CapabilityUrls capabilities;
private String method = "GET";
private RetsVersion version;
private boolean strict;
private NetworkEventMonitor monitor;
private static final Log LOG = LogFactory.getLog(RetsTransport.class);
private static Map MONITOR_MSGS = new HashMap(){{
put(ChangePasswordRequest.class, "Transmitting change password request");
put(GetObjectRequest.class, "Retrieving media object");
put(LoginRequest.class, "Logging in");
put(GetMetadataRequest.class, "Retrieving metadata");
put(LogoutRequest.class, "Logging out");
put(SearchRequest.class, "Executing search");
}};
/**
* Create a new transport instance.
* @param client An http client (make sure you call setUserCredentials
* on it before carrying out any transactions).
* @param capabilities the initial capabilities url list. This can be
* replaced with a more up to date version at any time (for example,
* post-login()) with setCapabilities()
*
* @see RetsHttpClient#setUserCredentials
*/
public RetsTransport(RetsHttpClient client, CapabilityUrls capabilities) {
this(client, capabilities, RetsVersion.DEFAULT, false);
}
/**
* Create a new transport instance to speak a specific RETS version.
* @param client an http client
* @param capabilities the initial capabilities url list
* @param version the RETS version to use during initial negotiation
* (RetsTransport will automatically switch to whatever version the
* server supports).
*/
public RetsTransport(RetsHttpClient client, CapabilityUrls capabilities, RetsVersion version, boolean strict) {
this.client = client;
this.capabilities = capabilities;
this.doVersionHeader(version);
this.strict = strict;
this.client.addDefaultHeader("Accept", "*/*");
this.monitor = new NullNetworkEventMonitor();
}
/**
* Query the current RetsVersion being used in this RetsTransport.
*
* Initially, this will be the value with which this object was
* constructed.
*
* However, this value may change after login.
*
* @return the current RetsVersion value being used by the
* RetsTransport.
*/
public RetsVersion getRetsVersion() {
return this.version;
}
public boolean isStrict() {
return this.strict;
}
public void setStrict(boolean strict) {
this.strict = strict;
}
public void setMonitor(NetworkEventMonitor monitor) {
if (monitor == null) {
monitor = new NullNetworkEventMonitor();
}
this.monitor = monitor;
}
/**
* Set our RetsHttpClient up with the correct default RETS version to use,
* default to RETS 1.5.
* @param retsVersion
*/
private void doVersionHeader(RetsVersion retsVersion) {
if (this.client == null)
return;
if (retsVersion == null)
retsVersion = RetsVersion.DEFAULT;
this.version = retsVersion;
this.client.addDefaultHeader(RetsVersion.RETS_VERSION_HEADER, this.version.toString());
}
/**
* replace the capabilities url list with a new one
* @param capabilities the new capabilities url list
*/
public void setCapabilities(CapabilityUrls capabilities) {
this.capabilities = capabilities;
}
/**
* switch to a specific HttpMethodName, POST/GET, where the
* method is supported. Where GET is not supported, POST
* will be used.
* @param method the HttpMethodName to use
*/
public void setMethod(String method) {
this.method = method;
}
/**
* Available as an integration last resort
*/
public RetsHttpResponse doRequest(RetsHttpRequest req) throws RetsException {
Object monitorobj = null;
String msg = getMonitorMessage(req);
monitorobj = this.monitor.eventStart(msg);
req.setVersion(this.version);
req.setUrl(this.capabilities);
RetsHttpResponse httpResponse;
try {
httpResponse = this.client.doRequest(this.method, req);
} finally {
this.monitor.eventFinish(monitorobj);
}
return httpResponse;
}
private String getMonitorMessage(RetsHttpRequest req) {
String msg = (String) MONITOR_MSGS.get(req.getClass());
if (msg == null) {
msg = "communicating with network";
}
return msg;
}
/**
* Logs into the server. This transaction gets a list of capability URLs
* encapsulated in the LoginResponse that should typically be given back
* to the transport object with setCapabilities(). RETS Specification,
* section 4.
*
* @param req The login request
* @return the LoginResponse object
* @throws RetsException if the login failed or something went wrong on the
* network
* @see #setCapabilities
*/
public LoginResponse login(LoginRequest req) throws RetsException {
RetsHttpResponse retsHttpResponse = this.doRequest(req);
String versionHeader = retsHttpResponse.getHeader(RetsVersion.RETS_VERSION_HEADER);
// may be null, which is fine, return null, dont throw
RetsVersion retsVersion = RetsVersion.getVersion(versionHeader);
if( retsVersion == null && this.strict )
throw new RetsException(String.format("RETS Version is a required response header, version '%s' is unrecognized",versionHeader));
// skip updating the client version if its not set (correctly) by the server
if( retsVersion != null ) this.doVersionHeader(retsVersion);
LoginResponse response = new LoginResponse(this.capabilities.getLoginUrl());
String sessionId = retsHttpResponse.getCookie(RETS_SESSION_ID_HEADER);
response.setSessionId(sessionId);
response.setStrict(this.strict);
response.parse(retsHttpResponse.getInputStream(), this.version);
return response;
}
/**
* Logs out of the server. No other transactions should be called until
* another login() succeeds. RETS Specification, Section 6. Logout is
* an optional transaction. This method returns null if the server does
* not support the Logout transaction.
*
* @return LogoutResponse or null if logout is not supported
* @throws RetsException if there is a network or remote server error
*/
public LogoutResponse logout() throws RetsException {
if (this.capabilities.getLogoutUrl() == null) {
return null;
}
RetsHttpRequest req = new LogoutRequest();
RetsHttpResponse httpResponse = doRequest(req);
LogoutResponse response = new LogoutResponse();
response.setStrict(this.strict);
try {
response.parse(httpResponse.getInputStream(), this.version);
} catch(RetsException e) {
if (e.getMessage().contains("Invalid number of children")){// most RETS servers have issues logging out for some reason.
LOG.warn("unsual response for logout request, but log out successful.");
}
}
return response;
}
/**
* Perform a non-streaming search and pass all results from the
* SearchRequest to the given collector.
*
* 12/06/20 Added charset, needed for sax parser
* @param req the search request
* @param collector the result object that will store the data
*/
public void search(SearchRequest req, SearchResultCollector collector) throws RetsException {
RetsHttpResponse httpResponse = doRequest(req);
new SearchResultHandler(collector).parse(httpResponse.getInputStream(), httpResponse.getCharset());
}
/**
* Override processing of the search completely by providing a
* SearchResultProcessor to process the results of the Search.
*
* @param req the search request
* @param processor the result object that will process the data
*/
public SearchResultSet search(SearchRequest req, SearchResultProcessor processor) throws RetsException {
RetsHttpResponse httpResponse = doRequest(req);
return processor.parse(httpResponse.getInputStream());
}
/**
*
* @param req GetObject request
* @return a GetObjectResponse
* @throws RetsException if the request is not valid or a network error
* occurs
*/
public GetObjectResponse getObject(GetObjectRequest req) throws RetsException {
if (this.capabilities.getGetObjectUrl() == null) {
throw new RetsException("Server does not support GetObject transaction.");
}
req.setUrl(this.capabilities);
RetsHttpResponse httpResponse = this.client.doRequest(this.method, req);
GetObjectResponse result = new GetObjectResponse(httpResponse.getHeaders(), httpResponse.getInputStream());
return result;
}
public Metadata getMetadata(String location) throws RetsException {
boolean compact = Boolean.getBoolean("rets-client.metadata.compact");
GetMetadataRequest req = new GetMetadataRequest("SYSTEM", "*");
if (compact) {
req.setCompactFormat();
}
try {
RetsHttpResponse httpResponse = doRequest(req);
Object monitorobj = null;
monitorobj = this.monitor.eventStart("Parsing metadata");
try {
SAXBuilder xmlBuilder = new SAXBuilder();
Document xmlDocument = xmlBuilder.build(httpResponse.getInputStream());
if (!location.equals("null")){
XMLOutputter outputter = new XMLOutputter();
FileWriter writer = new FileWriter(location);
outputter.output(xmlDocument, writer);
outputter.outputString(xmlDocument);
}
MetadataBuilder metadataBuilder;
if (req.isCompactFormat()) {
metadataBuilder = new JDomCompactBuilder();
} else {
metadataBuilder = new JDomStandardBuilder();
}
metadataBuilder.setStrict(this.strict);
return metadataBuilder.doBuild(xmlDocument);
} finally {
this.monitor.eventFinish(monitorobj);
}
} catch (Exception e) {
throw new RetsException(e);
}
}
public GetMetadataResponse getMetadata(GetMetadataRequest req) throws RetsException {
RetsHttpResponse httpResponse = doRequest(req);
Object monitorobj = null;
monitorobj = this.monitor.eventStart("Parsing metadata");
try {
try {
return new GetMetadataResponse(httpResponse.getInputStream(), req.isCompactFormat(),this.strict);
} catch (InvalidReplyCodeException e) {
e.setRequestInfo(req.toString());
throw e;
}
} finally {
this.monitor.eventFinish(monitorobj);
}
}
public boolean changePassword(ChangePasswordRequest req) throws RetsException {
RetsHttpResponse httpResponse = doRequest(req);
ChangePasswordResponse response = new ChangePasswordResponse(httpResponse.getInputStream());
// response will throw an exception if there is an error code
return (response != null);
}
}

45
src/main/java/com/ossez/reoc/rets/client/RetsUtil.java Normal file
View File

@ -0,0 +1,45 @@
package com.ossez.reoc.rets.client;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import org.apache.commons.codec.DecoderException;
import org.apache.commons.codec.EncoderException;
import org.apache.commons.codec.net.URLCodec;
import org.apache.commons.lang.exception.NestableRuntimeException;
/** Random utility functions. */
public class RetsUtil {
public static void copyStream(InputStream in, OutputStream out) throws IOException {
byte[] buf = new byte[512];
int count;
while (true) {
count = in.read(buf);
if (count < 1) {
in.close();
out.close();
return;
}
while (count > 0) {
out.write(buf);
}
}
}
public static String urlEncode(String string) {
try {
return new URLCodec().encode(string);
} catch (EncoderException e) {
throw new NestableRuntimeException(e);
}
}
public static String urlDecode(String string) {
try {
return new URLCodec().decode(string);
} catch (DecoderException e) {
throw new NestableRuntimeException(e);
}
}
}

96
src/main/java/com/ossez/reoc/rets/client/RetsVersion.java Normal file
View File

@ -0,0 +1,96 @@
package com.ossez.reoc.rets.client;
import java.io.Serializable;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang.math.NumberUtils;
public class RetsVersion implements Serializable {
public static final String RETS_VERSION_HEADER = "RETS-Version";
public static final RetsVersion RETS_10 = new RetsVersion(1, 0, 0, 0);
public static final RetsVersion RETS_15 = new RetsVersion(1, 5, 0, 0);
public static final RetsVersion RETS_16 = new RetsVersion(1, 6, 0, 0);
public static final RetsVersion RETS_17 = new RetsVersion(1, 7, 0, 0);
public static final RetsVersion RETS_1_7_2 = new RetsVersion(1, 7, 2, 0);
public static final RetsVersion DEFAULT = RETS_1_7_2;
private int mMajor;
private int mMinor;
private int mRevision;
private int mDraft;
public RetsVersion(int major, int minor) {
this(major,minor,0,0);
}
/** @deprecated use <code>new RetsVersion(major, minor, 0, draft)</code> */
@Deprecated
public RetsVersion(int major, int minor, int draft) {
this(major,minor,0,draft);
}
public RetsVersion(int major, int minor, int revision, int draft) {
this.mMajor = major;
this.mMinor = minor;
this.mRevision = revision;
this.mDraft = draft;
}
public int getMajor() {
return this.mMajor;
}
public int getMinor() {
return this.mMinor;
}
public int getRevision() {
return this.mRevision;
}
public int getDraft() {
return this.mDraft;
}
@Override
public String toString() {
if (this.mRevision == 0) {
if (this.mDraft == 0) {
return "RETS/" + this.mMajor + "." + this.mMinor;
}
return "RETS/" + this.mMajor + "." + this.mMinor + "d" + this.mDraft;
}
if (this.mDraft == 0) {
return "RETS/" + this.mMajor + "." + this.mMinor + "." + this.mRevision;
}
return "RETS/" + this.mMajor + "." + this.mMinor + "." + this.mRevision + "d" + this.mDraft;
}
@Override
public boolean equals(Object o) {
if (o instanceof RetsVersion) {
RetsVersion v = (RetsVersion) o;
if ((v.getMajor() == this.mMajor) && (v.getMinor() == this.mMinor) && (v.getRevision() == this.mRevision) && (v.getDraft() == this.mDraft)) {
return true;
}
}
return false;
}
public static RetsVersion getVersion(String ver) {
if( StringUtils.isEmpty(ver) ) return null;
String[] split = StringUtils.trimToEmpty(ver).split("\\.");
int ma = NumberUtils.toInt(split[0],1);
int mn = split.length > 1 ? NumberUtils.toInt(split[1],0) : 0;
int re = 0;
int dr = 0;
if (split.length > 2) {
split = StringUtils.defaultString(split[2]).split("d");
re = NumberUtils.toInt(split[0],0);
dr = split.length > 1 ? NumberUtils.toInt(split[1],0) : 0;
}
return new RetsVersion(ma,mn,re,dr);
}
}

122
src/main/java/com/ossez/reoc/rets/client/SearchRequest.java Normal file
View File

@ -0,0 +1,122 @@
package com.ossez.reoc.rets.client;
/**
*
* The search request sent from search() in RetsSession
*
*/
public class SearchRequest extends RetsHttpRequest {
public static final int COUNT_NONE = 1;
public static final int COUNT_FIRST = 2;
public static final int COUNT_ONLY = 3;
public static final String FORMAT_STANDARD_XML = "STANDARD-XML";
public static final String FORMAT_COMPACT = "COMPACT";
public static final String FORMAT_COMPACT_DECODED = "COMPACT-DECODED";
public static final String RETS_DMQL1 = "DMQL";
public static final String RETS_DMQL2 = "DMQL2";
public static final String KEY_TYPE = "SearchType";
public static final String KEY_CLASS = "Class";
public static final String KEY_DMQLVERSION = "QueryType";
public static final String KEY_QUERY = "Query";
public static final String KEY_COUNT = "Count";
public static final String KEY_FORMAT = "Format";
public static final String KEY_LIMIT = "Limit";
public static final String KEY_OFFSET = "Offset";
public static final String KEY_SELECT = "Select";
public static final String KEY_RESTRICTEDINDICATOR = "RestrictedIndicator";
public static final String KEY_STANDARDNAMES = "StandardNames";
private String type;
public SearchRequest(String stype, String sclass, String query) {
setQueryParameter(KEY_TYPE, stype);
this.type = stype;
setQueryParameter(KEY_CLASS, sclass);
setQueryParameter(KEY_QUERY, query);
setQueryParameter(KEY_FORMAT, FORMAT_COMPACT);
setQueryParameter(KEY_DMQLVERSION, RETS_DMQL2);
}
@Override
public void setUrl(CapabilityUrls urls) {
setUrl(urls.getSearchUrl());
}
public String getType() {
return this.type;
}
public void setCountNone() {
setQueryParameter(KEY_COUNT, null);
}
public void setCountFirst() {
setQueryParameter(KEY_COUNT, "1");
}
public void setCountOnly() {
setQueryParameter(KEY_COUNT, "2");
}
public void setFormatCompact() {
setQueryParameter(KEY_FORMAT, FORMAT_COMPACT);
}
public void setFormatCompactDecoded() {
setQueryParameter(KEY_FORMAT, FORMAT_COMPACT_DECODED);
}
public void setFormatStandardXml() {
setQueryParameter(KEY_FORMAT, FORMAT_STANDARD_XML);
}
public void setFormatStandardXml(String dtdVersion) {
setQueryParameter(KEY_FORMAT, FORMAT_STANDARD_XML + ":" + dtdVersion);
}
public void setLimit(int count) {
setQueryParameter(KEY_LIMIT, Integer.toString(count));
}
public void setLimitNone() {
setQueryParameter(KEY_LIMIT, null);
}
public void setSelect(String sel) {
setQueryParameter(KEY_SELECT, sel);
}
public void setRestrictedIndicator(String rest) {
setQueryParameter(KEY_RESTRICTEDINDICATOR, rest);
}
public void setStandardNames() {
setQueryParameter(KEY_STANDARDNAMES, "1");
}
public void setSystemNames() {
setQueryParameter(KEY_STANDARDNAMES, null);
}
public void setOffset(int offset) {
setQueryParameter(KEY_OFFSET, Integer.toString(offset));
}
public void setOffsetNone() {
setQueryParameter(KEY_OFFSET, null);
}
/** TODO should the search automatically handle this??? shouldn't this be setable by vendor is that predicatable? */
@Override
public void setVersion(RetsVersion ver) {
if (RetsVersion.RETS_10.equals(ver)) {
setQueryParameter(KEY_DMQLVERSION, RETS_DMQL1);
} else {
setQueryParameter(KEY_DMQLVERSION, RETS_DMQL2);
}
}
}

23
src/main/java/com/ossez/reoc/rets/client/SearchResult.java Normal file
View File

@ -0,0 +1,23 @@
package com.ossez.reoc.rets.client;
import java.util.NoSuchElementException;
import java.util.Iterator;
/**
* Interface for retrieving additional information from of a result from a RETS query/search
*
*/
public interface SearchResult extends SearchResultInfo {
public String[] getRow(int idx) throws NoSuchElementException;
public Iterator iterator();
public String[] getColumns();
public boolean isMaxrows();
public int getCount();
public boolean isComplete();
}

18
src/main/java/com/ossez/reoc/rets/client/SearchResultCollector.java Normal file
View File

@ -0,0 +1,18 @@
package com.ossez.reoc.rets.client;
/**
* Interface for a setting properties of a result from a query (used by SearchResultHandler)
*/
public interface SearchResultCollector {
public void setCount(int count);
public void setColumns(String[] columns);
public boolean addRow(String[] row);
public void setMaxrows();
public void setComplete();
}

280
src/main/java/com/ossez/reoc/rets/client/SearchResultHandler.java Normal file
View File

@ -0,0 +1,280 @@
package com.ossez.reoc.rets.client;
import java.io.IOException;
import java.io.InputStream;
import java.util.LinkedList;
import java.util.List;
import java.util.StringTokenizer;
import javax.xml.parsers.SAXParser;
import javax.xml.parsers.SAXParserFactory;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.xml.sax.Attributes;
import org.xml.sax.ContentHandler;
import org.xml.sax.ErrorHandler;
import org.xml.sax.InputSource;
import org.xml.sax.Locator;
import org.xml.sax.SAXException;
import org.xml.sax.SAXParseException;
import org.xml.sax.XMLReader;
/**
*
* Handles XML parsing from response setting the proper fields using a SearchResultCollector
*
*/
public class SearchResultHandler implements ContentHandler, ErrorHandler{
private static final Log LOG = LogFactory.getLog(SearchResultHandler.class);
private static SAXParserFactory FACTORY = SAXParserFactory.newInstance();
private int dataCount;
private SearchResultCollector collector;
private StringBuffer currentEntry;
private String delimiter;
private Locator locator;
private String[] columns;
private InvalidReplyCodeHandler invalidReplyCodeHandler;
private CompactRowPolicy compactRowPolicy;
public SearchResultHandler(SearchResultCollector r) {
this(r, InvalidReplyCodeHandler.FAIL, CompactRowPolicy.DEFAULT);
}
public SearchResultHandler(SearchResultCollector r, InvalidReplyCodeHandler invalidReplyCodeHandler, CompactRowPolicy badRowPolicy) {
this.compactRowPolicy = badRowPolicy;
if (r == null)
throw new NullPointerException("SearchResultCollector must not be null");
if (invalidReplyCodeHandler == null)
throw new NullPointerException("InvalidReplyCodeHandler must not be null");
if (badRowPolicy == null)
throw new NullPointerException("BadRowPolicy must not be null");
this.collector = r;
this.dataCount = 0;
this.invalidReplyCodeHandler = invalidReplyCodeHandler;
}
public void startElement(String uri, String localName, String qName, Attributes atts) throws SAXException {
String name = localName;
if (localName.equals("")) {
name = qName;
}
if (name.equals("RETS") || name.equals("RETS-STATUS")) {
String rawrepcode = atts.getValue("ReplyCode");
try {
int repcode = Integer.parseInt(rawrepcode);
if (repcode > 0) {
try {
if (ReplyCode.MAXIMUM_RECORDS_EXCEEDED.equals(repcode))
return;
if (ReplyCode.NO_RECORDS_FOUND.equals(repcode))
return;
if (name.equals("RETS"))
this.invalidReplyCodeHandler.invalidRetsReplyCode(repcode);
else
this.invalidReplyCodeHandler.invalidRetsStatusReplyCode(repcode);
} catch (InvalidReplyCodeException e) {
String text = atts.getValue("", "ReplyText");
e.setRemoteMessage(text);
throw new SAXException(e);
}
}
} catch (NumberFormatException e) {
throw new SAXParseException("Invalid ReplyCode '" + rawrepcode + "'", this.locator);
}
return;
}
if (name == "COUNT") {
String s = atts.getValue("Records");
if (s == null) {
s = atts.getValue("", "Records");
if (s == null) {
throw new SAXParseException("COUNT tag has no Records " + "attribute", this.locator);
}
}
int i = Integer.parseInt(s, 10);
this.collector.setCount(i);
return;
}
if (name == "DELIMITER") {
String s = atts.getValue("value");
if (s == null) {
s = atts.getValue("", "value");
if (s == null) {
throw new RuntimeException("Invalid Delimiter");
}
}
int i = Integer.parseInt(s, 16);
this.delimiter = "" + (char) i;
return;
}
if (name == "COLUMNS" || name == "DATA") {
this.currentEntry = new StringBuffer();
return;
}
if (name == "MAXROWS") {
this.collector.setMaxrows();
return;
}
// Unknown tag. danger, will.
LOG.warn("Unknown tag: " + name + ", qName = " + qName);
}
public void characters(char[] ch, int start, int length) {
if (this.currentEntry != null) {
this.currentEntry.append(ch, start, length);
}
}
public void ignorableWhitespace(char[] ch, int start, int length) {
// we ignore NOZINK!
characters(ch, start, length);
}
/** do NOT use string.split() unless your prepared to deal with loss due to token boundary conditions */
private String[] split(String input) throws SAXParseException {
if (this.delimiter == null) {
throw new SAXParseException("Invalid compact format - DELIMITER not specified", this.locator);
}
if( !input.startsWith(this.delimiter) ){
throw new SAXParseException("Invalid compact format", this.locator);
}
StringTokenizer tkn = new StringTokenizer(input, this.delimiter, true);
List list = new LinkedList();
tkn.nextToken(); // junk the first element
String last = null;
while (tkn.hasMoreTokens()) {
String next = tkn.nextToken();
if (next.equals(this.delimiter)) {
if (last == null) {
list.add("");
} else {
last = null;
}
} else {
list.add(next);
last = next;
}
}
return (String[]) list.toArray(new String[0]);
}
public void endElement(String uri, String localName, String qName) throws SAXParseException {
String name = localName;
if (name.equals("")) {
name = qName;
}
if (name.equals("COLUMNS") || name.equals("DATA")) {
String[] contents = split(this.currentEntry.toString());
if (name.equals("COLUMNS")) {
this.collector.setColumns(contents);
this.columns = contents;
} else {
if( this.compactRowPolicy.apply(this.dataCount, this.columns, contents) ) {
this.dataCount++;
this.collector.addRow(contents);
}
}
this.currentEntry = null;
}
}
public void startDocument() {
LOG.info("Start document");
}
public void endDocument() {
LOG.info("Document ended");
this.collector.setComplete();
}
public void startPrefixMapping(String prefix, String uri) throws SAXException {
// LOG.debug("prefix mapping: " + prefix);
}
public void endPrefixMapping(String prefix) throws SAXException {
// LOG.debug("prefix mapping: " + prefix);
}
public void processingInstruction(String target, String data) throws SAXException {
throw new SAXException("processing instructions not supported: " + "target=" + target + ", data=" + data);
}
public void skippedEntity(String name) throws SAXException {
throw new SAXException("skipped entities not supported: name=" + name);
}
public void setDocumentLocator(Locator locator) {
this.locator = locator;
}
public void error(SAXParseException e) throws SAXException {
throw e;
}
public void fatalError(SAXParseException e) throws SAXException {
throw e;
}
public void warning(SAXParseException e) {
LOG.warn("an error occured while parsing. Attempting to continue", e);
}
public void parse(InputSource src) throws RetsException {
parse(src, null);
}
/**
*
* created in order to pass the charset to the parser for proper encoding
* @param str
* @param charset
* @throws RetsException
*/
public void parse(InputStream str, String charset) throws RetsException {
parse(new InputSource(str), charset);
try {
str.close();
} catch (IOException e) {
throw new RetsException(e);
}
}
/**
* Pareses given source with the given charset
*
* @param src
* @throws RetsException
*/
public void parse(InputSource src, String charset) throws RetsException {
String encoding = src.getEncoding();
if (encoding == null && (charset != null)){
encoding = charset;
LOG.warn("Charset from headers:" + charset + ". Setting as correct encoding for parsing");
src.setEncoding(encoding);
}
try {
SAXParser p = FACTORY.newSAXParser();
XMLReader r = p.getXMLReader();
r.setContentHandler(this);
r.setErrorHandler(this);
r.parse(src);
} catch (SAXException se) {
if (se.getException() != null && se.getException() instanceof RetsException) {
throw (RetsException) se.getException();
}
throw new RetsException(se);
} catch (Exception e) {
LOG.error("An exception occured", e);
throw new RetsException(e);
}
}
}

87
src/main/java/com/ossez/reoc/rets/client/SearchResultImpl.java Normal file
View File

@ -0,0 +1,87 @@
package com.ossez.reoc.rets.client;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.NoSuchElementException;
import org.apache.commons.logging.LogFactory;
/**
* Concrete Implementation of SearchResult interface
*
*/
public class SearchResultImpl implements SearchResult, SearchResultCollector {
private String[] columnNames;
private int count;
private List<String[]> rows;
private boolean maxRows;
private boolean complete;
public SearchResultImpl() {
this.count = 0;
this.rows = new ArrayList<String[]>();
this.maxRows = false;
this.complete = false;
}
public void setCount(int count) {
this.count = count;
}
public int getCount() {
if (this.count > 0) {
return this.count;
}
return this.rows.size();
}
public int getRowCount() {
return this.rows.size();
}
public void setColumns(String[] columns) {
this.columnNames = columns;
}
public String[] getColumns() {
return this.columnNames;
}
public boolean addRow(String[] row) {
if (row.length > this.columnNames.length) {
throw new IllegalArgumentException(String.format("Invalid number of result columns: got %s, expected %s",row.length, this.columnNames.length));
}
if (row.length < this.columnNames.length) {
LogFactory.getLog(SearchResultCollector.class).warn(String.format("Row %s: Invalid number of result columns: got %s, expected ",this.rows.size(), row.length, this.columnNames.length));
}
return this.rows.add(row);
}
public String[] getRow(int idx) {
if (idx >= this.rows.size()) {
throw new NoSuchElementException();
}
return this.rows.get(idx);
}
public Iterator iterator() {
return this.rows.iterator();
}
public void setMaxrows() {
this.maxRows = true;
}
public boolean isMaxrows() {
return this.maxRows;
}
public void setComplete() {
this.complete = true;
}
public boolean isComplete() {
return this.complete;
}
}

25
src/main/java/com/ossez/reoc/rets/client/SearchResultInfo.java Normal file
View File

@ -0,0 +1,25 @@
package com.ossez.reoc.rets.client;
/**
* Interface that describes high level information
* about the results of a search.
* @author jrayburn
*/
public interface SearchResultInfo {
public int getCount() throws RetsException;
public String[] getColumns() throws RetsException;
/** @throws IllegalStateException */
public boolean isMaxrows() throws RetsException, IllegalStateException;
/**
* Indicates that processing of this search
* is complete.
*
* @return true if this SearchResultSet is finished processing.
* @throws RetsException Thrown if there is an error
* processing the SearchResultSet.
*/
public boolean isComplete() throws RetsException;
}

13
src/main/java/com/ossez/reoc/rets/client/SearchResultProcessor.java Normal file
View File

@ -0,0 +1,13 @@
package com.ossez.reoc.rets.client;
import java.io.InputStream;
import java.io.Reader;
/**
* Interface for parsing results from a RETS query/search
*/
public interface SearchResultProcessor {
public SearchResultSet parse(InputStream in) throws RetsException;
public SearchResultSet parse(Reader in) throws RetsException;
}

15
src/main/java/com/ossez/reoc/rets/client/SearchResultSet.java Normal file
View File

@ -0,0 +1,15 @@
package com.ossez.reoc.rets.client;
/**
* Iterator style interface for processing the results
* of a RETS search a single time. Information about the
* search can be retrieved once processing is complete by
* calling the getInfo() method.
*
* @author YuCheng Hu
*/
public interface SearchResultSet extends SearchResultInfo {
public String[] next() throws RetsException;
public boolean hasNext() throws RetsException;
}

53
src/main/java/com/ossez/reoc/rets/client/SingleObjectResponse.java Normal file
View File

@ -0,0 +1,53 @@
package com.ossez.reoc.rets.client;
import java.io.InputStream;
import java.util.Map;
import com.ossez.reoc.rets.common.util.CaseInsensitiveTreeMap;
/**
* Representation of a single object returned
* from a RETS server.
*
* @author jrayburn
*/
public class SingleObjectResponse {
public static final String CONTENT_TYPE = "Content-Type";
public static final String LOCATION = "Location";
public static final String CONTENT_DESCRIPTION = "Content-Description";
public static final String OBJECT_ID = "Object-ID";
public static final String CONTENT_ID = "Content-ID";
private Map headers;
private InputStream inputStream;
public SingleObjectResponse(Map headers, InputStream in) {
this.headers = new CaseInsensitiveTreeMap(headers);
this.inputStream = in;
}
public String getType() {
return (String) this.headers.get(CONTENT_TYPE);
}
public String getContentID() {
return (String) this.headers.get(CONTENT_ID);
}
public String getObjectID() {
return (String) this.headers.get(OBJECT_ID);
}
public String getDescription() {
return (String) this.headers.get(CONTENT_DESCRIPTION);
}
public String getLocation() {
return (String) this.headers.get(LOCATION);
}
public InputStream getInputStream() {
return this.inputStream;
}
}

64
src/main/java/com/ossez/reoc/rets/client/SinglePartInputStream.java Normal file
View File

@ -0,0 +1,64 @@
package com.ossez.reoc.rets.client;
import java.io.FilterInputStream;
import java.io.IOException;
import java.io.PushbackInputStream;
class SinglePartInputStream extends FilterInputStream {
private static final int EOS = -1;
private final String boundary;
private boolean eos;
SinglePartInputStream(PushbackInputStream partInput, String boundary) {
super(partInput);
this.boundary = boundary;
}
@Override
public int read() throws IOException {
int read = this.getPushBackStream().read();
// was this the start of a boundary?
if( read != '\r' && read != '\n' ) return read;
this.getPushBackStream().unread(read);
byte[] peek = new byte[ "\r\n".length() + this.boundary.length()];
// if so, check and see if the rest of the boundary is next
int peekRead = this.getPushBackStream().read(peek);
this.getPushBackStream().unread(peek, 0, peekRead);
if( new String(peek).contains(this.boundary) ) return EOS;
// if not, just a coincidence, just return the byte
return this.getPushBackStream().read();
}
@Override
public int read(byte[] b, int off, int len) throws IOException {
if(this.eos) return EOS;
int read = off;
for( ; read < off + len; read++) {
int nextByte = this.read();
if(nextByte == EOS) {
this.eos = true;
break;
}
b[read] = (byte) nextByte;
}
return ( read - off );
}
@Override
public int read(byte[] b) throws IOException {
return this.read(b, 0, b.length);
}
@Override
public void close() {
// noop - part of a larger stream
}
private PushbackInputStream getPushBackStream() {
return (PushbackInputStream) this.in;
}
}

325
src/main/java/com/ossez/reoc/rets/client/StreamingSearchResultProcessor.java Normal file
View File

@ -0,0 +1,325 @@
package com.ossez.reoc.rets.client;
import java.io.InputStream;
import java.io.Reader;
import java.util.LinkedList;
import org.apache.commons.lang.exception.NestableRuntimeException;
import org.apache.commons.logging.LogFactory;
import org.xml.sax.InputSource;
/**
* SearchResultProcessor that returns a streaming SearchResult implementation.
*
* @author jrayburn
*/
public class StreamingSearchResultProcessor implements SearchResultProcessor {
private final int mBufferSize;
private final int mTimeout;
private InvalidReplyCodeHandler mInvalidReplyCodeHandler;
private CompactRowPolicy mCompactRowPolicy;
/**
* Construct a StreamingSearchResultProcessor.
*
* Waits indefinitely for buffer to be read from by
* client.
*
* @param bufferSize
* How many rows to buffer
*/
public StreamingSearchResultProcessor(int bufferSize) {
this(bufferSize, 0);
}
/**
* Construct a StreamingSearchResultProcessor.
*
* Waits <code>timeout</code> milliseconds for buffer to
* be read from by client.
*
* @param bufferSize
* How many rows to buffer
*
* @param timeout
* How long to wait, in milliseconds, for the buffer
* to be read from when full. 0 indicates an indefinite
* wait.
*/
public StreamingSearchResultProcessor(int bufferSize, int timeout) {
super();
this.mBufferSize = bufferSize;
this.mTimeout = timeout;
}
/** how to deal with badly delimited data */
public void setCompactRowPolicy(CompactRowPolicy badRowPolicy) {
this.mCompactRowPolicy = badRowPolicy;
}
private CompactRowPolicy getCompactRowPolicy() {
if (this.mCompactRowPolicy == null)
return CompactRowPolicy.DEFAULT;
return this.mCompactRowPolicy;
}
public void setInvalidRelyCodeHandler(InvalidReplyCodeHandler invalidReplyCodeHandler) {
this.mInvalidReplyCodeHandler = invalidReplyCodeHandler;
}
private InvalidReplyCodeHandler getInvalidRelyCodeHandler() {
if (this.mInvalidReplyCodeHandler == null)
return InvalidReplyCodeHandler.FAIL;
return this.mInvalidReplyCodeHandler;
}
public SearchResultSet parse(InputStream reader) {
return parse(new InputSource(reader));
}
public SearchResultSet parse(Reader reader) {
return parse(new InputSource(reader));
}
public SearchResultSet parse(InputSource source) {
StreamingSearchResult result = new StreamingSearchResult(this.mBufferSize, this.mTimeout);
StreamingThread thread = new StreamingThread(source, result, this.getInvalidRelyCodeHandler(), this.getCompactRowPolicy());
thread.start();
return result;
}
}
class StreamingThread extends Thread {
private StreamingSearchResult mResult;
private InputSource mSource;
private InvalidReplyCodeHandler mInvalidReplyCodeHandler;
private CompactRowPolicy badRowPolicy;
public StreamingThread(InputSource source, StreamingSearchResult result,InvalidReplyCodeHandler invalidReplyCodeHandler, CompactRowPolicy badRowPolicy) {
this.mSource = source;
this.mResult = result;
this.mInvalidReplyCodeHandler = invalidReplyCodeHandler;
this.badRowPolicy = badRowPolicy;
}
@Override
public void run() {
SearchResultHandler handler = new SearchResultHandler(this.mResult, this.mInvalidReplyCodeHandler, this.badRowPolicy);
try {
handler.parse(this.mSource);
} catch (RetsException e) {
this.mResult.setException(e);
} catch (Exception e) {
// socket timeouts, etc while obtaining xml bytes from InputSource ...
this.mResult.setException(new RetsException("Low level exception while attempting to parse input from source.", e));
}
}
}
class StreamingSearchResult implements SearchResultSet, SearchResultCollector {
private static final int PREPROCESS = 0;
private static final int BUFFER_AVAILABLE = 1;
private static final int BUFFER_FULL = 2;
private static final int COMPLETE = 3;
private final int timeout;
private final int bufferSize;
private final LinkedList<String[]> buffer;
private boolean mMaxrows;
private int state;
private String[] columns;
private int count;
private RetsException exception;
public StreamingSearchResult(int bufferSize, int timeout) {
if (bufferSize < 1)
throw new IllegalArgumentException("[bufferSize=" + bufferSize + "] must be greater than zero");
if (timeout < 0)
throw new IllegalArgumentException("[timeout=" + timeout + "] must be greater than or equal to zero");
this.bufferSize = bufferSize;
this.timeout = timeout;
this.state = PREPROCESS;
this.buffer = new LinkedList<String[]>();
this.count = -1;
this.columns = null;
this.exception = null;
}
// ------------ Producer Methods
public synchronized boolean addRow(String[] row) {
if (row.length > this.columns.length) {
throw new IllegalArgumentException(String.format("Invalid number of result columns: got %s, expected %s",row.length, this.columns.length));
}
if (row.length < this.columns.length) {
LogFactory.getLog(SearchResultCollector.class).warn(String.format("Row %s: Invalid number of result columns: got %s, expected ",this.count, row.length, this.columns.length));
}
if (state() > BUFFER_FULL) {
if (this.exception == null)
setException(new RetsException("Attempting to add rows to buffer when in complete state"));
throw new NestableRuntimeException(this.exception);
}
// check complete.
while (checkRuntime() && state() == BUFFER_FULL) {
_wait();
if (state() >= BUFFER_FULL) {
if (this.exception == null)
setException(new RetsException("Timeout writing to streaming result set buffer, timeout length = "
+ this.timeout));
throw new NestableRuntimeException(this.exception);
}
}
this.buffer.addLast(row);
if (this.bufferSize == this.buffer.size())
pushState(BUFFER_FULL);
else
pushState(BUFFER_AVAILABLE);
this.notifyAll();
return true;
}
public synchronized void setComplete() {
pushState(COMPLETE);
notifyAll();
}
public synchronized void setCount(int count) {
this.count = count;
pushState(PREPROCESS);
notifyAll();
}
public synchronized void setColumns(String[] columns) {
this.columns = columns;
pushState(BUFFER_AVAILABLE);
notifyAll();
}
public synchronized void setMaxrows() {
this.mMaxrows = true;
pushState(COMPLETE);
notifyAll();
}
synchronized void setException(RetsException e) {
this.exception = e;
pushState(COMPLETE);
notifyAll();
}
// ----------- Consumer Methods
public synchronized boolean hasNext() throws RetsException {
// wait for someone to add data to the queue
// or flag complete
while (checkException() && state() < COMPLETE) {
if (!this.buffer.isEmpty())
return true;
_wait();
}
return !this.buffer.isEmpty();
}
public synchronized String[] next() throws RetsException {
checkException();
String[] row = this.buffer.removeFirst();
if (this.state < COMPLETE)
pushState(BUFFER_AVAILABLE);
this.notifyAll();
return row;
}
public synchronized int getCount() throws RetsException {
while (checkException() && state() < BUFFER_AVAILABLE) {
_wait();
}
return this.count;
}
public synchronized String[] getColumns() throws RetsException {
while (checkException() && state() < BUFFER_AVAILABLE) {
_wait();
}
return this.columns;
}
public synchronized boolean isMaxrows() throws RetsException {
checkException();
if (!isComplete())
throw new IllegalStateException("Cannot call isMaxRows until isComplete == true");
return this.mMaxrows;
}
public synchronized SearchResultInfo getInfo() throws RetsException {
checkException();
if (!isComplete())
throw new IllegalStateException("Cannot call isMaxRows until isComplete == true");
return this;
}
public synchronized boolean isComplete() throws RetsException {
checkException();
return state() >= COMPLETE;
}
private synchronized boolean checkRuntime() {
try {
return checkException();
} catch (RetsException e) {
throw new NestableRuntimeException(e);
}
}
private synchronized boolean checkException() throws RetsException {
// considering doing something here to maintain the original
// stack trace but also provide the stack trace from this
// location...
if (this.exception != null)
throw this.exception;
return true;
}
private void _wait() {
try {
wait(this.timeout);
} catch (InterruptedException e) {
pushState(COMPLETE);
throw new NestableRuntimeException(e);
}
}
private void pushState(int newState) {
if (this.state >= COMPLETE && newState < COMPLETE)
throw new IllegalStateException("Cannot revert from complete state");
if (this.state > PREPROCESS && newState <= PREPROCESS)
throw new IllegalStateException("Cannot revert to preprocess state");
if (newState < this.state && newState != BUFFER_AVAILABLE && this.state != BUFFER_FULL)
throw new IllegalStateException("Cannot go back in state unless reverting to buffer available from full");
this.state = newState;
}
private int state() {
return this.state;
}
}

15
src/main/java/com/ossez/reoc/rets/client/VersionInsensitiveRequest.java Normal file
View File

@ -0,0 +1,15 @@
package com.ossez.reoc.rets.client;
public abstract class VersionInsensitiveRequest extends RetsHttpRequest {
/**
* Abstract class of subclasses where the Version of RETS is not needed (Password Request, Login Request, etc.)
*/
public VersionInsensitiveRequest() {
super();
}
@Override
public void setVersion(RetsVersion version) {
//noop - I don't care about version
}
}

41
src/main/java/com/ossez/reoc/rets/client/metadata_tables.xml Normal file
View File

@ -0,0 +1,41 @@
<table name="SYSTEM">
<table name="RESOURCE" id-column="ResourceID">
<table name="CLASS" id-column="ClassName" path-attributes="Resource">
<table name="TABLE" id-column="SystemName"
path-attributes="Resource,Class"/>
<table name="UPDATE" id-column="UpdateName"
path-attributes="Resource,Class">
<table name="UPDATE_TYPE" id-column="SystemName"
path-attributes="Resource,Class,Update"/>
</table>
</table>
<table name="UPDATE_HELP" id-column="UpdateHelpID"
path-attributes="Resource"/>
<table name="OBJECT" id-column="ObjectType" path-attributes="Resource"/>
<table name="SEARCH_HELP" id-column="SearchHelpID"
path-attributes="Resource"/>
<table name="EDITMASK" id-column="EditMaskId" path-attributes="Resource"/>
<table name="LOOKUP" id-column="LookupName" path-attributes="Resource">
<table name="LOOKUP_TYPE" id-column="LongValue"
path-attributes="Resource,Lookup"/>
</table>
<table name="VALIDATION_LOOKUP" id-column="ValidationLookupName"
path-attributes="Resource">
<table name="VALIDATION_LOOKUP_TYPE" id-column="ValidText"
path-attributes="Resource,ValidationLookup"/>
</table>
<table name="VALIDATION_EXTERNAL" id-column="ValidationExternalName"
path-attributes="Resource">
<table name="VALIDATION_EXTERNAL_TYPE" id-column="SearchField"
path-attributes="Resource,ValidationExternal"/>
</table>
<table name="VALIDATION_EXPRESSION" id-column="ValidationExpressionID"
path-attributes="Resource"/>
</table>
<table name="FOREIGNKEYS" id-column="ForeignKeyID"/>
</table>

16
src/main/java/com/ossez/reoc/rets/common/metadata/AttrType.java Normal file
View File

@ -0,0 +1,16 @@
/*
* cart: CRT's Awesome RETS Tool
*
* Author: David Terrell
* Copyright (c) 2003, The National Association of REALTORS
* Distributed under a BSD-style license. See LICENSE.TXT for details.
*/
package com.ossez.reoc.rets.common.metadata;
import java.io.Serializable;
public interface AttrType<T> extends Serializable {
public T parse(String value, boolean strict) throws MetaParseException;
public Class<T> getType();
public String render(T value);
}

706
src/main/java/com/ossez/reoc/rets/common/metadata/JDomCompactBuilder.java Normal file
View File

@ -0,0 +1,706 @@
/*
* cart: CRT's Awesome RETS Tool
*
* Author: David Terrell
* Copyright (c) 2003, The National Association of REALTORS
* Distributed under a BSD-style license. See LICENSE.TXT for details.
*/
package com.ossez.reoc.rets.common.metadata;
import java.io.IOException;
import java.util.LinkedList;
import java.util.List;
import java.util.StringTokenizer;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.jdom.Document;
import org.jdom.Element;
import org.jdom.JDOMException;
import org.jdom.input.SAXBuilder;
import com.ossez.reoc.rets.common.metadata.types.MClass;
import com.ossez.reoc.rets.common.metadata.types.MEditMask;
import com.ossez.reoc.rets.common.metadata.types.MLookup;
import com.ossez.reoc.rets.common.metadata.types.MLookupType;
import com.ossez.reoc.rets.common.metadata.types.MObject;
import com.ossez.reoc.rets.common.metadata.types.MResource;
import com.ossez.reoc.rets.common.metadata.types.MSearchHelp;
import com.ossez.reoc.rets.common.metadata.types.MSystem;
import com.ossez.reoc.rets.common.metadata.types.MTable;
import com.ossez.reoc.rets.common.metadata.types.MUpdate;
import com.ossez.reoc.rets.common.metadata.types.MUpdateType;
import com.ossez.reoc.rets.common.metadata.types.MValidationExpression;
import com.ossez.reoc.rets.common.metadata.types.MValidationExternal;
import com.ossez.reoc.rets.common.metadata.types.MValidationExternalType;
import com.ossez.reoc.rets.common.metadata.types.MValidationLookup;
import com.ossez.reoc.rets.common.metadata.types.MValidationLookupType;
import org.xml.sax.InputSource;
public class JDomCompactBuilder extends MetadataBuilder {
public static final String CONTAINER_PREFIX = "METADATA-";
public static final String CONTAINER_ROOT = "RETS";
public static final String CONTAINER_METADATA = "METADATA";
public static final String CONTAINER_SYSTEM = "METADATA-SYSTEM";
public static final String CONTAINER_RESOURCE = "METADATA-RESOURCE";
public static final String CONTAINER_FOREIGNKEY = "METADATA-FOREIGN_KEY";
public static final String CONTAINER_CLASS = "METADATA-CLASS";
public static final String CONTAINER_TABLE = "METADATA-TABLE";
public static final String CONTAINER_UPDATE = "METADATA-UPDATE";
public static final String CONTAINER_UPDATETYPE = "METADATA-UPDATE_TYPE";
public static final String CONTAINER_OBJECT = "METADATA-OBJECT";
public static final String CONTAINER_SEARCHHELP = "METADATA-SEARCH_HELP";
public static final String CONTAINER_EDITMASK = "METADATA-EDITMASK";
public static final String CONTAINER_UPDATEHELP = "METADATA-UPDATE_HELP";
public static final String CONTAINER_LOOKUP = "METADATA-LOOKUP";
public static final String CONTAINER_LOOKUPTYPE = "METADATA-LOOKUP_TYPE";
public static final String CONTAINER_VALIDATIONLOOKUP = "METADATA-VALIDATION_LOOKUP";
public static final String CONTAINER_VALIDATIONLOOKUPTYPE = "METADATA-VALIDATION_LOOKUP_TYPE";
public static final String CONTAINER_VALIDATIONEXPRESSION = "METADATA-VALIDATION_EXPRESSION";
public static final String CONTAINER_VALIDATIONEXTERNAL = "METADATA-VALIDATION_EXTERNAL";
public static final String CONTAINER_VALIDATIONEXTERNALTYPE = "METADATA-VALIDATION_EXTERNAL_TYPE";
public static final String ELEMENT_SYSTEM = "SYSTEM";
public static final String COLUMNS = "COLUMNS";
public static final String DATA = "DATA";
public static final String ATTRIBUTE_RESOURCE = "Resource";
public static final String ATTRIBUTE_CLASS = "Class";
public static final String ATTRIBUTE_UPDATE = "Update";
public static final String ATTRIBUTE_LOOKUP = "Lookup";
public static final String ATTRIBUTE_VALIDATIONEXTERNAL = "ValidationExternal";
public static final String ATTRIBUTE_VALIDATIONLOOKUP = "ValidationLookup";
private static final Log LOG = LogFactory.getLog(JDomCompactBuilder.class);
@Override
public Metadata doBuild(Object src) throws MetadataException {
return build((Document) src);
}
public Metadata build(InputSource source) throws MetadataException {
SAXBuilder builder = new SAXBuilder();
Document document;
try {
document = builder.build(source);
} catch (JDOMException e) {
throw new MetadataException("Couldn't build document", e);
} catch (IOException e) {
throw new MetadataException("Couldn't build document", e);
}
return build(document);
}
@Override
public MetaObject[] parse(Object src) throws MetadataException {
return parse((Document) src);
}
public MetaObject[] parse(Document src) throws MetadataException {
Element root = src.getRootElement();
if (!root.getName().equals(CONTAINER_ROOT)) {
throw new MetadataException("Invalid root element");
}
Element container = root.getChild(CONTAINER_SYSTEM);
if (container != null) {
MSystem sys = processSystem(container);
if (root.getChild(CONTAINER_RESOURCE) != null) {
Metadata m = new Metadata(sys);
recurseAll(m, root);
}
return new MetaObject[] { sys };
}
container = root.getChild(CONTAINER_RESOURCE);
if (container != null) {
return processResource(container);
}
container = root.getChild(CONTAINER_CLASS);
if (container != null) {
return processClass(container);
}
container = root.getChild(CONTAINER_TABLE);
if (container != null) {
return processTable(container);
}
container = root.getChild(CONTAINER_UPDATE);
if (container != null) {
return processUpdate(container);
}
container = root.getChild(CONTAINER_UPDATETYPE);
if (container != null) {
return processUpdateType(container);
}
container = root.getChild(CONTAINER_OBJECT);
if (container != null) {
return processObject(container);
}
container = root.getChild(CONTAINER_SEARCHHELP);
if (container != null) {
return processSearchHelp(container);
}
container = root.getChild(CONTAINER_EDITMASK);
if (container != null) {
return processEditMask(container);
}
container = root.getChild(CONTAINER_LOOKUP);
if (container != null) {
return processLookup(container);
}
container = root.getChild(CONTAINER_LOOKUPTYPE);
if (container != null) {
return processLookupType(container);
}
container = root.getChild(CONTAINER_VALIDATIONLOOKUP);
if (container != null) {
return processValidationLookup(container);
}
container = root.getChild(CONTAINER_VALIDATIONLOOKUPTYPE);
if (container != null) {
return processValidationLookupType(container);
}
container = root.getChild(CONTAINER_VALIDATIONEXTERNAL);
if (container != null) {
return processValidationExternal(container);
}
container = root.getChild(CONTAINER_VALIDATIONEXTERNALTYPE);
if (container != null) {
return processValidationExternalType(container);
}
container = root.getChild(CONTAINER_VALIDATIONEXPRESSION);
if (container != null) {
return processValidationExpression(container);
}
return null;
}
public Metadata build(Document src) throws MetadataException {
Element root = src.getRootElement();
if (!root.getName().equals(CONTAINER_ROOT)) {
throw new MetadataException("Invalid root element");
}
Element element = root.getChild(CONTAINER_SYSTEM);
if (element == null) {
throw new MetadataException("Missing element " + CONTAINER_SYSTEM);
}
MSystem sys = processSystem(element);
Metadata metadata;
metadata = new Metadata(sys);
recurseAll(metadata, root);
return metadata;
}
private void recurseAll(Metadata metadata, Element root) throws MetaParseException {
attachResource(metadata, root);
attachClass(metadata, root);
attachTable(metadata, root);
attachUpdate(metadata, root);
attachUpdateType(metadata, root);
attachObject(metadata, root);
attachSearchHelp(metadata, root);
attachEditMask(metadata, root);
attachLookup(metadata, root);
attachLookupType(metadata, root);
attachValidationLookup(metadata, root);
attachValidationLookupType(metadata, root);
attachValidationExternal(metadata, root);
attachValidationExternalType(metadata, root);
attachValidationExpression(metadata, root);
}
private void setAttributes(MetaObject obj, String[] columns, String[] data) {
int count = columns.length;
if (count > data.length) {
count = data.length;
}
for (int i = 0; i < count; i++) {
String column = columns[i];
String datum = data[i];
if (!datum.equals("")) {
setAttribute(obj, column, datum);
}
}
}
private String[] getColumns(Element el) {
Element cols = el.getChild(COLUMNS);
return split(cols);
}
/** do NOT use string.split() unless your prepared to deal with loss due to token boundary conditions */
private String[] split(Element el) {
if( el == null ) return null;
final String delimiter = "\t";
StringTokenizer tkn = new StringTokenizer(el.getText(), delimiter, true);
List list = new LinkedList();
tkn.nextToken(); // junk the first element
String last = null;
while (tkn.hasMoreTokens()) {
String next = tkn.nextToken();
if (next.equals(delimiter)) {
if (last == null) {
list.add("");
} else {
last = null;
}
} else {
list.add(next);
last = next;
}
}
return (String[]) list.toArray(new String[0]);
}
/**
* Gets an attribute that is not expected to be null (i.e. an attribute that
* MUST exist).
*
* @param element Element
* @param name Attribute name
* @return value of attribute
* @throws MetaParseException if the value is null.
*/
private String getNonNullAttribute(Element element, String name) throws MetaParseException {
String value = element.getAttributeValue(name);
if (value == null) {
throw new MetaParseException("Attribute '" + name + "' not found on tag " + toString(element));
}
return value;
}
private String toString(Element element) {
StringBuffer buffer = new StringBuffer();
List attributes = element.getAttributes();
buffer.append("'").append(element.getName()).append("'");
buffer.append(", attributes: ").append(attributes);
return buffer.toString();
}
private MSystem processSystem(Element container) {
Element element = container.getChild(ELEMENT_SYSTEM);
MSystem system = buildSystem();
// system metadata is such a hack. the first one here is by far my favorite
String comment = container.getChildText(MSystem.COMMENTS);
String systemId = element.getAttributeValue(MSystem.SYSTEMID);
String systemDescription = element.getAttributeValue(MSystem.SYSTEMDESCRIPTION);
String version = container.getAttributeValue(MSystem.VERSION);
String date = container.getAttributeValue(MSystem.DATE);
setAttribute(system, MSystem.COMMENTS, comment);
setAttribute(system, MSystem.SYSTEMID, systemId);
setAttribute(system, MSystem.SYSTEMDESCRIPTION, systemDescription);
setAttribute(system, MSystem.VERSION, version);
setAttribute(system, MSystem.DATE, date);
return system;
}
private void attachResource(Metadata metadata, Element root) {
MSystem system = metadata.getSystem();
List containers = root.getChildren(CONTAINER_RESOURCE);
for (int i = 0; i < containers.size(); i++) {
Element container = (Element) containers.get(i);
MResource[] resources = this.processResource(container);
for (int j = 0; j < resources.length; j++) {
system.addChild(MetadataType.RESOURCE, resources[j]);
}
}
}
private MResource[] processResource(Element resourceContainer) {
String[] columns = getColumns(resourceContainer);
List rows = resourceContainer.getChildren(DATA);
MResource[] resources = new MResource[rows.size()];
for (int i = 0; i < rows.size(); i++) {
Element element = (Element) rows.get(i);
String[] data = split(element);
MResource resource = buildResource();
setAttributes(resource, columns, data);
resources[i] = resource;
}
return resources;
}
private void attachClass(Metadata metadata, Element root) throws MetaParseException {
List containers = root.getChildren(CONTAINER_CLASS);
for (int i = 0; i < containers.size(); i++) {
Element container = (Element) containers.get(i);
String resourceId = getNonNullAttribute(container, ATTRIBUTE_RESOURCE);
MResource resource = metadata.getResource(resourceId);
MClass[] classes = processClass(container);
for (int j = 0; j < classes.length; j++) {
resource.addChild(MetadataType.CLASS, classes[j]);
}
}
}
private MClass[] processClass(Element classContainer) throws MetaParseException {
String name = classContainer.getName();
String resourceId = getNonNullAttribute(classContainer, ATTRIBUTE_RESOURCE);
LOG.debug("resource name: " + resourceId + " for container " + name);
String[] columns = getColumns(classContainer);
List rows = classContainer.getChildren(DATA);
MClass[] classes = new MClass[rows.size()];
for (int i = 0; i < rows.size(); i++) {
Element element = (Element) rows.get(i);
String[] data = split(element);
MClass clazz = buildClass();
setAttributes(clazz, columns, data);
classes[i] = clazz;
}
return classes;
}
private void attachTable(Metadata metadata, Element root) throws MetaParseException {
List containers = root.getChildren(CONTAINER_TABLE);
for (int i = 0; i < containers.size(); i++) {
Element container = (Element) containers.get(i);
String resourceId = getNonNullAttribute(container, ATTRIBUTE_RESOURCE);
String className = getNonNullAttribute(container, ATTRIBUTE_CLASS);
MClass clazz = metadata.getMClass(resourceId, className);
if (clazz == null) {
//MarketLinx Strikes!!!
LOG.warn("Found table metadata for resource class: " + resourceId + ":" + className
+ " but there is no class metadata for " + resourceId + ":" + className);
continue;
}
MTable[] fieldMetadata = processTable(container);
for (int j = 0; j < fieldMetadata.length; j++) {
clazz.addChild(MetadataType.TABLE, fieldMetadata[j]);
}
}
}
private MTable[] processTable(Element tableContainer) {
String[] columns = getColumns(tableContainer);
List rows = tableContainer.getChildren(DATA);
MTable[] fieldMetadata = new MTable[rows.size()];
for (int i = 0; i < rows.size(); i++) {
Element element = (Element) rows.get(i);
String[] data = split(element);
MTable mTable = buildTable();
setAttributes(mTable, columns, data);
fieldMetadata[i] = mTable;
}
return fieldMetadata;
}
private void attachUpdate(Metadata metadata, Element root) throws MetaParseException {
List containers = root.getChildren(CONTAINER_UPDATE);
for (int i = 0; i < containers.size(); i++) {
Element container = (Element) containers.get(i);
MClass parent = metadata.getMClass(getNonNullAttribute(container, ATTRIBUTE_RESOURCE), getNonNullAttribute(
container, ATTRIBUTE_CLASS));
MUpdate[] updates = processUpdate(container);
for (int j = 0; j < updates.length; j++) {
parent.addChild(MetadataType.UPDATE, updates[j]);
}
}
}
private MUpdate[] processUpdate(Element container) {
String[] columns = getColumns(container);
List rows = container.getChildren(DATA);
MUpdate[] updates = new MUpdate[rows.size()];
for (int i = 0; i < rows.size(); i++) {
Element element = (Element) rows.get(i);
String[] data = split(element);
MUpdate update = buildUpdate();
setAttributes(update, columns, data);
updates[i] = update;
}
return updates;
}
private void attachUpdateType(Metadata metadata, Element root) throws MetaParseException {
List containers = root.getChildren(CONTAINER_UPDATETYPE);
for (int i = 0; i < containers.size(); i++) {
Element container = (Element) containers.get(i);
MUpdate parent = metadata.getUpdate(getNonNullAttribute(container, ATTRIBUTE_RESOURCE),
getNonNullAttribute(container, ATTRIBUTE_CLASS), getNonNullAttribute(container, ATTRIBUTE_UPDATE));
MUpdateType[] updateTypes = processUpdateType(container);
for (int j = 0; j < updateTypes.length; j++) {
parent.addChild(MetadataType.UPDATE_TYPE, updateTypes[j]);
}
}
}
private MUpdateType[] processUpdateType(Element container) {
String[] columns = getColumns(container);
List rows = container.getChildren(DATA);
MUpdateType[] updateTypes = new MUpdateType[rows.size()];
for (int i = 0; i < rows.size(); i++) {
Element element = (Element) rows.get(i);
String[] data = split(element);
MUpdateType updateType = buildUpdateType();
setAttributes(updateType, columns, data);
updateTypes[i] = updateType;
}
return updateTypes;
}
private void attachObject(Metadata metadata, Element root) throws MetaParseException {
List containers = root.getChildren(CONTAINER_OBJECT);
for (int i = 0; i < containers.size(); i++) {
Element container = (Element) containers.get(i);
MResource parent = metadata.getResource(getNonNullAttribute(container, ATTRIBUTE_RESOURCE));
MObject[] objects = processObject(container);
for (int j = 0; j < objects.length; j++) {
parent.addChild(MetadataType.OBJECT, objects[j]);
}
}
}
private MObject[] processObject(Element objectContainer) {
String[] columns = getColumns(objectContainer);
List rows = objectContainer.getChildren(DATA);
MObject[] objects = new MObject[rows.size()];
for (int i = 0; i < rows.size(); i++) {
Element element = (Element) rows.get(i);
String[] data = split(element);
MObject object = buildObject();
setAttributes(object, columns, data);
objects[i] = object;
}
return objects;
}
private void attachSearchHelp(Metadata metadata, Element root) throws MetaParseException {
List containers = root.getChildren(CONTAINER_SEARCHHELP);
for (int i = 0; i < containers.size(); i++) {
Element container = (Element) containers.get(i);
MResource parent = metadata.getResource(getNonNullAttribute(container, ATTRIBUTE_RESOURCE));
MSearchHelp[] searchHelps = processSearchHelp(container);
for (int j = 0; j < searchHelps.length; j++) {
parent.addChild(MetadataType.SEARCH_HELP, searchHelps[j]);
}
}
}
private MSearchHelp[] processSearchHelp(Element container) {
String[] columns = getColumns(container);
List rows = container.getChildren(DATA);
MSearchHelp[] searchHelps = new MSearchHelp[rows.size()];
for (int i = 0; i < rows.size(); i++) {
Element element = (Element) rows.get(i);
String[] data = split(element);
MSearchHelp searchHelp = buildSearchHelp();
setAttributes(searchHelp, columns, data);
searchHelps[i] = searchHelp;
}
return searchHelps;
}
private void attachEditMask(Metadata metadata, Element root) throws MetaParseException {
List containers = root.getChildren(CONTAINER_EDITMASK);
for (int i = 0; i < containers.size(); i++) {
Element container = (Element) containers.get(i);
MResource parent = metadata.getResource(getNonNullAttribute(container, ATTRIBUTE_RESOURCE));
MEditMask[] editMasks = processEditMask(container);
for (int j = 0; j < editMasks.length; j++) {
parent.addChild(MetadataType.EDITMASK, editMasks[j]);
}
}
}
private MEditMask[] processEditMask(Element container) {
String[] columns = getColumns(container);
List rows = container.getChildren(DATA);
MEditMask[] editMasks = new MEditMask[rows.size()];
for (int i = 0; i < rows.size(); i++) {
Element element = (Element) rows.get(i);
String[] data = split(element);
MEditMask editMask = buildEditMask();
setAttributes(editMask, columns, data);
editMasks[i] = editMask;
}
return editMasks;
}
private void attachLookup(Metadata metadata, Element root) throws MetaParseException {
List containers = root.getChildren(CONTAINER_LOOKUP);
for (int i = 0; i < containers.size(); i++) {
Element container = (Element) containers.get(i);
MResource parent = metadata.getResource(getNonNullAttribute(container, ATTRIBUTE_RESOURCE));
MLookup[] lookups = processLookup(container);
for (int j = 0; j < lookups.length; j++) {
parent.addChild(MetadataType.LOOKUP, lookups[j]);
}
}
}
private MLookup[] processLookup(Element container) {
String[] columns = getColumns(container);
List rows = container.getChildren(DATA);
MLookup[] lookups = new MLookup[rows.size()];
for (int i = 0; i < rows.size(); i++) {
Element element = (Element) rows.get(i);
String[] data = split(element);
MLookup lookup = buildLookup();
setAttributes(lookup, columns, data);
lookups[i] = lookup;
}
return lookups;
}
private void attachLookupType(Metadata metadata, Element root) throws MetaParseException {
List containers = root.getChildren(CONTAINER_LOOKUPTYPE);
for (int i = 0; i < containers.size(); i++) {
Element container = (Element) containers.get(i);
MLookup parent = metadata.getLookup(getNonNullAttribute(container, ATTRIBUTE_RESOURCE),
getNonNullAttribute(container, ATTRIBUTE_LOOKUP));
if (parent == null) {
LOG.warn("Skipping lookup type: could not find lookup for tag " + toString(container));
continue;
}
MLookupType[] lookupTypes = processLookupType(container);
for (int j = 0; j < lookupTypes.length; j++) {
parent.addChild(MetadataType.LOOKUP_TYPE, lookupTypes[j]);
}
}
}
private MLookupType[] processLookupType(Element container) {
String[] columns = getColumns(container);
List rows = container.getChildren(DATA);
MLookupType[] lookupTypes = new MLookupType[rows.size()];
for (int i = 0; i < rows.size(); i++) {
Element element = (Element) rows.get(i);
String[] data = split(element);
MLookupType lookupType = buildLookupType();
setAttributes(lookupType, columns, data);
lookupTypes[i] = lookupType;
}
return lookupTypes;
}
private void attachValidationLookup(Metadata metadata, Element root) throws MetaParseException {
List containers = root.getChildren(CONTAINER_VALIDATIONLOOKUP);
for (int i = 0; i < containers.size(); i++) {
Element container = (Element) containers.get(i);
MResource parent = metadata.getResource(getNonNullAttribute(container, ATTRIBUTE_RESOURCE));
MValidationLookup[] validationLookups = processValidationLookup(container);
for (int j = 0; j < validationLookups.length; j++) {
parent.addChild(MetadataType.VALIDATION_LOOKUP, validationLookups[j]);
}
}
}
private MValidationLookup[] processValidationLookup(Element container) {
String[] columns = getColumns(container);
List rows = container.getChildren(DATA);
MValidationLookup[] validationLookups = new MValidationLookup[rows.size()];
for (int i = 0; i < rows.size(); i++) {
Element element = (Element) rows.get(i);
String[] data = split(element);
MValidationLookup validationLookup = buildValidationLookup();
setAttributes(validationLookup, columns, data);
validationLookups[i] = validationLookup;
}
return validationLookups;
}
private void attachValidationLookupType(Metadata metadata, Element root) throws MetaParseException {
List containers = root.getChildren(CONTAINER_VALIDATIONLOOKUPTYPE);
for (int i = 0; i < containers.size(); i++) {
Element container = (Element) containers.get(i);
MValidationLookup parent = metadata.getValidationLookup(getNonNullAttribute(container, ATTRIBUTE_RESOURCE),
getNonNullAttribute(container, ATTRIBUTE_VALIDATIONLOOKUP));
MValidationLookupType[] validationLookupTypes = processValidationLookupType(container);
for (int j = 0; j < validationLookupTypes.length; j++) {
parent.addChild(MetadataType.VALIDATION_LOOKUP_TYPE, validationLookupTypes[j]);
}
}
}
private MValidationLookupType[] processValidationLookupType(Element container) {
String[] columns = getColumns(container);
List rows = container.getChildren(DATA);
MValidationLookupType[] validationLookupTypes = new MValidationLookupType[rows.size()];
for (int i = 0; i < rows.size(); i++) {
Element element = (Element) rows.get(i);
String[] data = split(element);
MValidationLookupType validationLookupType = buildValidationLookupType();
setAttributes(validationLookupType, columns, data);
validationLookupTypes[i] = validationLookupType;
}
return validationLookupTypes;
}
private void attachValidationExternal(Metadata metadata, Element root) {
List containers = root.getChildren(CONTAINER_VALIDATIONEXTERNAL);
for (int i = 0; i < containers.size(); i++) {
Element container = (Element) containers.get(i);
MResource parent = metadata.getResource(container.getAttributeValue(ATTRIBUTE_RESOURCE));
MValidationExternal[] validationExternals = processValidationExternal(container);
for (int j = 0; j < validationExternals.length; j++) {
parent.addChild(MetadataType.VALIDATION_EXTERNAL, validationExternals[j]);
}
}
}
private MValidationExternal[] processValidationExternal(Element container) {
String[] columns = getColumns(container);
List rows = container.getChildren(DATA);
MValidationExternal[] validationExternals = new MValidationExternal[rows.size()];
for (int i = 0; i < rows.size(); i++) {
Element element = (Element) rows.get(i);
String[] data = split(element);
MValidationExternal validationExternal = buildValidationExternal();
setAttributes(validationExternal, columns, data);
validationExternals[i] = validationExternal;
}
return validationExternals;
}
private void attachValidationExternalType(Metadata metadata, Element root) throws MetaParseException {
List containers = root.getChildren(CONTAINER_VALIDATIONEXTERNALTYPE);
for (int i = 0; i < containers.size(); i++) {
Element container = (Element) containers.get(i);
MValidationExternal parent = metadata.getValidationExternal(getNonNullAttribute(container,
ATTRIBUTE_RESOURCE), getNonNullAttribute(container, ATTRIBUTE_VALIDATIONEXTERNAL));
MValidationExternalType[] validationExternalTypes = processValidationExternalType(container);
for (int j = 0; j < validationExternalTypes.length; j++) {
parent.addChild(MetadataType.VALIDATION_EXTERNAL_TYPE, validationExternalTypes[j]);
}
}
}
private MValidationExternalType[] processValidationExternalType(Element container) {
String[] columns = getColumns(container);
List rows = container.getChildren(DATA);
MValidationExternalType[] validationExternalTypes = new MValidationExternalType[rows.size()];
for (int i = 0; i < rows.size(); i++) {
Element element = (Element) rows.get(i);
String[] data = split(element);
MValidationExternalType validationExternalType = buildValidationExternalType();
setAttributes(validationExternalType, columns, data);
validationExternalTypes[i] = validationExternalType;
}
return validationExternalTypes;
}
private void attachValidationExpression(Metadata metadata, Element root) throws MetaParseException {
List containers = root.getChildren(CONTAINER_VALIDATIONEXPRESSION);
for (int i = 0; i < containers.size(); i++) {
Element container = (Element) containers.get(i);
MResource parent = metadata.getResource(getNonNullAttribute(container, ATTRIBUTE_RESOURCE));
MValidationExpression[] expressions = processValidationExpression(container);
for (int j = 0; j < expressions.length; j++) {
parent.addChild(MetadataType.VALIDATION_EXPRESSION, expressions[j]);
}
}
}
private MValidationExpression[] processValidationExpression(Element container) {
String[] columns = getColumns(container);
List rows = container.getChildren(DATA);
MValidationExpression[] expressions = new MValidationExpression[rows.size()];
for (int i = 0; i < expressions.length; i++) {
Element element = (Element) rows.get(i);
String[] data = split(element);
MValidationExpression expression = buildValidationExpression();
setAttributes(expression, columns, data);
expressions[i] = expression;
}
return expressions;
}
}

628
src/main/java/com/ossez/reoc/rets/common/metadata/JDomStandardBuilder.java Normal file
View File

@ -0,0 +1,628 @@
/*
* cart: CRT's Awesome RETS Tool
*
* Author: David Terrell
* Copyright (c) 2003, The National Association of REALTORS
* Distributed under a BSD-style license. See LICENSE.TXT for details.
*/
package com.ossez.reoc.rets.common.metadata;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.HashMap;
import org.jdom.Attribute;
import org.jdom.Document;
import org.jdom.Element;
import com.ossez.reoc.rets.common.metadata.types.MClass;
import com.ossez.reoc.rets.common.metadata.types.MEditMask;
import com.ossez.reoc.rets.common.metadata.types.MForeignKey;
import com.ossez.reoc.rets.common.metadata.types.MLookup;
import com.ossez.reoc.rets.common.metadata.types.MLookupType;
import com.ossez.reoc.rets.common.metadata.types.MObject;
import com.ossez.reoc.rets.common.metadata.types.MResource;
import com.ossez.reoc.rets.common.metadata.types.MSearchHelp;
import com.ossez.reoc.rets.common.metadata.types.MSystem;
import com.ossez.reoc.rets.common.metadata.types.MTable;
import com.ossez.reoc.rets.common.metadata.types.MUpdate;
import com.ossez.reoc.rets.common.metadata.types.MUpdateHelp;
import com.ossez.reoc.rets.common.metadata.types.MUpdateType;
import com.ossez.reoc.rets.common.metadata.types.MValidationExpression;
import com.ossez.reoc.rets.common.metadata.types.MValidationExternal;
import com.ossez.reoc.rets.common.metadata.types.MValidationExternalType;
import com.ossez.reoc.rets.common.metadata.types.MValidationLookup;
import com.ossez.reoc.rets.common.metadata.types.MValidationLookupType;
/** Parses apart a complete Standard-XML response, returns a Metadata object */
public class JDomStandardBuilder extends MetadataBuilder {
public static final String ELEMENT_SYSTEM = "System";
public static final String ELEMENT_RESOURCE = "Resource";
public static final String ELEMENT_FOREIGNKEY = "ForeignKey";
public static final String ELEMENT_CLASS = "Class";
public static final String ELEMENT_TABLE = "Field";
public static final String ELEMENT_UPDATE = "UpdateType";
public static final String ELEMENT_UPDATETYPE = "UpdateField";
public static final String ELEMENT_OBJECT = "Object";
public static final String ELEMENT_SEARCHHELP = "SearchHelp";
public static final String ELEMENT_EDITMASK = "EditMask";
public static final String ELEMENT_UPDATEHELP = "UpdateHelp";
public static final String ELEMENT_LOOKUP = "Lookup";
public static final String ELEMENT_LOOKUPTYPE = "LookupType";
public static final String ELEMENT_VALIDATIONLOOKUP = "ValidationLookup";
public static final String ELEMENT_VALIDATIONLOOKUPTYPE = "ValidationLookupType";
public static final String ELEMENT_VALIDATIONEXPRESSION = "ValidationExpression";
public static final String ELEMENT_VALIDATIONEXTERNAL = "ValidationExternalType";
public static final String ELEMENT_VALIDATIONEXTERNALTYPE = "ValidationExternal";
public static final String ATTRIBUTE_RESOURCEID = ELEMENT_RESOURCE;
public static final String ATTRIBUTE_CLASSNAME = ELEMENT_CLASS;
public static final String ATTRIBUTE_UPDATE = ELEMENT_UPDATE;
public static final String ATTRIBUTE_LOOKUP = ELEMENT_LOOKUP;
public static final String ATTRIBUTE_VALIDATIONLOOKUP = ELEMENT_VALIDATIONLOOKUP;
public static final String ATTRIBUTE_VALIDATIONEXTERNAL = ELEMENT_VALIDATIONEXTERNAL;
public static final Map sType2Element = new HashMap();
static {
sType2Element.put(MetadataType.SYSTEM, ELEMENT_SYSTEM);
sType2Element.put(MetadataType.RESOURCE, ELEMENT_RESOURCE);
sType2Element.put(MetadataType.FOREIGNKEYS, ELEMENT_FOREIGNKEY);
sType2Element.put(MetadataType.CLASS, ELEMENT_CLASS);
sType2Element.put(MetadataType.TABLE, ELEMENT_TABLE);
sType2Element.put(MetadataType.UPDATE, ELEMENT_UPDATE);
sType2Element.put(MetadataType.UPDATE_TYPE, ELEMENT_UPDATETYPE);
sType2Element.put(MetadataType.SEARCH_HELP, ELEMENT_SEARCHHELP);
sType2Element.put(MetadataType.EDITMASK, ELEMENT_EDITMASK);
sType2Element.put(MetadataType.UPDATE_HELP, ELEMENT_UPDATEHELP);
sType2Element.put(MetadataType.LOOKUP, ELEMENT_LOOKUP);
sType2Element.put(MetadataType.LOOKUP_TYPE, ELEMENT_LOOKUPTYPE);
sType2Element.put(MetadataType.VALIDATION_LOOKUP, ELEMENT_VALIDATIONLOOKUP);
sType2Element.put(MetadataType.VALIDATION_LOOKUP_TYPE, ELEMENT_VALIDATIONLOOKUPTYPE);
sType2Element.put(MetadataType.VALIDATION_EXTERNAL, ELEMENT_VALIDATIONEXTERNAL);
sType2Element.put(MetadataType.VALIDATION_EXTERNAL_TYPE, ELEMENT_VALIDATIONEXTERNALTYPE);
sType2Element.put(MetadataType.VALIDATION_EXPRESSION, ELEMENT_VALIDATIONEXPRESSION);
}
@Override
public Metadata doBuild(Object src) throws MetadataException {
return build((Document) src);
}
public Metadata build(Document src) throws MetadataException {
Element element = src.getRootElement();
expectElement(element, CONTAINER_ROOT);
element = getElement(element, CONTAINER_METADATA);
return build(element);
}
@Override
public MetaObject[] parse(Object src) throws MetadataException {
return parse((Document) src);
}
public MetaObject[] parse(Document src) throws MetadataException {
Element element = src.getRootElement();
expectElement(element, CONTAINER_ROOT);
Element container = getElement(element, CONTAINER_METADATA);
boolean recurse = checkForRecursion(container);
List list = container.getChildren();
if (list.size() == 0) {
return null;
}
return processContainer(null, (Element) list.get(0), recurse);
}
/**
* Function to determine if a request contains recursive data or not.
* This is done here instead of inside processContainer because, well,
* it's easier and more reliable (processContainer might not figure out
* that a request is recursive until the third or 4th child if there are
* no children for the first couple of elements.
*
* @param top The outside METADATA container.
* @return true if the request is recursive
*
*/
private boolean checkForRecursion(Element top) {
/*
* this seems like a really nasty loop. However, if there are a
* lot of recursive elements, we'll find out pretty quickly, and if
* we fall all the way to the end then there probably wasn't that
* much to look through.
*/
Iterator children = top.getChildren().iterator();
while (children.hasNext()) {
/* each of these is a container (METADATA-*) type */
Element element = (Element) children.next();
Iterator iterator = element.getChildren().iterator();
while (iterator.hasNext()) {
/* each of these is an item element */
Element child = (Element) iterator.next();
Iterator subtypes = child.getChildren().iterator();
while (subtypes.hasNext()) {
Element subtype = (Element) subtypes.next();
if (subtype.getName().startsWith(CONTAINER_PREFIX)) {
return true;
}
}
}
}
return false;
}
private MetaObject[] processContainer(MetaObject parent, Element container, boolean recursion) {
MetadataType type = (MetadataType) sContainer2Type.get(container.getName());
if (type == null) {
throw new RuntimeException("no matching type for container " + container.getName());
}
List elements = container.getChildren((String) sType2Element.get(type));
String path = getPath(container);
List output = null;
if (parent == null) {
output = new LinkedList();
}
for (int i = 0; i < elements.size(); i++) {
Element element = (Element) elements.get(i);
MetaObject obj = newType(type);
setAttributes(obj, element);
if (output != null) {
output.add(obj);
}
if (parent != null) {
parent.addChild(type, obj);
} else {
/**
* Weirdness abounds. There IS an ID attribute of System,
* and the SystemID is included in the Metadata container
* attributes, but the system id is not part of the metadata
* request path for a getMetadata request, so we ignore it.
*/
if (!type.equals(MetadataType.SYSTEM)) {
obj.setPath(path);
}
}
if (recursion) {
MetadataType[] childTypes = obj.getChildTypes();
for (int j = 0; j < childTypes.length; j++) {
MetadataType childType = childTypes[j];
Element childContainer = element.getChild(CONTAINER_PREFIX + childType.name());
if (childContainer == null) {
obj.addChild(childType, null);
} else {
processContainer(obj, childContainer, true);
}
}
}
}
if (output == null) {
return null;
}
return (MetaObject[]) output.toArray(new MetaObject[0]);
}
String getPath(Element container) {
String resource = container.getAttributeValue(ATTRIBUTE_RESOURCEID);
if (resource == null) {
return null;
}
String classname = container.getAttributeValue(ATTRIBUTE_CLASSNAME);
if (classname != null) {
String update = container.getAttributeValue(ATTRIBUTE_UPDATE);
if (update != null) {
return resource + ":" + classname + ":" + update;
}
return resource + ":" + classname;
}
String lookup = container.getAttributeValue(ATTRIBUTE_LOOKUP);
if (lookup != null) {
return resource + ":" + lookup;
}
String vallkp = container.getAttributeValue(ATTRIBUTE_VALIDATIONLOOKUP);
if (vallkp != null) {
return resource + ":" + vallkp;
}
String vale = container.getAttributeValue(ATTRIBUTE_VALIDATIONEXTERNAL);
if (vale != null) {
return resource + ":" + vale;
}
return resource;
}
public Metadata build(Element element) throws MetadataException {
expectElement(element, CONTAINER_METADATA);
element = getElement(element, CONTAINER_SYSTEM);
//maybe i get the attribute here
MSystem sys = processSystem(element);
return new Metadata(sys);
}
private Element getElement(Element parent, String type) throws MetadataException {
Element element = parent.getChild(type);
if (element == null) {
throw new MetadataException("Missing element " + type);
}
return element;
}
private void expectElement(Element element, String type) throws MetadataException {
if (!element.getName().equalsIgnoreCase(type)) {// changed to ignore case
throw new MetadataException("Expecting element " + type + ", got " + element.getName());
}
}
private void setAttributes(MetaObject obj, Element el) {
List children = el.getChildren();
for (int i = 0; i < children.size(); i++) {
Element child = (Element) children.get(i);
String name = child.getName();
if (!name.startsWith(CONTAINER_PREFIX)) {
String value = child.getTextTrim();
setAttribute(obj, name, value);
} else {
// LOG.info("skipping container element " + name);
}
}
}
//when atrributes from the xml element are needed
public void setAttributesFromXMLAttr(MetaObject obj, Element el) {
Iterator attrIter = el.getParentElement().getAttributes().iterator();
while(attrIter.hasNext()){
Attribute attr = (Attribute) attrIter.next();
String name = attr.getName();
String value= attr.getValue().trim();
setAttribute(obj, name, value);
}
}
/**
* If we're a recursive request, initialize all possible child types so
* we don't have to try to pull them later, dynamically
*/
private void init(MetaObject item) {
MetadataType[] childTypes = item.getChildTypes();
for (int i = 0; i < childTypes.length; i++) {
MetadataType type = childTypes[i];
item.addChild(type, null);
}
}
private MSystem processSystem(Element container) {
Element element = container.getChild(ELEMENT_SYSTEM);
if (element == null){
element = container.getChild(ELEMENT_SYSTEM.toUpperCase());
}
MSystem system = buildSystem();
init(system);
setAttributesFromXMLAttr(system, element);
setAttributes(system, element);
Element child;
child = element.getChild(CONTAINER_RESOURCE);
if (child != null) {
processResource(system, child);
}
child = element.getChild(CONTAINER_FOREIGNKEY);
if (child != null) {
processForeignKey(system, child);
}
return system;
}
private void processResource(MSystem system, Element container) {
List resources = container.getChildren(ELEMENT_RESOURCE);
for (int i = 0; i < resources.size(); i++) {
Element element = (Element) resources.get(i);
MResource resource = buildResource();
init(resource);
setAttributes(resource, element);
system.addChild(MetadataType.RESOURCE, resource);
Element child;
child = element.getChild(CONTAINER_CLASS);
if (child != null) {
processClass(resource, child);
}
child = element.getChild(CONTAINER_OBJECT);
if (child != null) {
processObject(resource, child);
}
child = element.getChild(CONTAINER_SEARCH_HELP);
if (child != null) {
processSearchHelp(resource, child);
}
child = element.getChild(CONTAINER_EDITMASK);
if (child != null) {
processEditMask(resource, child);
}
child = element.getChild(CONTAINER_LOOKUP);
if (child != null) {
processLookup(resource, child);
}
child = element.getChild(CONTAINER_UPDATEHELP);
if (child != null) {
processUpdateHelp(resource, child);
}
child = element.getChild(CONTAINER_VALIDATIONLOOKUP);
if (child != null) {
processValidationLookup(resource, child);
}
child = element.getChild(CONTAINER_VALIDATIONEXPRESSION);
if (child != null) {
processValidationExpression(resource, child);
}
child = element.getChild(CONTAINER_VALIDATIONEXTERNAL);
if (child != null) {
processValidationExternal(resource, child);
}
}
}
private void processEditMask(MResource parent, Element container) {
List elements = container.getChildren(ELEMENT_EDITMASK);
for (int i = 0; i < elements.size(); i++) {
Element element = (Element) elements.get(i);
MEditMask mask = buildEditMask();
setAttributes(mask, element);
parent.addChild(MetadataType.EDITMASK, mask);
}
}
private void processLookup(MResource parent, Element container) {
List elements15 = container.getChildren(ELEMENT_LOOKUP);
List elements17 = container.getChildren(ELEMENT_LOOKUPTYPE);
List elements;
//some Rets Servers have lookuptype and lookup elements interchanged
if (elements15.isEmpty()){
elements = elements17;
} else {
elements = elements15;
}
for (int i = 0; i < elements.size(); i++) {
Element element = (Element) elements.get(i);
MLookup lookup = buildLookup();
init(lookup);
setAttributes(lookup, element);
parent.addChild(MetadataType.LOOKUP, lookup);
Element child = element.getChild(CONTAINER_LOOKUPTYPE);
if (child != null) {
processLookupType(lookup, child);
}
}
}
private void processLookupType(MLookup parent, Element container) {
List elements15 = container.getChildren(ELEMENT_LOOKUPTYPE);// check spec
List elements17 = container.getChildren(ELEMENT_LOOKUP);
List elements;
//some Rets Servers have lookuptype and lookup elements interchanged
if (elements15.isEmpty()){
elements = elements17;
} else {
elements = elements15;
}
for (int i = 0; i < elements.size(); i++) {
Element element = (Element) elements.get(i);
MLookupType type = buildLookupType();
setAttributes(type, element);
parent.addChild(MetadataType.LOOKUP_TYPE, type);
}
}
private void processUpdateHelp(MResource parent, Element container) {
List elements = container.getChildren(ELEMENT_UPDATEHELP);
for (int i = 0; i < elements.size(); i++) {
Element element = (Element) elements.get(i);
MUpdateHelp help = buildUpdateHelp();
setAttributes(help, element);
parent.addChild(MetadataType.UPDATE_HELP, help);
}
}
private void processValidationLookup(MResource parent, Element container) {
List elements = container.getChildren(ELEMENT_VALIDATIONLOOKUP);
for (int i = 0; i < elements.size(); i++) {
Element element = (Element) elements.get(i);
MValidationLookup lookup = buildValidationLookup();
init(lookup);
setAttributes(lookup, element);
parent.addChild(MetadataType.VALIDATION_LOOKUP, lookup);
Element child = element.getChild(CONTAINER_VALIDATIONLOOKUPTYPE);
if (child != null) {
processValidationLookupType(lookup, child);
}
}
}
private void processValidationLookupType(MValidationLookup parent, Element container) {
List elements = container.getChildren(ELEMENT_VALIDATIONLOOKUPTYPE);
for (int i = 0; i < elements.size(); i++) {
Element element = (Element) elements.get(i);
MValidationLookupType lookupType = buildValidationLookupType();
setAttributes(lookupType, element);
parent.addChild(MetadataType.VALIDATION_LOOKUP_TYPE, lookupType);
}
}
private void processValidationExpression(MResource parent, Element container) {
List elements = container.getChildren(ELEMENT_VALIDATIONEXPRESSION);
for (int i = 0; i < elements.size(); i++) {
Element element = (Element) elements.get(i);
MValidationExpression expression = buildValidationExpression();
setAttributes(expression, element);
parent.addChild(MetadataType.VALIDATION_EXPRESSION, expression);
}
}
private void processValidationExternal(MResource parent, Element container) {
List elements = container.getChildren(ELEMENT_VALIDATIONEXTERNAL);
for (int i = 0; i < elements.size(); i++) {
Element element = (Element) elements.get(i);
MValidationExternal external = buildValidationExternal();
init(external);
setAttributes(external, element);
parent.addChild(MetadataType.VALIDATION_EXTERNAL, external);
Element child = element.getChild(CONTAINER_VALIDATIONEXTERNALTYPE);
if (child != null) {
processValidationExternalType(external, child);
}
}
}
private void processValidationExternalType(MValidationExternal parent, Element container) {
List elements = container.getChildren(ELEMENT_VALIDATIONEXTERNALTYPE);
for (int i = 0; i < elements.size(); i++) {
Element element = (Element) elements.get(i);
MValidationExternalType type = buildValidationExternalType();
setAttributes(type, element);
parent.addChild(MetadataType.VALIDATION_EXTERNAL_TYPE, type);
}
}
private void processSearchHelp(MResource parent, Element container) {
List searchhelps = container.getChildren(ELEMENT_SEARCHHELP);
for (int i = 0; i < searchhelps.size(); i++) {
Element element = (Element) searchhelps.get(i);
MSearchHelp searchhelp = buildSearchHelp();
setAttributes(searchhelp, element);
parent.addChild(MetadataType.SEARCH_HELP, searchhelp);
}
}
private void processObject(MResource parent, Element container) {
List objects = container.getChildren(ELEMENT_OBJECT);
for (int i = 0; i < objects.size(); i++) {
Element element = (Element) objects.get(i);
MObject obj = buildObject();
setAttributes(obj, element);
parent.addChild(MetadataType.OBJECT, obj);
}
}
private void processClass(MResource parent, Element container) {
List classes = container.getChildren(ELEMENT_CLASS);
for (int i = 0; i < classes.size(); i++) {
Element element = (Element) classes.get(i);
MClass clazz = buildClass();
init(clazz);
setAttributes(clazz, element);
parent.addChild(MetadataType.CLASS, clazz);
Element child;
child = element.getChild(CONTAINER_TABLE);
if (child != null) {
processTable(clazz, child);
}
child = element.getChild(CONTAINER_UPDATE);
if (child != null) {
processUpdate(clazz, child);
}
}
}
private void processTable(MClass parent, Element container) {
List tables = container.getChildren(ELEMENT_TABLE);
for (int i = 0; i < tables.size(); i++) {
Element element = (Element) tables.get(i);
MTable table = buildTable();
setAttributes(table, element);
parent.addChild(MetadataType.TABLE, table);
}
}
private void processUpdate(MClass parent, Element container) {
List updates = container.getChildren(ELEMENT_UPDATE);
for (int i = 0; i < updates.size(); i++) {
Element element = (Element) updates.get(i);
MUpdate update = buildUpdate();
init(update);
setAttributes(update, element);
parent.addChild(MetadataType.UPDATE, update);
Element child = element.getChild(CONTAINER_UPDATE_TYPE);
if (child != null) {
processUpdateType(update, child);
}
}
}
private void processUpdateType(MUpdate parent, Element container) {
List updateFields = container.getChildren(ELEMENT_UPDATETYPE);
for (int i = 0; i < updateFields.size(); i++) {
Element element = (Element) updateFields.get(i);
MUpdateType updateType = buildUpdateType();
parent.addChild(MetadataType.UPDATE_TYPE, updateType);
setAttributes(updateType, element);
}
}
private void processForeignKey(MSystem system, Element container) {
List fkeys = container.getChildren("ForeignKey");
for (int i = 0; i < fkeys.size(); i++) {
Element element = (Element) fkeys.get(i);
MForeignKey foreignKey = buildForeignKey();
setAttributes(foreignKey, element);
system.addChild(MetadataType.FOREIGNKEYS, foreignKey);
}
}
public static final String CONTAINER_PREFIX = "METADATA-";
public static final String CONTAINER_ROOT = "RETS";
public static final String CONTAINER_METADATA = "METADATA";
public static final String CONTAINER_SYSTEM = "METADATA-SYSTEM";
public static final String CONTAINER_RESOURCE = "METADATA-RESOURCE";
public static final String CONTAINER_FOREIGNKEY = "METADATA-FOREIGN_KEYS";
public static final String CONTAINER_CLASS = "METADATA-CLASS";
public static final String CONTAINER_TABLE = "METADATA-TABLE";
public static final String CONTAINER_UPDATE = "METADATA-UPDATE";
public static final String CONTAINER_UPDATE_TYPE = "METADATA-UPDATE_TYPE";
public static final String CONTAINER_OBJECT = "METADATA-OBJECT";
public static final String CONTAINER_SEARCH_HELP = "METADATA-SEARCH_HELP";
public static final String CONTAINER_EDITMASK = "METADATA-EDITMASK";
public static final String CONTAINER_UPDATEHELP = "METADATA-UPDATE_HELP";
public static final String CONTAINER_LOOKUP = "METADATA-LOOKUP";
public static final String CONTAINER_LOOKUPTYPE = "METADATA-LOOKUP_TYPE";
public static final String CONTAINER_VALIDATIONLOOKUP = "METADATA-VALIDATION_LOOKUP";
public static final String CONTAINER_VALIDATIONLOOKUPTYPE = "METADATA-VALIDATION_LOOKUP_TYPE";
public static final String CONTAINER_VALIDATIONEXPRESSION = "METADATA-VALIDATION_EXPRESSION";
public static final String CONTAINER_VALIDATIONEXTERNAL = "METADATA-VALIDATION_EXTERNAL";
public static final String CONTAINER_VALIDATIONEXTERNALTYPE = "METADATA-VALIDATION_EXTERNAL_TYPE";
public static final Map sContainer2Type = new HashMap();
static {
for (int i = 0; i < MetadataType.values().length; i++) {
MetadataType type = MetadataType.values()[i];
sContainer2Type.put(CONTAINER_PREFIX + type.name(), type);
}
/* you have got to be kidding me. The spec (compact) says
METADATA-FOREIGNKEYS and that's the request type but the DTD says
METADATA-FOREIGN_KEY.
I think I'm going to be sick.
*/
sContainer2Type.remove(CONTAINER_PREFIX + MetadataType.FOREIGNKEYS.name());
sContainer2Type.put(CONTAINER_FOREIGNKEY, MetadataType.FOREIGNKEYS);
}
}

20
src/main/java/com/ossez/reoc/rets/common/metadata/MetaCollector.java Normal file
View File

@ -0,0 +1,20 @@
/*
* cart: CRT's Awesome RETS Tool
*
* Author: David Terrell
* Copyright (c) 2003, The National Association of REALTORS
* Distributed under a BSD-style license. See LICENSE.TXT for details.
*/
package com.ossez.reoc.rets.common.metadata;
import java.io.Serializable;
/** Interface for Metadata objects to collect their children. */
public interface MetaCollector extends Serializable {
/**
* @param path path to the parent object.
*/
public MetaObject[] getMetadata(MetadataType type, String path) throws MetadataException;
public MetaObject[] getMetadataRecursive(MetadataType type, String path) throws MetadataException;
}

366
src/main/java/com/ossez/reoc/rets/common/metadata/MetaObject.java Normal file
View File

@ -0,0 +1,366 @@
package com.ossez.reoc.rets.common.metadata;
import java.io.Serializable;
import java.util.Collection;
import java.util.Collections;
//import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
import java.util.Set;
import com.ossez.reoc.rets.common.util.CaseInsensitiveTreeMap;
import org.apache.commons.lang.builder.EqualsBuilder;
import org.apache.commons.lang.builder.HashCodeBuilder;
import org.apache.commons.lang.builder.ToStringBuilder;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import com.ossez.reoc.rets.common.metadata.attrib.AttrAlphanum;
import com.ossez.reoc.rets.common.metadata.attrib.AttrBoolean;
import com.ossez.reoc.rets.common.metadata.attrib.AttrDate;
import com.ossez.reoc.rets.common.metadata.attrib.AttrNumeric;
import com.ossez.reoc.rets.common.metadata.attrib.AttrNumericPositive;
import com.ossez.reoc.rets.common.metadata.attrib.AttrPlaintext;
import com.ossez.reoc.rets.common.metadata.attrib.AttrText;
import com.ossez.reoc.rets.common.metadata.attrib.AttrVersion;
public abstract class MetaObject implements Serializable {
private static final Log LOG = LogFactory.getLog(MetaObject.class);
/** a standard parser used by different child types */
protected static final AttrType sAlphanum = new AttrAlphanum(0, 0);
protected static final AttrType sAlphanum64 = new AttrAlphanum(1, 64);
protected static final AttrType sAlphanum32 = new AttrAlphanum(1, 32);
protected static final AttrType sAlphanum24 = new AttrAlphanum(1, 24);
protected static final AttrType sAlphanum10 = new AttrAlphanum(1, 10);
protected static final AttrType sPlaintext = new AttrPlaintext(0, 0);
protected static final AttrType sPlaintext1024 = new AttrPlaintext(1, 1024);
protected static final AttrType sPlaintext512 = new AttrPlaintext(1, 512);
protected static final AttrType sPlaintext128 = new AttrPlaintext(1, 128);
protected static final AttrType sPlaintext64 = new AttrPlaintext(1, 64);
protected static final AttrType sPlaintext32 = new AttrPlaintext(1, 32);
protected static final AttrType sText = new AttrText(0, 0);
protected static final AttrType sText1024 = new AttrText(1, 1024);
protected static final AttrType sText512 = new AttrText(1, 512);
protected static final AttrType sText256 = new AttrText(1, 256);
protected static final AttrType sText128 = new AttrText(1, 128);
protected static final AttrType sText64 = new AttrText(1, 64);
protected static final AttrType sText32 = new AttrText(1, 32);
protected static final AttrType sAttrBoolean = new AttrBoolean();
protected static final AttrType sAttrDate = new AttrDate();
protected static final AttrType sAttrNumeric = new AttrNumeric();
protected static final AttrType sAttrNumericPositive = new AttrNumericPositive();
protected static final AttrType sAttrVersion = new AttrVersion();
protected static final AttrType sAttrMetadataEntryId = sAlphanum32;
protected static final MetadataType[] sNoChildren = new MetadataType[0];
protected static final AttrType retsid = sAlphanum32;
protected static final AttrType retsname = sAlphanum64;
public static final boolean STRICT_PARSING = true;
public static final boolean LOOSE_PARSING = false;
public static final boolean DEFAULT_PARSING = LOOSE_PARSING;
/** the metdata path to this object */
protected String path;
/** map of child type to map of child id to child object */
protected Map childTypes;
/** map of attribute name to attribute object (as parsed by attrtype) */
protected Map attributes;
/** map of attribute name to AttrType parser */
protected Map attrTypes;
private static Map<CacheKey,Map> sAttributeMapCache = new HashMap<CacheKey,Map>();
private MetaCollector mCollector;
private boolean strict;
public MetaObject(boolean strictParsing) {
this.strict = strictParsing;
if (strictParsing) {
this.attributes = new HashMap();
} else {
this.attributes = new CaseInsensitiveTreeMap();
}
this.attrTypes = this.getAttributeMap(strictParsing);
MetadataType[] types = getChildTypes();
this.childTypes = new HashMap();
for (int i = 0; i < types.length; i++) {
this.childTypes.put(types[i], null);
}
}
private Map getAttributeMap(boolean strictParsing) {
synchronized (sAttributeMapCache) {
Map<CacheKey,Map> map = sAttributeMapCache.get(new CacheKey(this, strictParsing));
if (map == null) {
if (strictParsing) {
map = new HashMap();
} else {
map = new CaseInsensitiveTreeMap();
}
addAttributesToMap(map);
// Let's make sure no one mucks with the map later
map = Collections.unmodifiableMap(map);
sAttributeMapCache.put(new CacheKey(this, strictParsing), map);
if (LOG.isDebugEnabled()) {
LOG.debug("Adding to attribute cache: " + this.getClass().getName() + ", " + strictParsing);
}
}
return map;
}
}
public static void clearAttributeMapCache() {
synchronized (sAttributeMapCache) {
sAttributeMapCache.clear();
}
}
public Collection getChildren(MetadataType type) {
if (!this.childTypes.containsKey(type)) {
// throw new IllegalArgumentException?
return null;
}
Object o = this.childTypes.get(type);
if (o == null) {
if (!fetchChildren(type)) {
return Collections.EMPTY_SET;
}
o = this.childTypes.get(type);
}
if (o instanceof Map) {
Map m = (Map) o;
return m.values();
}
return (Collection) o;
}
private boolean fetchChildren(MetadataType type) {
this.childTypes.put(type, new HashMap());
try {
MetaObject[] children = null;
if (this.mCollector != null) {
children = this.mCollector.getMetadata(type, getPath());
}
if (children == null) {
return false;
}
for (int i = 0; i < children.length; i++) {
MetaObject child = children[i];
addChild(type, child);
}
} catch (MetadataException e) {
LOG.error(toString() + " unable to fetch " + type.name() + " children");
return false;
}
return true;
}
public MetaObject getChild(MetadataType type, String id) {
if (id == null) {
return null;
}
try {
if (this.childTypes.get(type) == null && this.mCollector != null) {
if (!fetchChildren(type)) {
return null;
}
}
Map m = (Map) this.childTypes.get(type);
if (m == null) {
return null;
}
return (MetaObject) m.get(id);
} catch (ClassCastException e) {
return null;
}
}
public Object getAttribute(String key) {
return this.attributes.get(key);
}
public Set getKnownAttributes() {
return this.attrTypes.keySet();
}
public String getAttributeAsString(String key) {
Object value = this.attributes.get(key);
if (value == null) {
return null;
}
if (this.attrTypes.containsKey(key)) {
AttrType type = (AttrType) this.attrTypes.get(key);
return type.render(value);
}
return value.toString();
}
protected Object getTypedAttribute(String key, Class type) {
AttrType atype = (AttrType) this.attrTypes.get(key);
if (atype == null) {
return null;
}
if (atype.getType() == type) {
return this.attributes.get(key);
}
LOG.warn("type mismatch, expected " + type.getName() + " but" + " got " + atype.getType().getName());
return null;
}
public String getDateAttribute(String key) {
return (String) getTypedAttribute(key, String.class);
}
public String getStringAttribute(String key) {
return (String) getTypedAttribute(key, String.class);
}
public int getIntAttribute(String key) {
Integer i = (Integer) getTypedAttribute(key, Integer.class);
if (i == null) {
return 0;
}
return i.intValue();
}
public boolean getBooleanAttribute(String key) {
Boolean b = (Boolean) getTypedAttribute(key, Boolean.class);
if (b == null) {
return false;
}
return b.booleanValue();
}
public void setAttribute(String key, String value) {
if (value == null) {
// LOG.warning()
return;
}
if (this.attrTypes.containsKey(key)) {
AttrType type = (AttrType) this.attrTypes.get(key);
try {
this.attributes.put(key, type.parse(value,this.strict));
} catch (MetaParseException e) {
LOG.warn(toString() + " couldn't parse attribute " + key + ", value " + value + ": " + e.getMessage());
}
} else {
this.attributes.put(key, value);
LOG.warn("Unknown key (" + toString() + "): " + key);
}
}
public void addChild(MetadataType type, MetaObject child) {
if (this.childTypes.containsKey(type)) {
Object obj = this.childTypes.get(type);
Map map;
if (obj == null) {
map = new HashMap();
this.childTypes.put(type, map);
} else {
map = (Map) obj;
}
if (child == null) {
return;
}
String id = child.getId();
child.setPath(this.getPath());
child.setCollector(this.mCollector);
if (id != null) {
map.put(id, child);
}
return;
}
}
public String getId() {
String idAttr = getIdAttr();
if (idAttr == null) {
/** cheap hack so everything's a damn map */
return Integer.toString(hashCode());
}
return getAttributeAsString(idAttr);
}
public String getPath() {
return this.path;
}
protected void setPath(String parent) {
if (parent == null || parent.equals("")) {
this.path = getId();
} else {
this.path = parent + ":" + getId();
}
}
@Override
public String toString() {
ToStringBuilder tsb = new ToStringBuilder(this);
Iterator iter = getKnownAttributes().iterator();
while (iter.hasNext()) {
String key = (String) iter.next();
tsb.append(key, getAttributeAsString(key));
}
return tsb.toString();
}
public void setCollector(MetaCollector c) {
this.mCollector = c;
Iterator iterator = this.childTypes.keySet().iterator();
while (iterator.hasNext()) {
MetadataType type = (MetadataType) iterator.next();
Map map = (Map) this.childTypes.get(type);
if (map == null) {
continue;
}
Collection children = map.values();
for (Iterator iter = children.iterator(); iter.hasNext();) {
MetaObject object = (MetaObject) iter.next();
object.setCollector(c);
}
}
}
public abstract MetadataType[] getChildTypes();
protected abstract String getIdAttr();
/**
* Adds attributes to an attribute map. This is called by the MetaObject
* constructor to initialize a map of atributes. This map may be cached,
* so this method may not be called for every object construction.
*
* @param attributeMap Map to add attributes to
*/
protected abstract void addAttributesToMap(Map attributeMap);
}
class CacheKey {
private Class mClass;
private boolean strictParsing;
public CacheKey(MetaObject metaObject, boolean strictParsing) {
this.mClass = metaObject.getClass();
this.strictParsing = strictParsing;
}
@Override
public boolean equals(Object obj) {
if (!(obj instanceof CacheKey)) {
return false;
}
CacheKey rhs = (CacheKey) obj;
return new EqualsBuilder().append(this.mClass, rhs.mClass).append(this.strictParsing, rhs.strictParsing).isEquals();
}
@Override
public int hashCode() {
return new HashCodeBuilder().append(this.mClass).append(this.strictParsing).toHashCode();
}
}

26
src/main/java/com/ossez/reoc/rets/common/metadata/MetaParseException.java Normal file
View File

@ -0,0 +1,26 @@
/*
* cart: CRT's Awesome RETS Tool
*
* Author: David Terrell
* Copyright (c) 2003, The National Association of REALTORS
* Distributed under a BSD-style license. See LICENSE.TXT for details.
*/
package com.ossez.reoc.rets.common.metadata;
public class MetaParseException extends MetadataException {
public MetaParseException() {
super();
}
public MetaParseException(String msg) {
super(msg);
}
public MetaParseException(Throwable cause) {
super(cause);
}
public MetaParseException(String msg, Throwable cause) {
super(msg, cause);
}
}

154
src/main/java/com/ossez/reoc/rets/common/metadata/Metadata.java Normal file
View File

@ -0,0 +1,154 @@
package com.ossez.reoc.rets.common.metadata;
import java.io.Serializable;
import com.ossez.reoc.rets.common.metadata.types.MSystem;
import com.ossez.reoc.rets.common.metadata.types.MResource;
import com.ossez.reoc.rets.common.metadata.types.MForeignKey;
import com.ossez.reoc.rets.common.metadata.types.MClass;
import com.ossez.reoc.rets.common.metadata.types.MTable;
import com.ossez.reoc.rets.common.metadata.types.MUpdate;
import com.ossez.reoc.rets.common.metadata.types.MUpdateType;
import com.ossez.reoc.rets.common.metadata.types.MObject;
import com.ossez.reoc.rets.common.metadata.types.MValidationExternal;
import com.ossez.reoc.rets.common.metadata.types.MValidationLookup;
import com.ossez.reoc.rets.common.metadata.types.MLookup;
import com.ossez.reoc.rets.common.metadata.types.MSearchHelp;
public class Metadata implements Serializable {
protected MSystem system;
public Metadata(MetaCollector collector) throws MetadataException {
MetaObject[] sys = collector.getMetadata(MetadataType.SYSTEM, null);
if (sys != null && sys.length == 1) {
try {
this.system = (MSystem) sys[0];
} catch (ClassCastException e) {
throw new MetadataException(e);
}
this.system.setCollector(collector);
}
}
public Metadata(MSystem system) {
this.system = system;
}
public MSystem getSystem() {
return this.system;
}
public MResource getResource(String resourceId) {
return this.system.getMResource(resourceId);
}
public MForeignKey getForeignKey(String foreignKeyId) {
return this.system.getMForeignKey(foreignKeyId);
}
public MClass getMClass(String resourceId, String className) {
MResource resource = getResource(resourceId);
if (resource == null) {
return null;
}
return resource.getMClass(className);
}
public MTable getTable(String resourceId, String className, String systemName) {
MClass clazz = getMClass(resourceId, className);
if (clazz == null) {
return null;
}
return clazz.getMTable(systemName);
}
public MUpdate getUpdate(String resourceId, String className, String updateName) {
MClass clazz = getMClass(resourceId, className);
if (clazz == null) {
return null;
}
return clazz.getMUpdate(updateName);
}
public MUpdateType getUpdateType(String resourceId, String className, String updateName, String systemName) {
MUpdate update = getUpdate(resourceId, className, updateName);
if (update == null) {
return null;
}
return update.getMUpdateType(systemName);
}
public MObject getObject(String resourceId, String objectType) {
MResource resource = getResource(resourceId);
if (resource == null) {
return null;
}
return resource.getMObject(objectType);
}
public MLookup getLookup(String resourceId, String lookupName) {
MResource resource = getResource(resourceId);
if (resource == null) {
return null;
}
return resource.getMLookup(lookupName);
}
public MSearchHelp getSearchHelp(String resourceId, String searchHelpId) {
MResource resource = getResource(resourceId);
if (resource == null) {
return null;
}
return resource.getMSearchHelp(searchHelpId);
}
public MValidationExternal getValidationExternal(String resourceId, String validationExternalName) {
MResource resource = getResource(resourceId);
if (resource == null) {
return null;
}
return resource.getMValidationExternal(validationExternalName);
}
public MValidationLookup getValidationLookup(String resourceId, String validationLookupName) {
MResource resource = getResource(resourceId);
if (resource == null) {
return null;
}
return resource.getMValidationLookup(validationLookupName);
}
private String getResourceId(MetaObject obj) {
String path = obj.getPath();
int index = path.indexOf(':');
if (index == -1) {
return null;
}
String resource = path.substring(0, index);
return resource;
}
public MResource getResource(MTable field) {
String resource = getResourceId(field);
return getResource(resource);
}
public MLookup getLookup(MTable field) {
String resource = getResourceId(field);
return getLookup(resource, field.getLookupName());
}
public MSearchHelp getSearchHelp(MTable field) {
String searchHelpID = field.getSearchHelpID();
if (searchHelpID == null) {
return null;
}
String resource = getResourceId(field);
return getSearchHelp(resource, searchHelpID);
}
public MResource getResource(MClass clazz) {
return getResource(getResourceId(clazz));
}
}

203
src/main/java/com/ossez/reoc/rets/common/metadata/MetadataBuilder.java Normal file
View File

@ -0,0 +1,203 @@
/*
* cart: CRT's Awesome RETS Tool
*
* Author: David Terrell
* Copyright (c) 2003, The National Association of REALTORS
* Distributed under a BSD-style license. See LICENSE.TXT for details.
*/
package com.ossez.reoc.rets.common.metadata;
import com.ossez.reoc.rets.common.metadata.types.MClass;
import com.ossez.reoc.rets.common.metadata.types.MEditMask;
import com.ossez.reoc.rets.common.metadata.types.MForeignKey;
import com.ossez.reoc.rets.common.metadata.types.MLookup;
import com.ossez.reoc.rets.common.metadata.types.MLookupType;
import com.ossez.reoc.rets.common.metadata.types.MObject;
import com.ossez.reoc.rets.common.metadata.types.MResource;
import com.ossez.reoc.rets.common.metadata.types.MSearchHelp;
import com.ossez.reoc.rets.common.metadata.types.MSystem;
import com.ossez.reoc.rets.common.metadata.types.MTable;
import com.ossez.reoc.rets.common.metadata.types.MUpdate;
import com.ossez.reoc.rets.common.metadata.types.MUpdateHelp;
import com.ossez.reoc.rets.common.metadata.types.MUpdateType;
import com.ossez.reoc.rets.common.metadata.types.MValidationExpression;
import com.ossez.reoc.rets.common.metadata.types.MValidationExternal;
import com.ossez.reoc.rets.common.metadata.types.MValidationExternalType;
import com.ossez.reoc.rets.common.metadata.types.MValidationLookup;
import com.ossez.reoc.rets.common.metadata.types.MValidationLookupType;
public abstract class MetadataBuilder {
protected MetadataBuilder() {
this.mStrict = false;
}
public boolean isStrict() {
return this.mStrict;
}
public void setStrict(boolean strict) {
this.mStrict = strict;
}
protected Metadata finish(MSystem system) {
return new Metadata(system);
}
protected static void setAttribute(MetaObject obj, String key, String value) {
obj.setAttribute(key, value);
}
protected MSystem buildSystem() {
MSystem system = new MSystem(this.mStrict);
return system;
}
protected MResource buildResource() {
MResource resource = new MResource(this.mStrict);
return resource;
}
protected MForeignKey buildForeignKey() {
MForeignKey key = new MForeignKey(this.mStrict);
return key;
}
protected MClass buildClass() {
MClass clazz = new MClass(this.mStrict);
return clazz;
}
protected MTable buildTable() {
MTable table = new MTable(this.mStrict);
return table;
}
protected MUpdate buildUpdate() {
MUpdate update = new MUpdate(this.mStrict);
return update;
}
protected MUpdateType buildUpdateType() {
MUpdateType updatetype = new MUpdateType(this.mStrict);
return updatetype;
}
protected MObject buildObject() {
MObject obj = new MObject(this.mStrict);
return obj;
}
protected MSearchHelp buildSearchHelp() {
MSearchHelp help = new MSearchHelp(this.mStrict);
return help;
}
protected MEditMask buildEditMask() {
MEditMask mask = new MEditMask(this.mStrict);
return mask;
}
protected MLookup buildLookup() {
MLookup lookup = new MLookup(this.mStrict);
return lookup;
}
protected MLookupType buildLookupType() {
MLookupType type = new MLookupType(this.mStrict);
return type;
}
protected MUpdateHelp buildUpdateHelp() {
MUpdateHelp help = new MUpdateHelp(this.mStrict);
return help;
}
protected MValidationLookup buildValidationLookup() {
MValidationLookup lookup = new MValidationLookup(this.mStrict);
return lookup;
}
protected MValidationExternalType buildValidationExternalType() {
MValidationExternalType type = new MValidationExternalType(this.mStrict);
return type;
}
protected MValidationExpression buildValidationExpression() {
MValidationExpression expression = new MValidationExpression(this.mStrict);
return expression;
}
protected MValidationExternal buildValidationExternal() {
MValidationExternal external = new MValidationExternal(this.mStrict);
return external;
}
protected MValidationLookupType buildValidationLookupType() {
MValidationLookupType lookupType = new MValidationLookupType(this.mStrict);
return lookupType;
}
public abstract Metadata doBuild(Object src) throws MetadataException;
public abstract MetaObject[] parse(Object src) throws MetadataException;
protected MetaObject newType(MetadataType type) {
if (type == MetadataType.SYSTEM) {
return buildSystem();
}
if (type == MetadataType.RESOURCE) {
return buildResource();
}
if (type == MetadataType.FOREIGNKEYS) {
return buildForeignKey();
}
if (type == MetadataType.CLASS) {
return buildClass();
}
if (type == MetadataType.TABLE) {
return buildTable();
}
if (type == MetadataType.UPDATE) {
return buildUpdate();
}
if (type == MetadataType.UPDATE_TYPE) {
return buildUpdateType();
}
if (type == MetadataType.OBJECT) {
return buildObject();
}
if (type == MetadataType.SEARCH_HELP) {
return buildSearchHelp();
}
if (type == MetadataType.EDITMASK) {
return buildEditMask();
}
if (type == MetadataType.UPDATE_HELP) {
return buildUpdateHelp();
}
if (type == MetadataType.LOOKUP) {
return buildLookup();
}
if (type == MetadataType.LOOKUP_TYPE) {
return buildLookupType();
}
if (type == MetadataType.VALIDATION_LOOKUP) {
return buildValidationLookup();
}
if (type == MetadataType.VALIDATION_LOOKUP_TYPE) {
return buildValidationLookupType();
}
if (type == MetadataType.VALIDATION_EXTERNAL) {
return buildValidationExternal();
}
if (type == MetadataType.VALIDATION_EXTERNAL_TYPE) {
return buildValidationExternalType();
}
if (type == MetadataType.VALIDATION_EXPRESSION) {
return buildValidationExpression();
}
throw new RuntimeException("No metadata type class found for " + type.name());
}
private boolean mStrict;
}

31
src/main/java/com/ossez/reoc/rets/common/metadata/MetadataElement.java Normal file
View File

@ -0,0 +1,31 @@
package com.ossez.reoc.rets.common.metadata;
public enum MetadataElement {
SYSTEM("System"),// might need to provide enumeration for different versions 1.5 vs 1.7
RESOURCE("Resource"),
FOREIGNKEY("ForeignKey"),
CLASS("Class"),
TABLE("Field"),
UPDATE("UpdateType"),
UPDATETYPE("UpdateField"),
OBJECT("Object"),
SEARCHHELP("SearchHelp"),
EDITMASK("EditMask"),
UPDATEHELP("UpdateHelp"),
LOOKUP("Lookup"),
LOOKUPTYPE("LookupType"),
VALIDATIONLOOKUP("ValidationLookup"),
VALIDATIONLOOKUPTYPE("ValidationLookupType"),
VALIDATIONEXPRESSION("ValidationExpression"),
VALIDATIONEXTERNAL("ValidationExternalType"),
VALIDATIONEXTERNALTYPE("ValidationExternal");
private final String elementName;
MetadataElement(String elementName){
this.elementName = elementName;
}
public String elementName(){ return this.elementName;}
}

28
src/main/java/com/ossez/reoc/rets/common/metadata/MetadataException.java Normal file
View File

@ -0,0 +1,28 @@
/*
* cart: CRT's Awesome RETS Tool
*
* Author: David Terrell
* Copyright (c) 2003, The National Association of REALTORS
* Distributed under a BSD-style license. See LICENSE.TXT for details.
*/
package com.ossez.reoc.rets.common.metadata;
import org.apache.commons.lang.exception.NestableException;
public class MetadataException extends NestableException {
public MetadataException() {
super();
}
public MetadataException(String msg) {
super(msg);
}
public MetadataException(Throwable cause) {
super(cause);
}
public MetadataException(String msg, Throwable cause) {
super(msg, cause);
}
}

30
src/main/java/com/ossez/reoc/rets/common/metadata/MetadataType.java Normal file
View File

@ -0,0 +1,30 @@
/*
* cart: CRT's Awesome RETS Tool
*
* Author: David Terrell
* Copyright (c) 2003, The National Association of REALTORS
* Distributed under a BSD-style license. See LICENSE.TXT for details.
*/
package com.ossez.reoc.rets.common.metadata;
public enum MetadataType {
EDITMASK,
FOREIGNKEYS,
RESOURCE,
LOOKUP,
LOOKUP_TYPE,
OBJECT,
SEARCH_HELP,
SYSTEM,
TABLE,
UPDATE,
UPDATE_HELP,
UPDATE_TYPE,
VALIDATION_EXPRESSION,
VALIDATION_EXTERNAL,
VALIDATION_EXTERNAL_TYPE,
VALIDATION_LOOKUP,
VALIDATION_LOOKUP_TYPE,
CLASS;
}

49
src/main/java/com/ossez/reoc/rets/common/metadata/attrib/AttrAbstractText.java Normal file
View File

@ -0,0 +1,49 @@
/*
* cart: CRT's Awesome RETS Tool
*
* Author: David Terrell
* Copyright (c) 2003, The National Association of REALTORS
* Distributed under a BSD-style license. See LICENSE.TXT for details.
*/
package com.ossez.reoc.rets.common.metadata.attrib;
import com.ossez.reoc.rets.common.metadata.AttrType;
import com.ossez.reoc.rets.common.metadata.MetaParseException;
public abstract class AttrAbstractText implements AttrType<String> {
protected int min;
protected int max;
public AttrAbstractText(int min, int max) {
this.min = min;
this.max = max;
}
public String parse(String value, boolean strict) throws MetaParseException {
if( !strict )
return value;
int l = value.length();
if (this.min != 0 && l < this.min) {
throw new MetaParseException("Value too short (min " + this.min + "): " + l);
}
if (this.max != 0 && l > this.max) {
throw new MetaParseException("Value too long (max " + this.max + "): " + l);
}
checkContent(value);
return value;
}
public Class<String> getType() {
return String.class;
}
public String render(String value) {
return value;
}
protected abstract void checkContent(String value) throws MetaParseException;
}

31
src/main/java/com/ossez/reoc/rets/common/metadata/attrib/AttrAlphanum.java Normal file
View File

@ -0,0 +1,31 @@
/*
* cart: CRT's Awesome RETS Tool
*
* Author: David Terrell
* Copyright (c) 2003, The National Association of REALTORS
* Distributed under a BSD-style license. See LICENSE.TXT for details.
*/
package com.ossez.reoc.rets.common.metadata.attrib;
import com.ossez.reoc.rets.common.metadata.MetaParseException;
public class AttrAlphanum extends AttrAbstractText {
public AttrAlphanum(int min, int max) {
super(min, max);
}
@Override
protected void checkContent(String value) throws MetaParseException {
char[] chars = value.toCharArray();
for (int i = 0; i < chars.length; i++) {
char c = chars[i];
if (!Character.isLetterOrDigit(c)) {
// illegal but exist in CRT metadata
if ("_- ".indexOf(c) == -1) {
throw new MetaParseException("Invalid Alphanum character at position " + i + ": " + c);
}
}
}
}
}

54
src/main/java/com/ossez/reoc/rets/common/metadata/attrib/AttrBoolean.java Normal file
View File

@ -0,0 +1,54 @@
/*
* cart: CRT's Awesome RETS Tool
*
* Author: David Terrell
* Copyright (c) 2003, The National Association of REALTORS
* Distributed under a BSD-style license. See LICENSE.TXT for details.
*/
package com.ossez.reoc.rets.common.metadata.attrib;
import com.ossez.reoc.rets.common.metadata.AttrType;
import com.ossez.reoc.rets.common.metadata.MetaParseException;
public class AttrBoolean implements AttrType<Boolean> {
public Boolean parse(String value, boolean strict) throws MetaParseException {
if (value.equals("1")) {
return Boolean.TRUE;
}
if (value.equals("0")) {
return Boolean.FALSE;
}
if (value.equalsIgnoreCase("true")) {
return Boolean.TRUE;
}
if (value.equalsIgnoreCase("false")) {
return Boolean.FALSE;
}
if (value.equalsIgnoreCase("Y")) {
return Boolean.TRUE;
}
if (value.equalsIgnoreCase("N")) {
return Boolean.FALSE;
}
if (value.equals("")) {
return Boolean.FALSE;
}
if( strict )
throw new MetaParseException("Invalid boolean value: " + value);
return false;
}
public String render(Boolean value) {
if( value.booleanValue() ) return "1";
return "0";
}
public Class<Boolean> getType() {
return Boolean.class;
}
}

71
src/main/java/com/ossez/reoc/rets/common/metadata/attrib/AttrDate.java Normal file
View File

@ -0,0 +1,71 @@
/*
* cart: CRT's Awesome RETS Tool
*
* Author: David Terrell
* Copyright (c) 2003, The National Association of REALTORS
* Distributed under a BSD-style license. See LICENSE.TXT for details.
*
*
* Vangulo Changed:
* gives ability to handle dates in this format
* 2011-06-01T18:06:58
* should find a more elegant way
*/
package com.ossez.reoc.rets.common.metadata.attrib;
//import java.text.DateFormat;
//import java.text.ParseException;
//import java.text.SimpleDateFormat;
//import java.util.Date;
import com.ossez.reoc.rets.common.metadata.AttrType;
import com.ossez.reoc.rets.common.metadata.MetaParseException;
/**
* Converted this class to return a String instead of a
* Date object which allows for more flexiblity since
* Many Rets Servers format their dates differently
*
* @author vangulo
*
*/
public class AttrDate implements AttrType<String> {
// need date attribute to be flexible since different MLS's have
// different formats for dates
public String parse(String value, boolean strict) throws MetaParseException {
return value;
// Date d;
// try {
// d = this.df.parse(value);
// } catch (ParseException e) {
// if( strict )
// throw new MetaParseException(e);
// try {
// value = value.replaceAll("[A-Za-z]", " ");
// d = this.df1.parse(value);
// } catch (ParseException e1) {
// //e1.printStackTrace();
// return value;
// }
// return d;
// }
// return d;
}
public String render(String value) {
return value;
//Date date = value;
//return this.df.format(date);
}
public Class<String> getType() {
return String.class;
}
//private DateFormat df = new SimpleDateFormat("E, d MMM yyyy HH:mm:ss z");
//2011-06-01T18:06:58
//private DateFormat df1 = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
//Tuesday, 22-Dec-2009 21:03:18 GMT
//private DateFormat df2 = new SimpleDateFormat("E, dd-MMM-yyyy HH:mm:ss z");
}

31
src/main/java/com/ossez/reoc/rets/common/metadata/attrib/AttrEnum.java Normal file
View File

@ -0,0 +1,31 @@
/*
* cart: CRT's Awesome RETS Tool
*
* Author: David Terrell
* Copyright (c) 2003, The National Association of REALTORS
* Distributed under a BSD-style license. See LICENSE.TXT for details.
*/
package com.ossez.reoc.rets.common.metadata.attrib;
import java.util.HashMap;
import java.util.Map;
import java.util.Collections;
import com.ossez.reoc.rets.common.metadata.MetaParseException;
public class AttrEnum extends AttrAbstractText {
public AttrEnum(String[] values) {
super(0, 0);
this.map = new HashMap<String,String>();
for (String value : values) this.map.put(value, value);
this.map = Collections.unmodifiableMap(this.map);
}
@Override
protected void checkContent(String value) throws MetaParseException {
if( !this.map.containsKey(value) )
throw new MetaParseException("Invalid key: " + value);
}
private Map<String,String> map;
}

31
src/main/java/com/ossez/reoc/rets/common/metadata/attrib/AttrGenericText.java Normal file
View File

@ -0,0 +1,31 @@
/*
* cart: CRT's Awesome RETS Tool
*
* Author: David Terrell
* Copyright (c) 2003, The National Association of REALTORS
* Distributed under a BSD-style license. See LICENSE.TXT for details.
*/
package com.ossez.reoc.rets.common.metadata.attrib;
import com.ossez.reoc.rets.common.metadata.MetaParseException;
public class AttrGenericText extends AttrAbstractText {
private String mChars;
public AttrGenericText(int min, int max, String chars) {
super(min, max);
this.mChars = chars;
}
@Override
protected void checkContent(String value) throws MetaParseException {
char[] chars = value.toCharArray();
for (int i = 0; i < chars.length; i++) {
char c = chars[i];
if (this.mChars.indexOf(c) == -1) {
throw new MetaParseException("Invalid char (" + c + ") at position " + i);
}
}
}
}

Some files were not shown because too many files have changed in this diff Show More