Merge pull request #4 from apache/develop

Merging latest Apache NiFi develop branch
This commit is contained in:
Brian Ghigiarelli 2015-05-21 08:42:32 -04:00
commit ce16aab6c5
165 changed files with 3112 additions and 622 deletions

View File

@ -24,9 +24,11 @@ Apache NiFi is an easy to use, powerful, and reliable system to process and dist
## Getting Started
- Read through the [quickstart guide for development](http://nifi.incubator.apache.org/development/quickstart.html).
- Read through the [quickstart guide for development](http://nifi.incubator.apache.org/quickstart.html).
It will include information on getting a local copy of the source, give pointers on issue
tracking, and provide some warnings about common problems with development environments.
- For a more comprehensive guide to development and information about contributing to the project
read through the [NiFi Developer's Guide](http://nifi.incubator.apache.org/developer-guide.html).
- Optional: Build supporting modules. This should only be needed if the current 'nifi' module is in
the process of updating to a new version of either the 'nifi-parent' or 'nifi-nar-maven-plugin'
artifacts.

View File

@ -18,11 +18,11 @@
<parent>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-parent</artifactId>
<version>1.0.0-incubating-SNAPSHOT</version>
<version>1.0.0-incubating</version>
<relativePath />
</parent>
<artifactId>nifi-nar-maven-plugin</artifactId>
<version>1.0.1-incubating-SNAPSHOT</version>
<version>1.0.2-incubating-SNAPSHOT</version>
<packaging>maven-plugin</packaging>
<description>Apache NiFi Nar Plugin. It is currently a part of the Apache Incubator.</description>
<build>

View File

@ -23,7 +23,7 @@
</parent>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-parent</artifactId>
<version>1.0.0-incubating-SNAPSHOT</version>
<version>1.0.1-incubating-SNAPSHOT</version>
<packaging>pom</packaging>
<description>The nifi-parent enables each apache nifi project to ensure consistent approaches and DRY</description>
<url>http://nifi.incubator.apache.org</url>

View File

@ -129,7 +129,7 @@ module.exports = function (grunt) {
questions: [{
config: 'config.svn.username',
type: 'input',
message: 'Enter SVN username if different from current user:'
message: 'Enter SVN username (if different from current or configured user):'
}]
}
},
@ -160,9 +160,10 @@ module.exports = function (grunt) {
}, {
config: 'config.svn.password',
type: 'password',
message: 'SVN password:'
message: 'SVN password (if different from configured):'
}],
then: function () {
grunt.task.run('exec:add');
grunt.task.run('exec:commit');
}
}
@ -185,11 +186,11 @@ module.exports = function (grunt) {
command: function() {
var url = grunt.config('config.svn.url');
var username = grunt.config('config.svn.username');
if (username === '') {
return 'svn checkout ' + url + ' --trust-server-cert --non-interactive dist';
} else {
return 'svn checkout --username ' + username + ' ' + url + ' --trust-server-cert --non-interactive dist';
var command = 'svn checkout';
if (username !== '') {
command += (' --username ' + username);
}
return command + ' ' + url + ' --trust-server-cert --non-interactive dist';
},
stdout: true,
stderr: true
@ -206,17 +207,26 @@ module.exports = function (grunt) {
stdout: true,
stderr: true
},
add: {
cwd: 'dist',
command: 'svn add --force .',
stdout: true,
stderr: true
},
commit: {
cwd: 'dist',
command: function() {
var username = grunt.config('config.svn.username');
var password = grunt.config('config.svn.password');
var message = grunt.config('config.svn.commit.message');
if (username === '') {
return 'svn commit --password "' + password + '" -m "' + message + '" --trust-server-cert --non-interactive .';
} else {
return 'svn commit --username ' + username + ' --password "' + password + '" -m "' + message + '" --trust-server-cert --non-interactive .';
var command = 'svn commit';
if (username !== '') {
command += (' --username ' + username);
}
if (password !== '') {
command += (' --password ' + password);
}
return command + ' -m "' + message + '" --trust-server-cert --non-interactive .';
}
}
},
@ -251,6 +261,9 @@ module.exports = function (grunt) {
replacements: [{
from: /<div class="sub-title">.*<\/div>/g,
to: '<div class="sub-title">NiFi Rest Api</div>'
}, {
from: /<title>.*<\/title>/g,
to: '<title>NiFi Rest Api</title>'
}]
}
},

View File

@ -46,7 +46,8 @@ npm install
The site is built using [foundation][] a responsive front end framework.
Consequently, the site is using [sass][] and [compass][] for CSS pre-processing.
This will also require ruby to be installed along with sass and compass. Both
sass and compass can be installed via ruby once it is installed.
sass and compass can be installed via ruby once it is installed. In order to run
gem install ruby-devel might also need to be installed.
```bash
gem install compass

Binary file not shown.

After

Width:  |  Height:  |  Size: 141 KiB

View File

@ -23,6 +23,25 @@ title: Apache NiFi Downloads
<div class="large-12 columns">
<h2>Releases</h2>
<ul>
<li>0.1.0-incubating
<ul>
<li><a href="https://issues.apache.org/jira/secure/ReleaseNote.jspa?projectId=12316020&version=12329276">Release Notes</a></li>
<li>
Sources:
<ul>
<li><a href="https://www.apache.org/dyn/closer.cgi?path=/incubator/nifi/0.1.0-incubating/nifi-0.1.0-incubating-source-release.zip">nifi-0.1.0-incubating-source-release.zip</a> ( <a href="https://dist.apache.org/repos/dist/release/incubator/nifi/0.1.0-incubating/nifi-0.1.0-incubating-source-release.zip.asc">asc</a>, <a href="https://dist.apache.org/repos/dist/release/incubator/nifi/0.1.0-incubating/nifi-0.1.0-incubating-source-release.zip.md5">md5</a>, <a href="https://dist.apache.org/repos/dist/release/incubator/nifi/0.1.0-incubating/nifi-0.1.0-incubating-source-release.zip.sha1">sha1</a> )</li>
</ul>
</li>
<li>
Binaries
<ul>
<li><a href="https://www.apache.org/dyn/closer.cgi?path=/incubator/nifi/0.1.0-incubating/nifi-0.1.0-incubating-bin.tar.gz">nifi-0.1.0-incubating-bin.tar.gz</a> ( <a href="https://dist.apache.org/repos/dist/release/incubator/nifi/0.1.0-incubating/nifi-0.1.0-incubating-bin.tar.gz.asc">asc</a>, <a href="https://dist.apache.org/repos/dist/release/incubator/nifi/0.1.0-incubating/nifi-0.1.0-incubating-bin.tar.gz.md5">md5</a>, <a href="https://dist.apache.org/repos/dist/release/incubator/nifi/0.1.0-incubating/nifi-0.1.0-incubating-bin.tar.gz.sha1">sha1</a> )</li>
<li><a href="https://www.apache.org/dyn/closer.cgi?path=/incubator/nifi/0.1.0-incubating/nifi-0.1.0-incubating-bin.zip">nifi-0.1.0-incubating-bin.zip</a> ( <a href="https://dist.apache.org/repos/dist/release/incubator/nifi/0.1.0-incubating/nifi-0.1.0-incubating-bin.zip.asc">asc</a>, <a href="https://dist.apache.org/repos/dist/release/incubator/nifi/0.1.0-incubating/nifi-0.1.0-incubating-bin.zip.md5">md5</a>, <a href="https://dist.apache.org/repos/dist/release/incubator/nifi/0.1.0-incubating/nifi-0.1.0-incubating-bin.zip.sha1">sha1</a> )</li>
</ul>
</li>
</ul>
</li>
<li>0.0.2-incubating
<ul>
<li><a href="https://issues.apache.org/jira/secure/ReleaseNote.jspa?projectId=12316020&version=12329373">Release Notes</a></li>

View File

@ -23,7 +23,9 @@ title: Apache NiFi
</div>
</div>
<div class="medium-6 large-7 columns">
<img id="flow" src="images/flow.png" alt="NiFi flow"/>
<a class="th" href="images/flow.png">
<img id="flow" src="images/flow-th.png" alt="NiFi flow">
</a>
</div>
<div class="clear"></div>
</div>
@ -42,169 +44,34 @@ title: Apache NiFi
of the high-level capabilities and objectives of Apache NiFi include:
</p>
<ul>
<li>Web-based user interface for seamless experience between design, control, feedback, and monitoring
of data flows
<li>Web-based user interface
<ul><li>Seamless experience between design, control, feedback, and monitoring</li></ul>
</li>
<li>Highly configurable along several dimensions of quality of service such as loss tolerant versus
guaranteed delivery, low latency versus high throughput, and priority based queuing
<li>Highly configurable
<ul>
<li>Loss tolerant vs guaranteed delivery</li>
<li>Low latency vs high throughput</li>
<li>Dynamic prioritization</li>
<li>Flow can be modified at runtime</li>
<li>Back pressure</li>
</ul>
</li>
<li>Fine-grained data provenance for all data received, forked, joined, cloned, modified, sent, and
ultimately dropped as data reaches its configured end-state
<li>Data Provenance
<ul><li>Track dataflow from beginning to end</li></ul>
</li>
<li>Component-based extension model along well defined interfaces enabling rapid development and
effective testing
<li>Designed for extension
<ul>
<li>Build your own processors and more</li>
<li>Enables rapid development and effective testing</li>
</ul>
</li>
<li>Security: Supports various forms of SSL, SSH, HTTPS for system to system flows and content
encryption/decryption. And SSL, PKI, pluggable authorization for users.
<li>Secure
<ul>
<li>SSL, SSH, HTTPS, encrypted content, etc...</li>
<li>Pluggable role-based authentication/authorization</li>
</ul>
</li>
</ul>
</div>
</div>
<!--<div class="row">
<div class="large-12 columns">
<h1>Welcome to Foundation</h1>
</div>
</div>
<div class="row">
<div class="large-12 columns">
<div class="panel">
<h3>We&rsquo;re stoked you want to try Foundation! </h3>
<p>To get going, this file (index.html) includes some basic styles you can modify, play around with, or totally destroy to get going.</p>
<p>Once you've exhausted the fun in this document, you should check out:</p>
<div class="row">
<div class="large-4 medium-4 columns">
<p><a href="http://foundation.zurb.com/docs">Foundation Documentation</a><br />Everything you need to know about using the framework.</p>
</div>
<div class="large-4 medium-4 columns">
<p><a href="http://github.com/zurb/foundation">Foundation on Github</a><br />Latest code, issue reports, feature requests and more.</p>
</div>
<div class="large-4 medium-4 columns">
<p><a href="http://twitter.com/foundationzurb">@foundationzurb</a><br />Ping us on Twitter if you have questions. If you build something with this we'd love to see it (and send you a totally boss sticker).</p>
</div>
</div>
</div>
</div>
</div>
<div class="row">
<div class="large-8 medium-8 columns">
<h5>Here&rsquo;s your basic grid:</h5>
Grid Example
<div class="row">
<div class="large-12 columns">
<div class="callout panel">
<p><strong>This is a twelve column section in a row.</strong> Each of these includes a div.panel element so you can see where the columns are - it's not required at all for the grid.</p>
</div>
</div>
</div>
<div class="row">
<div class="large-6 medium-6 columns">
<div class="callout panel">
<p>Six columns</p>
</div>
</div>
<div class="large-6 medium-6 columns">
<div class="callout panel">
<p>Six columns</p>
</div>
</div>
</div>
<div class="row">
<div class="large-4 medium-4 small-4 columns">
<div class="callout panel">
<p>Four columns</p>
</div>
</div>
<div class="large-4 medium-4 small-4 columns">
<div class="callout panel">
<p>Four columns</p>
</div>
</div>
<div class="large-4 medium-4 small-4 columns">
<div class="callout panel">
<p>Four columns</p>
</div>
</div>
</div>
<hr />
<h5>We bet you&rsquo;ll need a form somewhere:</h5>
<form>
<div class="row">
<div class="large-12 columns">
<label>Input Label</label>
<input type="text" placeholder="large-12.columns" />
</div>
</div>
<div class="row">
<div class="large-4 medium-4 columns">
<label>Input Label</label>
<input type="text" placeholder="large-4.columns" />
</div>
<div class="large-4 medium-4 columns">
<label>Input Label</label>
<input type="text" placeholder="large-4.columns" />
</div>
<div class="large-4 medium-4 columns">
<div class="row collapse">
<label>Input Label</label>
<div class="small-9 columns">
<input type="text" placeholder="small-9.columns" />
</div>
<div class="small-3 columns">
<span class="postfix">.com</span>
</div>
</div>
</div>
</div>
<div class="row">
<div class="large-12 columns">
<label>Select Box</label>
<select>
<option value="husker">Husker</option>
<option value="starbuck">Starbuck</option>
<option value="hotdog">Hot Dog</option>
<option value="apollo">Apollo</option>
</select>
</div>
</div>
<div class="row">
<div class="large-6 medium-6 columns">
<label>Choose Your Favorite</label>
<input type="radio" name="pokemon" value="Red" id="pokemonRed"><label for="pokemonRed">Radio 1</label>
<input type="radio" name="pokemon" value="Blue" id="pokemonBlue"><label for="pokemonBlue">Radio 2</label>
</div>
<div class="large-6 medium-6 columns">
<label>Check these out</label>
<input id="checkbox1" type="checkbox"><label for="checkbox1">Checkbox 1</label>
<input id="checkbox2" type="checkbox"><label for="checkbox2">Checkbox 2</label>
</div>
</div>
<div class="row">
<div class="large-12 columns">
<label>Textarea Label</label>
<textarea placeholder="small-12.columns"></textarea>
</div>
</div>
</form>
</div>
<div class="large-4 medium-4 columns">
<h5>Try one of these buttons:</h5>
<p><a href="#" class="small button">Simple Button</a><br/>
<a href="#" class="small radius button">Radius Button</a><br/>
<a href="#" class="small round button">Round Button</a><br/>
<a href="#" class="medium success button">Success Btn</a><br/>
<a href="#" class="medium alert button">Alert Btn</a><br/>
<a href="#" class="medium secondary button">Secondary Btn</a></p>
<div class="panel">
<h5>So many components, girl!</h5>
<p>A whole kitchen sink of goodies comes with Foundation. Check out the docs to see them all, along with details on making them your own.</p>
<a href="http://foundation.zurb.com/docs/" class="small button">Go to Foundation Docs</a>
</div>
</div>
</div>-->

View File

@ -18,7 +18,7 @@
<parent>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi</artifactId>
<version>0.1.0-incubating-SNAPSHOT</version>
<version>0.1.1-incubating-SNAPSHOT</version>
</parent>
<artifactId>nifi-api</artifactId>
<packaging>jar</packaging>

View File

@ -851,3 +851,32 @@ For details see http://asm.ow2.org/asmdex-license.html
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
THE POSSIBILITY OF SUCH DAMAGE.
The binary distribution of this product bundles 'Hamcrest' which is available
under a BSD license. More details found here: http://hamcrest.org.
Copyright (c) 2000-2006, www.hamcrest.org
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
Redistributions of source code must retain the above copyright notice, this list of
conditions and the following disclaimer. Redistributions in binary form must reproduce
the above copyright notice, this list of conditions and the following disclaimer in
the documentation and/or other materials provided with the distribution.
Neither the name of Hamcrest nor the names of its contributors may be used to endorse
or promote products derived from this software without specific prior written
permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT
SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY
WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
DAMAGE.

View File

@ -491,6 +491,10 @@ The following binary components are provided under the Apache Software License v
This product includes software developed by
Saxonica (http://www.saxonica.com/).
(ASLv2) MongoDB Java Driver
The following NOTICE information applies:
Copyright (C) 2008-2013 10gen, Inc.
(ASLv2) Parquet MR
The following NOTICE information applies:
Parquet MR

View File

@ -9,13 +9,12 @@ by applicable law or agreed to in writing, software distributed under the
License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
OF ANY KIND, either express or implied. See the License for the specific
language governing permissions and limitations under the License. -->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi</artifactId>
<version>0.1.0-incubating-SNAPSHOT</version>
<version>0.1.1-incubating-SNAPSHOT</version>
</parent>
<artifactId>nifi-assembly</artifactId>
<packaging>pom</packaging>
@ -163,6 +162,17 @@ language governing permissions and limitations under the License. -->
<artifactId>nifi-kite-nar</artifactId>
<type>nar</type>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-dbcp-service-nar</artifactId>
<version>0.1.1-incubating-SNAPSHOT</version>
<type>nar</type>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-mongodb-nar</artifactId>
<type>nar</type>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-solr-nar</artifactId>
@ -171,25 +181,25 @@ language governing permissions and limitations under the License. -->
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-social-media-nar</artifactId>
<version>0.1.0-incubating-SNAPSHOT</version>
<version>0.1.1-incubating-SNAPSHOT</version>
<type>nar</type>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-hl7-nar</artifactId>
<version>0.1.0-incubating-SNAPSHOT</version>
<version>0.1.1-incubating-SNAPSHOT</version>
<type>nar</type>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-language-translation-nar</artifactId>
<version>0.1.0-incubating-SNAPSHOT</version>
<version>0.1.1-incubating-SNAPSHOT</version>
<type>nar</type>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-geo-nar</artifactId>
<version>0.1.0-incubating-SNAPSHOT</version>
<version>0.1.1-incubating-SNAPSHOT</version>
<type>nar</type>
</dependency>
</dependencies>

View File

@ -17,7 +17,7 @@
<parent>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi</artifactId>
<version>0.1.0-incubating-SNAPSHOT</version>
<version>0.1.1-incubating-SNAPSHOT</version>
</parent>
<artifactId>nifi-bootstrap</artifactId>
<packaging>jar</packaging>

View File

@ -17,7 +17,7 @@
<parent>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-commons</artifactId>
<version>0.1.0-incubating-SNAPSHOT</version>
<version>0.1.1-incubating-SNAPSHOT</version>
</parent>
<artifactId>nifi-data-provenance-utils</artifactId>
<packaging>jar</packaging>

View File

@ -17,7 +17,7 @@
<parent>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-commons</artifactId>
<version>0.1.0-incubating-SNAPSHOT</version>
<version>0.1.1-incubating-SNAPSHOT</version>
</parent>
<artifactId>nifi-expression-language</artifactId>
<packaging>jar</packaging>

View File

@ -17,7 +17,7 @@
<parent>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-commons</artifactId>
<version>0.1.0-incubating-SNAPSHOT</version>
<version>0.1.1-incubating-SNAPSHOT</version>
</parent>
<artifactId>nifi-flowfile-packager</artifactId>
<packaging>jar</packaging>

View File

@ -13,14 +13,13 @@
See the License for the specific language governing permissions and
limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-commons</artifactId>
<version>0.1.0-incubating-SNAPSHOT</version>
<version>0.1.1-incubating-SNAPSHOT</version>
</parent>
<artifactId>nifi-hl7-query-language</artifactId>

View File

@ -18,7 +18,7 @@
<parent>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-commons</artifactId>
<version>0.1.0-incubating-SNAPSHOT</version>
<version>0.1.1-incubating-SNAPSHOT</version>
</parent>
<artifactId>nifi-logging-utils</artifactId>
<description>Utilities for logging</description>

View File

@ -18,7 +18,7 @@
<parent>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-commons</artifactId>
<version>0.1.0-incubating-SNAPSHOT</version>
<version>0.1.1-incubating-SNAPSHOT</version>
</parent>
<artifactId>nifi-processor-utils</artifactId>
<packaging>jar</packaging>

View File

@ -18,7 +18,7 @@
<parent>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-commons</artifactId>
<version>0.1.0-incubating-SNAPSHOT</version>
<version>0.1.1-incubating-SNAPSHOT</version>
</parent>
<artifactId>nifi-properties</artifactId>
</project>

View File

@ -17,7 +17,7 @@
<parent>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-commons</artifactId>
<version>0.1.0-incubating-SNAPSHOT</version>
<version>0.1.1-incubating-SNAPSHOT</version>
</parent>
<artifactId>nifi-security-utils</artifactId>
<description>Contains security functionality.</description>

View File

@ -19,7 +19,7 @@
<parent>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-commons</artifactId>
<version>0.1.0-incubating-SNAPSHOT</version>
<version>0.1.1-incubating-SNAPSHOT</version>
</parent>
<artifactId>nifi-site-to-site-client</artifactId>
@ -42,7 +42,7 @@
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-client-dto</artifactId>
<version>0.1.0-incubating-SNAPSHOT</version>
<version>0.1.1-incubating-SNAPSHOT</version>
</dependency>
<dependency>

View File

@ -18,7 +18,7 @@
<parent>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-commons</artifactId>
<version>0.1.0-incubating-SNAPSHOT</version>
<version>0.1.1-incubating-SNAPSHOT</version>
</parent>
<artifactId>nifi-socket-utils</artifactId>
<description>Utilities for socket communication</description>

View File

@ -45,17 +45,19 @@ public final class ChannelDispatcher implements Runnable {
private final StreamConsumerFactory factory;
private final AtomicLong channelReaderFrequencyMilliseconds = new AtomicLong(DEFAULT_CHANNEL_READER_PERIOD_MILLISECONDS);
private final long timeout;
private final boolean readSingleDatagram;
private volatile boolean stop = false;
public static final long DEFAULT_CHANNEL_READER_PERIOD_MILLISECONDS = 100L;
public ChannelDispatcher(final Selector serverSocketSelector, final Selector socketChannelSelector, final ScheduledExecutorService service,
final StreamConsumerFactory factory, final BufferPool buffers, final long timeout, final TimeUnit unit) {
final StreamConsumerFactory factory, final BufferPool buffers, final long timeout, final TimeUnit unit, final boolean readSingleDatagram) {
this.serverSocketSelector = serverSocketSelector;
this.socketChannelSelector = socketChannelSelector;
this.executor = service;
this.factory = factory;
emptyBuffers = buffers;
this.timeout = TimeUnit.MILLISECONDS.convert(timeout, unit);
this.readSingleDatagram = readSingleDatagram;
}
public void setChannelReaderFrequency(final long period, final TimeUnit timeUnit) {
@ -136,7 +138,7 @@ public final class ChannelDispatcher implements Runnable {
// for a DatagramChannel we don't want to create a new reader unless it is a new DatagramChannel. The only
// way to tell if it's new is the lack of an attachment.
if (channel instanceof DatagramChannel && socketChannelKey.attachment() == null) {
reader = new DatagramChannelReader(UUID.randomUUID().toString(), socketChannelKey, emptyBuffers, factory);
reader = new DatagramChannelReader(UUID.randomUUID().toString(), socketChannelKey, emptyBuffers, factory, readSingleDatagram);
socketChannelKey.attach(reader);
final ScheduledFuture<?> readerFuture = executor.scheduleWithFixedDelay(reader, 10L, channelReaderFrequencyMilliseconds.get(),
TimeUnit.MILLISECONDS);

View File

@ -75,14 +75,14 @@ public final class ChannelListener {
private volatile long channelReaderFrequencyMSecs = 50;
public ChannelListener(final int threadPoolSize, final StreamConsumerFactory consumerFactory, final BufferPool bufferPool, int timeout,
TimeUnit unit) throws IOException {
TimeUnit unit, final boolean readSingleDatagram) throws IOException {
this.executor = Executors.newScheduledThreadPool(threadPoolSize + 1); // need to allow for long running ChannelDispatcher thread
this.serverSocketSelector = Selector.open();
this.socketChannelSelector = Selector.open();
this.bufferPool = bufferPool;
this.initialBufferPoolSize = bufferPool.size();
channelDispatcher = new ChannelDispatcher(serverSocketSelector, socketChannelSelector, executor, consumerFactory, bufferPool,
timeout, unit);
timeout, unit, readSingleDatagram);
executor.schedule(channelDispatcher, 50, TimeUnit.MILLISECONDS);
}

View File

@ -27,8 +27,12 @@ public final class DatagramChannelReader extends AbstractChannelReader {
public static final int MAX_UDP_PACKET_SIZE = 65507;
public DatagramChannelReader(final String id, final SelectionKey key, final BufferPool empties, final StreamConsumerFactory consumerFactory) {
private final boolean readSingleDatagram;
public DatagramChannelReader(final String id, final SelectionKey key, final BufferPool empties, final StreamConsumerFactory consumerFactory,
final boolean readSingleDatagram) {
super(id, key, empties, consumerFactory);
this.readSingleDatagram = readSingleDatagram;
}
/**
@ -45,7 +49,7 @@ public final class DatagramChannelReader extends AbstractChannelReader {
final DatagramChannel dChannel = (DatagramChannel) key.channel();
final int initialBufferPosition = buffer.position();
while (buffer.remaining() > MAX_UDP_PACKET_SIZE && key.isValid() && key.isReadable()) {
if (dChannel.receive(buffer) == null) {
if (dChannel.receive(buffer) == null || readSingleDatagram) {
break;
}
}

View File

@ -52,7 +52,7 @@ public final class ServerMain {
ChannelListener listener = null;
try {
executor.scheduleWithFixedDelay(bufferPool, 0L, 5L, TimeUnit.SECONDS);
listener = new ChannelListener(5, new ExampleStreamConsumerFactory(executor, consumerMap), bufferPool, 5, TimeUnit.MILLISECONDS);
listener = new ChannelListener(5, new ExampleStreamConsumerFactory(executor, consumerMap), bufferPool, 5, TimeUnit.MILLISECONDS, false);
listener.setChannelReaderSchedulingPeriod(50L, TimeUnit.MILLISECONDS);
listener.addDatagramChannel(null, 20000, 32 << 20);
LOGGER.info("Listening for UDP data on port 20000");

View File

@ -18,10 +18,10 @@
<parent>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-commons</artifactId>
<version>0.1.0-incubating-SNAPSHOT</version>
<version>0.1.1-incubating-SNAPSHOT</version>
</parent>
<artifactId>nifi-utils</artifactId>
<version>0.1.0-incubating-SNAPSHOT</version>
<version>0.1.1-incubating-SNAPSHOT</version>
<packaging>jar</packaging>
<!--
This project intentionally has no additional dependencies beyond that pulled in by the parent. It is a general purpose utility library

View File

@ -18,7 +18,7 @@
<parent>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-commons</artifactId>
<version>0.1.0-incubating-SNAPSHOT</version>
<version>0.1.1-incubating-SNAPSHOT</version>
</parent>
<artifactId>nifi-web-utils</artifactId>
<dependencies>

View File

@ -18,7 +18,7 @@
<parent>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-commons</artifactId>
<version>0.1.0-incubating-SNAPSHOT</version>
<version>0.1.1-incubating-SNAPSHOT</version>
</parent>
<artifactId>nifi-write-ahead-log</artifactId>
<packaging>jar</packaging>

View File

@ -18,7 +18,7 @@
<parent>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi</artifactId>
<version>0.1.0-incubating-SNAPSHOT</version>
<version>0.1.1-incubating-SNAPSHOT</version>
</parent>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-commons</artifactId>

View File

@ -4,7 +4,7 @@
<parent>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi</artifactId>
<version>0.1.0-incubating-SNAPSHOT</version>
<version>0.1.1-incubating-SNAPSHOT</version>
</parent>
<packaging>pom</packaging>
<artifactId>nifi-docs</artifactId>

View File

@ -18,7 +18,7 @@
<parent>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-external</artifactId>
<version>0.1.0-incubating-SNAPSHOT</version>
<version>0.1.1-incubating-SNAPSHOT</version>
</parent>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-spark-receiver</artifactId>

View File

@ -18,7 +18,7 @@
<parent>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi</artifactId>
<version>0.1.0-incubating-SNAPSHOT</version>
<version>0.1.1-incubating-SNAPSHOT</version>
</parent>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-external</artifactId>

View File

@ -18,7 +18,7 @@
<parent>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-maven-archetypes</artifactId>
<version>0.1.0-incubating-SNAPSHOT</version>
<version>0.1.1-incubating-SNAPSHOT</version>
</parent>
<artifactId>nifi-processor-bundle-archetype</artifactId>

View File

@ -18,7 +18,7 @@
<parent>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi</artifactId>
<version>0.1.0-incubating-SNAPSHOT</version>
<version>0.1.1-incubating-SNAPSHOT</version>
</parent>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-maven-archetypes</artifactId>

View File

@ -18,7 +18,7 @@
<parent>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi</artifactId>
<version>0.1.0-incubating-SNAPSHOT</version>
<version>0.1.1-incubating-SNAPSHOT</version>
</parent>
<artifactId>nifi-mock</artifactId>
<dependencies>

View File

@ -1,36 +1,36 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-aws-bundle</artifactId>
<version>0.1.0-incubating-SNAPSHOT</version>
</parent>
<artifactId>nifi-aws-nar</artifactId>
<packaging>nar</packaging>
<dependencies>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-aws-processors</artifactId>
<version>0.1.0-incubating-SNAPSHOT</version>
</dependency>
</dependencies>
</project>
<?xml version="1.0" encoding="UTF-8"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-aws-bundle</artifactId>
<version>0.1.1-incubating-SNAPSHOT</version>
</parent>
<artifactId>nifi-aws-nar</artifactId>
<packaging>nar</packaging>
<dependencies>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-aws-processors</artifactId>
<version>0.1.1-incubating-SNAPSHOT</version>
</dependency>
</dependencies>
</project>

View File

@ -1,71 +1,71 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-aws-bundle</artifactId>
<version>0.1.0-incubating-SNAPSHOT</version>
</parent>
<artifactId>nifi-aws-processors</artifactId>
<packaging>jar</packaging>
<dependencies>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-api</artifactId>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-processor-utils</artifactId>
</dependency>
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk</artifactId>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-mock</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-simple</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.rat</groupId>
<artifactId>apache-rat-plugin</artifactId>
<configuration>
<excludes>
<exclude>src/test/resources/hello.txt</exclude>
</excludes>
</configuration>
</plugin>
</plugins>
</build>
</project>
<?xml version="1.0" encoding="UTF-8"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-aws-bundle</artifactId>
<version>0.1.1-incubating-SNAPSHOT</version>
</parent>
<artifactId>nifi-aws-processors</artifactId>
<packaging>jar</packaging>
<dependencies>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-api</artifactId>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-processor-utils</artifactId>
</dependency>
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk</artifactId>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-mock</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-simple</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.rat</groupId>
<artifactId>apache-rat-plugin</artifactId>
<configuration>
<excludes>
<exclude>src/test/resources/hello.txt</exclude>
</excludes>
</configuration>
</plugin>
</plugins>
</build>
</project>

View File

@ -1,43 +1,43 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-nar-bundles</artifactId>
<version>0.1.0-incubating-SNAPSHOT</version>
</parent>
<artifactId>nifi-aws-bundle</artifactId>
<packaging>pom</packaging>
<modules>
<module>nifi-aws-processors</module>
<module>nifi-aws-nar</module>
</modules>
<dependencyManagement>
<dependencies>
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk</artifactId>
<version>1.9.24</version>
</dependency>
</dependencies>
</dependencyManagement>
</project>
<?xml version="1.0" encoding="UTF-8"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-nar-bundles</artifactId>
<version>0.1.1-incubating-SNAPSHOT</version>
</parent>
<artifactId>nifi-aws-bundle</artifactId>
<packaging>pom</packaging>
<modules>
<module>nifi-aws-processors</module>
<module>nifi-aws-nar</module>
</modules>
<dependencyManagement>
<dependencies>
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk</artifactId>
<version>1.9.24</version>
</dependency>
</dependencies>
</dependencyManagement>
</project>

View File

@ -17,7 +17,7 @@
<parent>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-framework-bundle</artifactId>
<version>0.1.0-incubating-SNAPSHOT</version>
<version>0.1.1-incubating-SNAPSHOT</version>
</parent>
<artifactId>nifi-framework-nar</artifactId>
<packaging>nar</packaging>

View File

@ -18,7 +18,7 @@
<parent>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-framework</artifactId>
<version>0.1.0-incubating-SNAPSHOT</version>
<version>0.1.1-incubating-SNAPSHOT</version>
</parent>
<artifactId>nifi-administration</artifactId>
<build>

View File

@ -18,7 +18,7 @@
<parent>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-framework</artifactId>
<version>0.1.0-incubating-SNAPSHOT</version>
<version>0.1.1-incubating-SNAPSHOT</version>
</parent>
<artifactId>nifi-client-dto</artifactId>
<dependencies>

View File

@ -18,7 +18,7 @@
<parent>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-framework</artifactId>
<version>0.1.0-incubating-SNAPSHOT</version>
<version>0.1.1-incubating-SNAPSHOT</version>
</parent>
<artifactId>nifi-cluster-authorization-provider</artifactId>
<dependencies>

View File

@ -1,41 +1,39 @@
<?xml version="1.0"?>
<project
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"
xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
<!-- Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with this
work for additional information regarding copyright ownership. The ASF licenses
this file to You under the Apache License, Version 2.0 (the "License"); you
may not use this file except in compliance with the License. You may obtain
a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless
required by applicable law or agreed to in writing, software distributed
under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
OR CONDITIONS OF ANY KIND, either express or implied. See the License for
the specific language governing permissions and limitations under the License. -->
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-framework</artifactId>
<version>0.1.0-incubating-SNAPSHOT</version>
</parent>
<artifactId>nifi-documentation</artifactId>
<dependencies>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-nar-utils</artifactId>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-api</artifactId>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-properties</artifactId>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-processor-utils</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
</project>
<?xml version="1.0"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<!-- Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with this
work for additional information regarding copyright ownership. The ASF licenses
this file to You under the Apache License, Version 2.0 (the "License"); you
may not use this file except in compliance with the License. You may obtain
a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless
required by applicable law or agreed to in writing, software distributed
under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
OR CONDITIONS OF ANY KIND, either express or implied. See the License for
the specific language governing permissions and limitations under the License. -->
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-framework</artifactId>
<version>0.1.1-incubating-SNAPSHOT</version>
</parent>
<artifactId>nifi-documentation</artifactId>
<dependencies>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-nar-utils</artifactId>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-api</artifactId>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-properties</artifactId>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-processor-utils</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
</project>

View File

@ -18,7 +18,7 @@
<parent>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-framework</artifactId>
<version>0.1.0-incubating-SNAPSHOT</version>
<version>0.1.1-incubating-SNAPSHOT</version>
</parent>
<artifactId>nifi-file-authorization-provider</artifactId>
<build>

View File

@ -18,7 +18,7 @@
<parent>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-framework</artifactId>
<version>0.1.0-incubating-SNAPSHOT</version>
<version>0.1.1-incubating-SNAPSHOT</version>
</parent>
<artifactId>nifi-framework-cluster-protocol</artifactId>
<packaging>jar</packaging>

View File

@ -18,7 +18,7 @@
<parent>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-framework</artifactId>
<version>0.1.0-incubating-SNAPSHOT</version>
<version>0.1.1-incubating-SNAPSHOT</version>
</parent>
<artifactId>nifi-framework-cluster-web</artifactId>
<packaging>jar</packaging>

View File

@ -18,7 +18,7 @@
<parent>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-framework</artifactId>
<version>0.1.0-incubating-SNAPSHOT</version>
<version>0.1.1-incubating-SNAPSHOT</version>
</parent>
<artifactId>nifi-framework-cluster</artifactId>
<packaging>jar</packaging>

View File

@ -18,7 +18,7 @@
<parent>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-framework</artifactId>
<version>0.1.0-incubating-SNAPSHOT</version>
<version>0.1.1-incubating-SNAPSHOT</version>
</parent>
<artifactId>nifi-framework-core-api</artifactId>
<dependencies>

View File

@ -18,7 +18,7 @@
<parent>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-framework</artifactId>
<version>0.1.0-incubating-SNAPSHOT</version>
<version>0.1.1-incubating-SNAPSHOT</version>
</parent>
<artifactId>nifi-framework-core</artifactId>
<packaging>jar</packaging>

View File

@ -17,7 +17,7 @@
<parent>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-framework</artifactId>
<version>0.1.0-incubating-SNAPSHOT</version>
<version>0.1.1-incubating-SNAPSHOT</version>
</parent>
<artifactId>nifi-nar-utils</artifactId>
<packaging>jar</packaging>

View File

@ -18,7 +18,7 @@
<parent>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-framework</artifactId>
<version>0.1.0-incubating-SNAPSHOT</version>
<version>0.1.1-incubating-SNAPSHOT</version>
</parent>
<artifactId>nifi-resources</artifactId>
<packaging>pom</packaging>

View File

@ -17,7 +17,7 @@
<parent>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-framework</artifactId>
<version>0.1.0-incubating-SNAPSHOT</version>
<version>0.1.1-incubating-SNAPSHOT</version>
</parent>
<artifactId>nifi-runtime</artifactId>
<packaging>jar</packaging>

View File

@ -17,7 +17,7 @@
<parent>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-framework</artifactId>
<version>0.1.0-incubating-SNAPSHOT</version>
<version>0.1.1-incubating-SNAPSHOT</version>
</parent>
<artifactId>nifi-security</artifactId>
<description>Contains security functionality common to NiFi.</description>

View File

@ -18,7 +18,7 @@
<parent>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-framework</artifactId>
<version>0.1.0-incubating-SNAPSHOT</version>
<version>0.1.1-incubating-SNAPSHOT</version>
</parent>
<artifactId>nifi-site-to-site</artifactId>
<dependencies>

View File

@ -18,7 +18,7 @@
<parent>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-framework</artifactId>
<version>0.1.0-incubating-SNAPSHOT</version>
<version>0.1.1-incubating-SNAPSHOT</version>
</parent>
<artifactId>nifi-user-actions</artifactId>
</project>

View File

@ -18,7 +18,7 @@
<parent>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-web</artifactId>
<version>0.1.0-incubating-SNAPSHOT</version>
<version>0.1.1-incubating-SNAPSHOT</version>
</parent>
<artifactId>nifi-custom-ui-utilities</artifactId>
<dependencies>

View File

@ -18,7 +18,7 @@
<parent>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-web</artifactId>
<version>0.1.0-incubating-SNAPSHOT</version>
<version>0.1.1-incubating-SNAPSHOT</version>
</parent>
<artifactId>nifi-jetty</artifactId>
<packaging>jar</packaging>

View File

@ -18,7 +18,7 @@
<parent>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-web</artifactId>
<version>0.1.0-incubating-SNAPSHOT</version>
<version>0.1.1-incubating-SNAPSHOT</version>
</parent>
<artifactId>nifi-ui-extension</artifactId>
<packaging>jar</packaging>

View File

@ -18,7 +18,7 @@
<parent>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-web</artifactId>
<version>0.1.0-incubating-SNAPSHOT</version>
<version>0.1.1-incubating-SNAPSHOT</version>
</parent>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-web-api</artifactId>

View File

@ -18,7 +18,7 @@
<parent>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-web</artifactId>
<version>0.1.0-incubating-SNAPSHOT</version>
<version>0.1.1-incubating-SNAPSHOT</version>
</parent>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-web-content-access</artifactId>

View File

@ -17,7 +17,7 @@
<parent>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-web</artifactId>
<version>0.1.0-incubating-SNAPSHOT</version>
<version>0.1.1-incubating-SNAPSHOT</version>
</parent>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-web-content-viewer</artifactId>

View File

@ -17,7 +17,7 @@
<parent>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-web</artifactId>
<version>0.1.0-incubating-SNAPSHOT</version>
<version>0.1.1-incubating-SNAPSHOT</version>
</parent>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-web-docs</artifactId>

View File

@ -17,7 +17,7 @@
<parent>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-web</artifactId>
<version>0.1.0-incubating-SNAPSHOT</version>
<version>0.1.1-incubating-SNAPSHOT</version>
</parent>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-web-error</artifactId>

View File

@ -18,7 +18,7 @@
<parent>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-web</artifactId>
<version>0.1.0-incubating-SNAPSHOT</version>
<version>0.1.1-incubating-SNAPSHOT</version>
</parent>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-web-optimistic-locking</artifactId>

View File

@ -18,7 +18,7 @@
<parent>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-web</artifactId>
<version>0.1.0-incubating-SNAPSHOT</version>
<version>0.1.1-incubating-SNAPSHOT</version>
</parent>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-web-security</artifactId>

View File

@ -18,7 +18,7 @@
<parent>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-web</artifactId>
<version>0.1.0-incubating-SNAPSHOT</version>
<version>0.1.1-incubating-SNAPSHOT</version>
</parent>
<artifactId>nifi-web-ui</artifactId>
<packaging>war</packaging>

View File

@ -17,7 +17,7 @@
<parent>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-framework</artifactId>
<version>0.1.0-incubating-SNAPSHOT</version>
<version>0.1.1-incubating-SNAPSHOT</version>
</parent>
<artifactId>nifi-web</artifactId>
<packaging>pom</packaging>
@ -40,31 +40,31 @@
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-web-api</artifactId>
<type>war</type>
<version>0.1.0-incubating-SNAPSHOT</version>
<version>0.1.1-incubating-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-web-error</artifactId>
<type>war</type>
<version>0.1.0-incubating-SNAPSHOT</version>
<version>0.1.1-incubating-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-web-docs</artifactId>
<type>war</type>
<version>0.1.0-incubating-SNAPSHOT</version>
<version>0.1.1-incubating-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-web-content-viewer</artifactId>
<type>war</type>
<version>0.1.0-incubating-SNAPSHOT</version>
<version>0.1.1-incubating-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-web-ui</artifactId>
<type>war</type>
<version>0.1.0-incubating-SNAPSHOT</version>
<version>0.1.1-incubating-SNAPSHOT</version>
</dependency>
</dependencies>
</dependencyManagement>

View File

@ -18,7 +18,7 @@
<parent>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-framework-bundle</artifactId>
<version>0.1.0-incubating-SNAPSHOT</version>
<version>0.1.1-incubating-SNAPSHOT</version>
</parent>
<artifactId>nifi-framework</artifactId>
<packaging>pom</packaging>

View File

@ -17,7 +17,7 @@
<parent>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-nar-bundles</artifactId>
<version>0.1.0-incubating-SNAPSHOT</version>
<version>0.1.1-incubating-SNAPSHOT</version>
</parent>
<artifactId>nifi-framework-bundle</artifactId>
<packaging>pom</packaging>
@ -31,92 +31,92 @@
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-framework-cluster-protocol</artifactId>
<version>0.1.0-incubating-SNAPSHOT</version>
<version>0.1.1-incubating-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-framework-cluster-web</artifactId>
<version>0.1.0-incubating-SNAPSHOT</version>
<version>0.1.1-incubating-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-file-authorization-provider</artifactId>
<version>0.1.0-incubating-SNAPSHOT</version>
<version>0.1.1-incubating-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-cluster-authorization-provider</artifactId>
<version>0.1.0-incubating-SNAPSHOT</version>
<version>0.1.1-incubating-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-framework-cluster</artifactId>
<version>0.1.0-incubating-SNAPSHOT</version>
<version>0.1.1-incubating-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-runtime</artifactId>
<version>0.1.0-incubating-SNAPSHOT</version>
<version>0.1.1-incubating-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-client-dto</artifactId>
<version>0.1.0-incubating-SNAPSHOT</version>
<version>0.1.1-incubating-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-web-content-access</artifactId>
<version>0.1.0-incubating-SNAPSHOT</version>
<version>0.1.1-incubating-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-security</artifactId>
<version>0.1.0-incubating-SNAPSHOT</version>
<version>0.1.1-incubating-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-framework-core-api</artifactId>
<version>0.1.0-incubating-SNAPSHOT</version>
<version>0.1.1-incubating-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-site-to-site</artifactId>
<version>0.1.0-incubating-SNAPSHOT</version>
<version>0.1.1-incubating-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-framework-core</artifactId>
<version>0.1.0-incubating-SNAPSHOT</version>
<version>0.1.1-incubating-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-user-actions</artifactId>
<version>0.1.0-incubating-SNAPSHOT</version>
<version>0.1.1-incubating-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-administration</artifactId>
<version>0.1.0-incubating-SNAPSHOT</version>
<version>0.1.1-incubating-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-jetty</artifactId>
<version>0.1.0-incubating-SNAPSHOT</version>
<version>0.1.1-incubating-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-web-optimistic-locking</artifactId>
<version>0.1.0-incubating-SNAPSHOT</version>
<version>0.1.1-incubating-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-web-security</artifactId>
<version>0.1.0-incubating-SNAPSHOT</version>
<version>0.1.1-incubating-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-documentation</artifactId>
<version>0.1.0-incubating-SNAPSHOT</version>
<version>0.1.1-incubating-SNAPSHOT</version>
</dependency>
</dependencies>
</dependencyManagement>

View File

@ -18,7 +18,7 @@
<parent>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-geo-bundle</artifactId>
<version>0.1.0-incubating-SNAPSHOT</version>
<version>0.1.1-incubating-SNAPSHOT</version>
</parent>
<artifactId>nifi-geo-nar</artifactId>
<packaging>nar</packaging>

View File

@ -18,7 +18,7 @@
<parent>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-geo-bundle</artifactId>
<version>0.1.0-incubating-SNAPSHOT</version>
<version>0.1.1-incubating-SNAPSHOT</version>
</parent>
<artifactId>nifi-geo-processors</artifactId>
<dependencies>

View File

@ -18,7 +18,7 @@
<parent>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-nar-bundles</artifactId>
<version>0.1.0-incubating-SNAPSHOT</version>
<version>0.1.1-incubating-SNAPSHOT</version>
</parent>
<artifactId>nifi-geo-bundle</artifactId>
@ -35,7 +35,7 @@
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-geo-processors</artifactId>
<version>0.1.0-incubating-SNAPSHOT</version>
<version>0.1.1-incubating-SNAPSHOT</version>
</dependency>
</dependencies>
</dependencyManagement>

View File

@ -17,7 +17,7 @@
<parent>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-hadoop-bundle</artifactId>
<version>0.1.0-incubating-SNAPSHOT</version>
<version>0.1.1-incubating-SNAPSHOT</version>
</parent>
<artifactId>nifi-hadoop-nar</artifactId>
<packaging>nar</packaging>

View File

@ -17,7 +17,7 @@
<parent>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-hadoop-bundle</artifactId>
<version>0.1.0-incubating-SNAPSHOT</version>
<version>0.1.1-incubating-SNAPSHOT</version>
</parent>
<artifactId>nifi-hdfs-processors</artifactId>
<packaging>jar</packaging>

View File

@ -42,6 +42,13 @@ import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.compress.BZip2Codec;
import org.apache.hadoop.io.compress.CompressionCodec;
import org.apache.hadoop.io.compress.CompressionCodecFactory;
import org.apache.hadoop.io.compress.DefaultCodec;
import org.apache.hadoop.io.compress.GzipCodec;
import org.apache.hadoop.io.compress.Lz4Codec;
import org.apache.hadoop.io.compress.SnappyCodec;
import org.apache.hadoop.net.NetUtils;
/**
@ -60,6 +67,13 @@ public abstract class AbstractHadoopProcessor extends AbstractProcessor {
public static final String DIRECTORY_PROP_NAME = "Directory";
public static final PropertyDescriptor COMPRESSION_CODEC = new PropertyDescriptor.Builder()
.name("Compression codec")
.required(false)
.allowableValues(BZip2Codec.class.getName(), DefaultCodec.class.getName(),
GzipCodec.class.getName(), Lz4Codec.class.getName(), SnappyCodec.class.getName())
.build();
protected static final List<PropertyDescriptor> properties;
static {
@ -228,6 +242,23 @@ public abstract class AbstractHadoopProcessor extends AbstractProcessor {
};
}
/**
* Returns the configured CompressionCodec, or null if none is configured.
*
* @param context the ProcessContext
* @param configuration the Hadoop Configuration
* @return CompressionCodec or null
*/
protected CompressionCodec getCompressionCodec(ProcessContext context, Configuration configuration) {
CompressionCodec codec = null;
if (context.getProperty(COMPRESSION_CODEC).isSet()) {
String compressionClassname = context.getProperty(COMPRESSION_CODEC).getValue();
CompressionCodecFactory ccf = new CompressionCodecFactory(configuration);
codec = ccf.getCodecByClassName(compressionClassname);
}
return codec;
}
/**
* Returns the relative path of the child that does not include the filename

View File

@ -17,6 +17,7 @@
package org.apache.nifi.processors.hadoop;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
@ -33,11 +34,11 @@ import java.util.regex.Pattern;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PathFilter;
import org.apache.hadoop.io.compress.CompressionCodec;
import org.apache.nifi.annotation.behavior.TriggerWhenEmpty;
import org.apache.nifi.annotation.documentation.CapabilityDescription;
import org.apache.nifi.annotation.documentation.SeeAlso;
@ -192,6 +193,7 @@ public class GetHDFS extends AbstractHadoopProcessor {
props.add(POLLING_INTERVAL);
props.add(BATCH_SIZE);
props.add(BUFFER_SIZE);
props.add(COMPRESSION_CODEC);
localProperties = Collections.unmodifiableList(props);
}
@ -329,7 +331,7 @@ public class GetHDFS extends AbstractHadoopProcessor {
protected void processBatchOfFiles(final List<Path> files, final ProcessContext context, final ProcessSession session) {
// process the batch of files
FSDataInputStream stream = null;
InputStream stream = null;
Configuration conf = getConfiguration();
FileSystem hdfs = getFileSystem();
final boolean keepSourceFiles = context.getProperty(KEEP_SOURCE_FILE).asBoolean();
@ -337,6 +339,7 @@ public class GetHDFS extends AbstractHadoopProcessor {
int bufferSize = bufferSizeProp != null ? bufferSizeProp.intValue() : conf.getInt(BUFFER_SIZE_KEY,
BUFFER_SIZE_DEFAULT);
final Path rootDir = new Path(context.getProperty(DIRECTORY).getValue());
final CompressionCodec codec = getCompressionCodec(context, conf);
for (final Path file : files) {
try {
if (!hdfs.exists(file)) {
@ -346,6 +349,9 @@ public class GetHDFS extends AbstractHadoopProcessor {
final String relativePath = getPathDifference(rootDir, file);
stream = hdfs.open(file, bufferSize);
if (codec != null) {
stream = codec.createInputStream(stream);
}
FlowFile flowFile = session.create();
final StopWatch stopWatch = new StopWatch(true);

View File

@ -19,6 +19,7 @@ package org.apache.nifi.processors.hadoop;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
@ -27,10 +28,10 @@ import java.util.Set;
import java.util.concurrent.TimeUnit;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.io.compress.CompressionCodec;
import org.apache.hadoop.ipc.RemoteException;
import org.apache.nifi.annotation.behavior.WritesAttribute;
import org.apache.nifi.annotation.documentation.CapabilityDescription;
@ -157,6 +158,7 @@ public class PutHDFS extends AbstractHadoopProcessor {
props.add(UMASK);
props.add(REMOTE_OWNER);
props.add(REMOTE_GROUP);
props.add(COMPRESSION_CODEC);
localProperties = Collections.unmodifiableList(props);
}
@ -215,6 +217,8 @@ public class PutHDFS extends AbstractHadoopProcessor {
final short replication = replicationProp != null ? replicationProp.shortValue() : hdfs
.getDefaultReplication(configuredRootDirPath);
final CompressionCodec codec = getCompressionCodec(context, configuration);
Path tempDotCopyFile = null;
try {
final Path tempCopyFile;
@ -266,10 +270,13 @@ public class PutHDFS extends AbstractHadoopProcessor {
@Override
public void process(InputStream in) throws IOException {
FSDataOutputStream fos = null;
OutputStream fos = null;
Path createdFile = null;
try {
fos = hdfs.create(tempCopyFile, true, bufferSize, replication, blockSize);
if (codec != null) {
fos = codec.createOutputStream(fos);
}
createdFile = tempCopyFile;
BufferedInputStream bis = new BufferedInputStream(in);
StreamUtils.copy(bis, fos);

View File

@ -19,6 +19,8 @@ package org.apache.nifi.processors.hadoop;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import java.io.IOException;
import java.io.InputStream;
import java.util.Collection;
import java.util.HashSet;
import java.util.List;
@ -30,8 +32,9 @@ import org.apache.nifi.util.MockFlowFile;
import org.apache.nifi.util.MockProcessContext;
import org.apache.nifi.util.TestRunner;
import org.apache.nifi.util.TestRunners;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.compress.CompressionCodec;
import org.apache.hadoop.io.compress.GzipCodec;
import org.junit.Assert;
import org.junit.Test;
@ -105,6 +108,19 @@ public class GetHDFSTest {
for (ValidationResult vr : results) {
Assert.assertTrue(vr.toString().contains("is invalid because Minimum File Age cannot be greater than Maximum File Age"));
}
results = new HashSet<>();
runner.setProperty(GetHDFS.DIRECTORY, "/target");
runner.setProperty(GetHDFS.COMPRESSION_CODEC, CompressionCodec.class.getName());
runner.enqueue(new byte[0]);
pc = runner.getProcessContext();
if (pc instanceof MockProcessContext) {
results = ((MockProcessContext) pc).validate();
}
Assert.assertEquals(1, results.size());
for (ValidationResult vr : results) {
Assert.assertTrue(vr.toString().contains("is invalid because Given value not found in allowed set"));
}
}
@Test
@ -115,9 +131,25 @@ public class GetHDFSTest {
runner.setProperty(GetHDFS.KEEP_SOURCE_FILE, "true");
runner.run();
List<MockFlowFile> flowFiles = runner.getFlowFilesForRelationship(GetHDFS.REL_SUCCESS);
assertEquals(3, flowFiles.size());
assertEquals(4, flowFiles.size());
for (MockFlowFile flowFile : flowFiles) {
assertTrue(flowFile.getAttribute(CoreAttributes.FILENAME.key()).startsWith("random"));
}
}
@Test
public void testGetFilesWithCompression() throws IOException {
TestRunner runner = TestRunners.newTestRunner(GetHDFS.class);
runner.setProperty(PutHDFS.DIRECTORY, "src/test/resources/testdata");
runner.setProperty(GetHDFS.FILE_FILTER_REGEX, "random.*.gz");
runner.setProperty(GetHDFS.COMPRESSION_CODEC, GzipCodec.class.getName());
runner.setProperty(GetHDFS.KEEP_SOURCE_FILE, "true");
runner.run();
List<MockFlowFile> flowFiles = runner.getFlowFilesForRelationship(GetHDFS.REL_SUCCESS);
assertEquals(1, flowFiles.size());
MockFlowFile flowFile = flowFiles.get(0);
assertTrue(flowFile.getAttribute(CoreAttributes.FILENAME.key()).startsWith("randombytes-1.gz"));
InputStream expected = getClass().getResourceAsStream("/testdata/randombytes-1");
flowFile.assertContentEquals(expected);
}
}

View File

@ -17,6 +17,7 @@
package org.apache.nifi.processors.hadoop;
import org.apache.nifi.processors.hadoop.PutHDFS;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
@ -33,10 +34,10 @@ import org.apache.nifi.processor.ProcessContext;
import org.apache.nifi.util.MockProcessContext;
import org.apache.nifi.util.TestRunner;
import org.apache.nifi.util.TestRunners;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.compress.CompressionCodec;
import org.junit.Assert;
import org.junit.Ignore;
import org.junit.Test;
@ -136,6 +137,20 @@ public class PutHDFSTest {
for (ValidationResult vr : results) {
assertTrue(vr.toString().contains("is invalid because octal umask [2000] is not a valid umask"));
}
results = new HashSet<>();
runner = TestRunners.newTestRunner(PutHDFS.class);
runner.setProperty(PutHDFS.DIRECTORY, "/target");
runner.setProperty(PutHDFS.COMPRESSION_CODEC, CompressionCodec.class.getName());
runner.enqueue(new byte[0]);
pc = runner.getProcessContext();
if (pc instanceof MockProcessContext) {
results = ((MockProcessContext) pc).validate();
}
Assert.assertEquals(1, results.size());
for (ValidationResult vr : results) {
Assert.assertTrue(vr.toString().contains("is invalid because Given value not found in allowed set"));
}
}
// The following only seems to work from cygwin...something about not finding the 'chmod' command.

View File

@ -17,7 +17,7 @@
<parent>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-nar-bundles</artifactId>
<version>0.1.0-incubating-SNAPSHOT</version>
<version>0.1.1-incubating-SNAPSHOT</version>
</parent>
<artifactId>nifi-hadoop-bundle</artifactId>
<packaging>pom</packaging>
@ -31,7 +31,7 @@
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-hdfs-processors</artifactId>
<version>0.1.0-incubating-SNAPSHOT</version>
<version>0.1.1-incubating-SNAPSHOT</version>
</dependency>
</dependencies>
</dependencyManagement>

View File

@ -13,7 +13,7 @@
<parent>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-hadoop-libraries-bundle</artifactId>
<version>0.1.0-incubating-SNAPSHOT</version>
<version>0.1.1-incubating-SNAPSHOT</version>
</parent>
<artifactId>nifi-hadoop-libraries-nar</artifactId>
<packaging>nar</packaging>

View File

@ -17,7 +17,7 @@
<parent>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-nar-bundles</artifactId>
<version>0.1.0-incubating-SNAPSHOT</version>
<version>0.1.1-incubating-SNAPSHOT</version>
</parent>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-hadoop-libraries-bundle</artifactId>

View File

@ -19,7 +19,7 @@
<parent>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-hl7-bundle</artifactId>
<version>0.1.0-incubating-SNAPSHOT</version>
<version>0.1.1-incubating-SNAPSHOT</version>
</parent>
<artifactId>nifi-hl7-nar</artifactId>
@ -29,7 +29,7 @@
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-hl7-processors</artifactId>
<version>0.1.0-incubating-SNAPSHOT</version>
<version>0.1.1-incubating-SNAPSHOT</version>
</dependency>
</dependencies>

View File

@ -19,7 +19,7 @@
<parent>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-hl7-bundle</artifactId>
<version>0.1.0-incubating-SNAPSHOT</version>
<version>0.1.1-incubating-SNAPSHOT</version>
</parent>
<artifactId>nifi-hl7-processors</artifactId>
@ -52,7 +52,7 @@
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-hl7-query-language</artifactId>
<version>0.1.0-incubating-SNAPSHOT</version>
<version>0.1.1-incubating-SNAPSHOT</version>
</dependency>
<dependency>

View File

@ -19,7 +19,7 @@
<parent>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-nar-bundles</artifactId>
<version>0.1.0-incubating-SNAPSHOT</version>
<version>0.1.1-incubating-SNAPSHOT</version>
</parent>
<artifactId>nifi-hl7-bundle</artifactId>

View File

@ -17,7 +17,7 @@
<parent>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-nar-bundles</artifactId>
<version>0.1.0-incubating-SNAPSHOT</version>
<version>0.1.1-incubating-SNAPSHOT</version>
</parent>
<artifactId>nifi-jetty-bundle</artifactId>
<packaging>nar</packaging>

View File

@ -17,7 +17,7 @@
<parent>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-kafka-bundle</artifactId>
<version>0.1.0-incubating-SNAPSHOT</version>
<version>0.1.1-incubating-SNAPSHOT</version>
</parent>
<artifactId>nifi-kafka-nar</artifactId>
<packaging>nar</packaging>

View File

@ -16,7 +16,7 @@
<parent>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-kafka-bundle</artifactId>
<version>0.1.0-incubating-SNAPSHOT</version>
<version>0.1.1-incubating-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>nifi-kafka-processors</artifactId>

View File

@ -17,7 +17,7 @@
<parent>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-nar-bundles</artifactId>
<version>0.1.0-incubating-SNAPSHOT</version>
<version>0.1.1-incubating-SNAPSHOT</version>
</parent>
<artifactId>nifi-kafka-bundle</artifactId>
<packaging>pom</packaging>
@ -30,7 +30,7 @@
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-kafka-processors</artifactId>
<version>0.1.0-incubating-SNAPSHOT</version>
<version>0.1.1-incubating-SNAPSHOT</version>
</dependency>
</dependencies>
</dependencyManagement>

View File

@ -18,7 +18,7 @@
<parent>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-kite-bundle</artifactId>
<version>0.1.0-incubating-SNAPSHOT</version>
<version>0.1.1-incubating-SNAPSHOT</version>
</parent>
<artifactId>nifi-kite-nar</artifactId>

View File

@ -18,7 +18,7 @@
<parent>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-kite-bundle</artifactId>
<version>0.1.0-incubating-SNAPSHOT</version>
<version>0.1.1-incubating-SNAPSHOT</version>
</parent>
<artifactId>nifi-kite-processors</artifactId>

View File

@ -18,7 +18,7 @@
<parent>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-nar-bundles</artifactId>
<version>0.1.0-incubating-SNAPSHOT</version>
<version>0.1.1-incubating-SNAPSHOT</version>
</parent>
<artifactId>nifi-kite-bundle</artifactId>
@ -36,7 +36,7 @@
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-kite-processors</artifactId>
<version>0.1.0-incubating-SNAPSHOT</version>
<version>0.1.1-incubating-SNAPSHOT</version>
</dependency>
</dependencies>
</dependencyManagement>

View File

@ -19,7 +19,7 @@
<parent>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-language-translation-bundle</artifactId>
<version>0.1.0-incubating-SNAPSHOT</version>
<version>0.1.1-incubating-SNAPSHOT</version>
</parent>
<artifactId>nifi-language-translation-nar</artifactId>
@ -29,7 +29,7 @@
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-yandex-processors</artifactId>
<version>0.1.0-incubating-SNAPSHOT</version>
<version>0.1.1-incubating-SNAPSHOT</version>
</dependency>
</dependencies>

View File

@ -19,7 +19,7 @@
<parent>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-language-translation-bundle</artifactId>
<version>0.1.0-incubating-SNAPSHOT</version>
<version>0.1.1-incubating-SNAPSHOT</version>
</parent>
<artifactId>nifi-yandex-processors</artifactId>

View File

@ -19,7 +19,7 @@
<parent>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-nar-bundles</artifactId>
<version>0.1.0-incubating-SNAPSHOT</version>
<version>0.1.1-incubating-SNAPSHOT</version>
</parent>
<artifactId>nifi-language-translation-bundle</artifactId>

View File

@ -0,0 +1,37 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-mongodb-bundle</artifactId>
<version>0.1.1-incubating-SNAPSHOT</version>
</parent>
<artifactId>nifi-mongodb-nar</artifactId>
<version>0.1.1-incubating-SNAPSHOT</version>
<packaging>nar</packaging>
<dependencies>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-mongodb-processors</artifactId>
<version>0.1.1-incubating-SNAPSHOT</version>
</dependency>
</dependencies>
</project>

View File

@ -0,0 +1,98 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-mongodb-bundle</artifactId>
<version>0.1.1-incubating-SNAPSHOT</version>
</parent>
<artifactId>nifi-mongodb-processors</artifactId>
<packaging>jar</packaging>
<dependencies>
<dependency>
<groupId>org.mongodb</groupId>
<artifactId>mongo-java-driver</artifactId>
<version>3.0.1</version>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-api</artifactId>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-processor-utils</artifactId>
</dependency>
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-mock</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-simple</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>com.github.joelittlejohn.embedmongo</groupId>
<artifactId>embedmongo-maven-plugin</artifactId>
<version>0.1.12</version>
<executions>
<execution>
<id>start</id>
<goals>
<goal>start</goal>
</goals>
<phase>test-compile</phase>
<configuration>
<databaseDirectory>${project.build.directory}/embedmongo/db</databaseDirectory>
<logging>file</logging>
<logFile>${project.build.directory}/embedmongo.log</logFile>
</configuration>
</execution>
<execution>
<id>stop</id>
<goals>
<goal>stop</goal>
</goals>
<phase>prepare-package</phase>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

Some files were not shown because too many files have changed in this diff Show More