Apache/nifi
1
0
Fork 0
mirror of https://github.com/apache/nifi.git synced 2025-02-27 22:19:08 +00:00
Code Issues Packages Projects Releases Wiki Activity

NIFI-527: Compress prov logs in 'chunks' and just store the chunk offsets in Lucene instead of byte offsets

Browse Source
This commit is contained in:
Mark Payne 2015-04-27 10:41:18 -04:00
parent a5ac48a03c
commit 7c41225e89
3 changed files with 31 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
nifi/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/ByteCountingInputStream.java
View File

@ -31,6 +31,11 @@ public class ByteCountingInputStream extends InputStream {
this.in = in;
}
public ByteCountingInputStream(final InputStream in, final long initialOffset) {
this.in = in;
this.bytesSkipped = initialOffset;
}
@Override
public int read() throws IOException {
final int fromSuper = in.read();

8
nifi/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/stream/io/ByteCountingOutputStream.java
View File

@ -27,6 +27,12 @@ public class ByteCountingOutputStream extends OutputStream {
public ByteCountingOutputStream(final OutputStream out) {
this.out = out;
}
public ByteCountingOutputStream(final OutputStream out, final long initialByteCount) {
this.out = out;
this.bytesWritten = initialByteCount;
}
@Override
public void write(int b) throws IOException {
@ -39,6 +45,8 @@ public class ByteCountingOutputStream extends OutputStream {
write(b, 0, b.length);
}
;
@Override
public void write(byte[] b, int off, int len) throws IOException {
out.write(b, off, len);

19
nifi/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/test/java/org/apache/nifi/provenance/TestPersistentProvenanceRepository.java
View File

@ -110,7 +110,24 @@ public class TestPersistentProvenanceRepository {
// we create but also to ensure that we have closed all of the file handles. If we leave any
// streams open, for instance, this will throw an IOException, causing our unit test to fail.
for ( final File storageDir : config.getStorageDirectories() ) {
FileUtils.deleteFile(storageDir, true);
int i;
for (i=0; i < 3; i++) {
try {
FileUtils.deleteFile(storageDir, true);
break;
} catch (final IOException ioe) {
// if there is a virus scanner, etc. running in the background we may not be able to
// delete the file. Wait a sec and try again.
if ( i == 2 ) {
throw ioe;
} else {
try {
Thread.sleep(1000L);
} catch (final InterruptedException ie) {
}
}
}
}
}
}