Apache
/
poi
mirror of https://github.com/apache/poi.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[bug-65184] Improve performance of POFSMiniStore getBlockAt. Thanks to sits 

git-svn-id: https://svn.apache.org/repos/asf/poi/trunk@1887604 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
PJ Fanning 2021-03-13 18:09:13 +00:00
parent 5758145ce8
commit c70b649e73
1 changed files with 135 additions and 137 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

272
src/java/org/apache/poi/poifs/filesystem/POIFSMiniStore.java
View File

@ -1,4 +1,3 @@
/* ==================================================================== /* ====================================================================
Licensed to the Apache Software Foundation (ASF) under one or more Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with contributor license agreements. See the NOTICE file distributed with
@ -33,141 +32,140 @@ import org.apache.poi.poifs.storage.HeaderBlock;
/** /**
* This class handles the MiniStream (small block store) * This class handles the MiniStream (small block store)
* in the NIO case for {@link POIFSFileSystem} * in the NIO case for {@link POIFSFileSystem}
*/ */
public class POIFSMiniStore extends BlockStore public class POIFSMiniStore extends BlockStore {
{
private final POIFSFileSystem _filesystem; private final POIFSFileSystem _filesystem;
private POIFSStream _mini_stream; private POIFSStream _mini_stream;
private final List<BATBlock> _sbat_blocks; private final List<BATBlock> _sbat_blocks;
private final HeaderBlock _header; private final HeaderBlock _header;
private final RootProperty _root; private final RootProperty _root;
POIFSMiniStore(POIFSFileSystem filesystem, RootProperty root, POIFSMiniStore(POIFSFileSystem filesystem, RootProperty root,
List<BATBlock> sbats, HeaderBlock header) List<BATBlock> sbats, HeaderBlock header) {
{ this._filesystem = filesystem;
this._filesystem = filesystem; this._sbat_blocks = sbats;
this._sbat_blocks = sbats; this._header = header;
this._header = header; this._root = root;
this._root = root;
this._mini_stream = new POIFSStream(filesystem, root.getStartBlock()); this._mini_stream = new POIFSStream(filesystem, root.getStartBlock());
} }
/** /**
* Load the block at the given offset. * Load the block at the given offset.
*/ */
protected ByteBuffer getBlockAt(final int offset) { protected ByteBuffer getBlockAt(final int offset) {
// Which big block is this? // Which big block is this?
int byteOffset = offset * POIFSConstants.SMALL_BLOCK_SIZE; int byteOffset = offset * POIFSConstants.SMALL_BLOCK_SIZE;
int bigBlockNumber = byteOffset / _filesystem.getBigBlockSize(); int bigBlockNumber = byteOffset / _filesystem.getBigBlockSize();
int bigBlockOffset = byteOffset % _filesystem.getBigBlockSize(); int bigBlockOffset = byteOffset % _filesystem.getBigBlockSize();
// Now locate the data block for it // Now locate the data block for it
Iterator<ByteBuffer> it = _mini_stream.getBlockIterator(); Iterator<ByteBuffer> it = _mini_stream.getBlockIterator();
for(int i=0; i<bigBlockNumber; i++) { for (int i = 0; i < bigBlockNumber; i++) {
it.next(); it.next();
} }
ByteBuffer dataBlock = it.next(); ByteBuffer dataBlock = it.next();
assert(dataBlock != null); assert (dataBlock != null);
// Position ourselves, and take a slice // Position ourselves, and take a slice
dataBlock.position( dataBlock.position(
dataBlock.position() + bigBlockOffset dataBlock.position() + bigBlockOffset
); );
ByteBuffer miniBuffer = dataBlock.slice(); ByteBuffer miniBuffer = dataBlock.slice();
miniBuffer.limit(POIFSConstants.SMALL_BLOCK_SIZE); miniBuffer.limit(POIFSConstants.SMALL_BLOCK_SIZE);
return miniBuffer; return miniBuffer;
} }
/** /**
* Load the block, extending the underlying stream if needed * Load the block, extending the underlying stream if needed
*/ */
protected ByteBuffer createBlockIfNeeded(final int offset) throws IOException { protected ByteBuffer createBlockIfNeeded(final int offset) throws IOException {
boolean firstInStore = false; boolean firstInStore = false;
if (_mini_stream.getStartBlock() == POIFSConstants.END_OF_CHAIN) { if (_mini_stream.getStartBlock() == POIFSConstants.END_OF_CHAIN) {
firstInStore = true; firstInStore = true;
} }
// Try to get it without extending the stream // Try to get it without extending the stream
if (! firstInStore) { if (!firstInStore) {
try { try {
return getBlockAt(offset); return getBlockAt(offset);
} catch(NoSuchElementException ignored) {} } catch (NoSuchElementException ignored) {
} }
}
// Need to extend the stream // Need to extend the stream
// TODO Replace this with proper append support // TODO Replace this with proper append support
// For now, do the extending by hand... // For now, do the extending by hand...
// Ask for another block // Ask for another block
int newBigBlock = _filesystem.getFreeBlock(); int newBigBlock = _filesystem.getFreeBlock();
_filesystem.createBlockIfNeeded(newBigBlock); _filesystem.createBlockIfNeeded(newBigBlock);
// If we are the first block to be allocated, initialise the stream // If we are the first block to be allocated, initialise the stream
if (firstInStore) { if (firstInStore) {
_filesystem._get_property_table().getRoot().setStartBlock(newBigBlock); _filesystem._get_property_table().getRoot().setStartBlock(newBigBlock);
_mini_stream = new POIFSStream(_filesystem, newBigBlock); _mini_stream = new POIFSStream(_filesystem, newBigBlock);
} else { } else {
// Tack it onto the end of our chain // Tack it onto the end of our chain
ChainLoopDetector loopDetector = _filesystem.getChainLoopDetector(); ChainLoopDetector loopDetector = _filesystem.getChainLoopDetector();
int block = _mini_stream.getStartBlock(); int block = _mini_stream.getStartBlock();
while(true) { while (true) {
loopDetector.claim(block); loopDetector.claim(block);
int next = _filesystem.getNextBlock(block); int next = _filesystem.getNextBlock(block);
if(next == POIFSConstants.END_OF_CHAIN) { if (next == POIFSConstants.END_OF_CHAIN) {
break; break;
} }
block = next; block = next;
} }
_filesystem.setNextBlock(block, newBigBlock); _filesystem.setNextBlock(block, newBigBlock);
} }
// This is now the new end // This is now the new end
_filesystem.setNextBlock(newBigBlock, POIFSConstants.END_OF_CHAIN); _filesystem.setNextBlock(newBigBlock, POIFSConstants.END_OF_CHAIN);
// Now try again, to get the real small block // Now try again, to get the real small block
return createBlockIfNeeded(offset); return createBlockIfNeeded(offset);
} }
/** /**
* Returns the BATBlock that handles the specified offset, * Returns the BATBlock that handles the specified offset,
* and the relative index within it * and the relative index within it
*/ */
protected BATBlockAndIndex getBATBlockAndIndex(final int offset) { protected BATBlockAndIndex getBATBlockAndIndex(final int offset) {
return BATBlock.getSBATBlockAndIndex( return BATBlock.getSBATBlockAndIndex(
offset, _header, _sbat_blocks offset, _header, _sbat_blocks
); );
} }
/** /**
* Works out what block follows the specified one. * Works out what block follows the specified one.
*/ */
protected int getNextBlock(final int offset) { protected int getNextBlock(final int offset) {
BATBlockAndIndex bai = getBATBlockAndIndex(offset); BATBlockAndIndex bai = getBATBlockAndIndex(offset);
return bai.getBlock().getValueAt( bai.getIndex() ); return bai.getBlock().getValueAt(bai.getIndex());
} }
/** /**
* Changes the record of what block follows the specified one. * Changes the record of what block follows the specified one.
*/ */
protected void setNextBlock(final int offset, final int nextBlock) { protected void setNextBlock(final int offset, final int nextBlock) {
BATBlockAndIndex bai = getBATBlockAndIndex(offset); BATBlockAndIndex bai = getBATBlockAndIndex(offset);
bai.getBlock().setValueAt( bai.getBlock().setValueAt(
bai.getIndex(), nextBlock bai.getIndex(), nextBlock
); );
} }
/** /**
* Finds a free block, and returns its offset. * Finds a free block, and returns its offset.
* This method will extend the file if needed, and if doing * This method will extend the file if needed, and if doing
* so, allocate new FAT blocks to address the extra space. * so, allocate new FAT blocks to address the extra space.
*/ */
protected int getFreeBlock() throws IOException { protected int getFreeBlock() throws IOException {
int sectorsPerSBAT = _filesystem.getBigBlockSizeDetails().getBATEntriesPerBlock(); int sectorsPerSBAT = _filesystem.getBigBlockSizeDetails().getBATEntriesPerBlock();
// First up, do we have any spare ones? // First up, do we have any spare ones?
int offset = 0; int offset = 0;
for (BATBlock sbat : _sbat_blocks) { for (BATBlock sbat : _sbat_blocks) {
// Check this one // Check this one
if (sbat.hasFreeSectors()) { if (sbat.hasFreeSectors()) {
@ -185,79 +183,79 @@ public class POIFSMiniStore extends BlockStore
offset += sectorsPerSBAT; offset += sectorsPerSBAT;
} }
// If we get here, then there aren't any // If we get here, then there aren't any
// free sectors in any of the SBATs // free sectors in any of the SBATs
// So, we need to extend the chain and add another // So, we need to extend the chain and add another
// Create a new BATBlock // Create a new BATBlock
BATBlock newSBAT = BATBlock.createEmptyBATBlock(_filesystem.getBigBlockSizeDetails(), false); BATBlock newSBAT = BATBlock.createEmptyBATBlock(_filesystem.getBigBlockSizeDetails(), false);
int batForSBAT = _filesystem.getFreeBlock(); int batForSBAT = _filesystem.getFreeBlock();
newSBAT.setOurBlockIndex(batForSBAT); newSBAT.setOurBlockIndex(batForSBAT);
// Are we the first SBAT? // Are we the first SBAT?
if(_header.getSBATCount() == 0) { if (_header.getSBATCount() == 0) {
// Tell the header that we've got our first SBAT there // Tell the header that we've got our first SBAT there
_header.setSBATStart(batForSBAT); _header.setSBATStart(batForSBAT);
_header.setSBATBlockCount(1); _header.setSBATBlockCount(1);
} else { } else {
// Find the end of the SBAT stream, and add the sbat in there // Find the end of the SBAT stream, and add the sbat in there
ChainLoopDetector loopDetector = _filesystem.getChainLoopDetector(); ChainLoopDetector loopDetector = _filesystem.getChainLoopDetector();
int batOffset = _header.getSBATStart(); int batOffset = _header.getSBATStart();
while(true) { while (true) {
loopDetector.claim(batOffset); loopDetector.claim(batOffset);
int nextBat = _filesystem.getNextBlock(batOffset); int nextBat = _filesystem.getNextBlock(batOffset);
if(nextBat == POIFSConstants.END_OF_CHAIN) { if (nextBat == POIFSConstants.END_OF_CHAIN) {
break; break;
} }
batOffset = nextBat; batOffset = nextBat;
} }
// Add it in at the end // Add it in at the end
_filesystem.setNextBlock(batOffset, batForSBAT); _filesystem.setNextBlock(batOffset, batForSBAT);
// And update the count // And update the count
_header.setSBATBlockCount( _header.setSBATBlockCount(
_header.getSBATCount() + 1 _header.getSBATCount() + 1
); );
} }
// Finish allocating // Finish allocating
_filesystem.setNextBlock(batForSBAT, POIFSConstants.END_OF_CHAIN); _filesystem.setNextBlock(batForSBAT, POIFSConstants.END_OF_CHAIN);
_sbat_blocks.add(newSBAT); _sbat_blocks.add(newSBAT);
// Return our first spot // Return our first spot
return offset; return offset;
} }
@Override @Override
protected ChainLoopDetector getChainLoopDetector() { protected ChainLoopDetector getChainLoopDetector() {
return new ChainLoopDetector( _root.getSize() ); return new ChainLoopDetector(_root.getSize());
} }
protected int getBlockStoreBlockSize() { protected int getBlockStoreBlockSize() {
return POIFSConstants.SMALL_BLOCK_SIZE; return POIFSConstants.SMALL_BLOCK_SIZE;
} }
/** /**
* Writes the SBATs to their backing blocks, and updates * Writes the SBATs to their backing blocks, and updates
* the mini-stream size in the properties. Stream size is * the mini-stream size in the properties. Stream size is
* based on full blocks used, not the data within the streams * based on full blocks used, not the data within the streams
*/ */
void syncWithDataSource() throws IOException { void syncWithDataSource() throws IOException {
int blocksUsed = 0; int blocksUsed = 0;
for (BATBlock sbat : _sbat_blocks) { for (BATBlock sbat : _sbat_blocks) {
ByteBuffer block = _filesystem.getBlockAt(sbat.getOurBlockIndex()); ByteBuffer block = _filesystem.getBlockAt(sbat.getOurBlockIndex());
sbat.writeData(block); sbat.writeData(block);
if (!sbat.hasFreeSectors()) { if (!sbat.hasFreeSectors()) {
blocksUsed += _filesystem.getBigBlockSizeDetails().getBATEntriesPerBlock(); blocksUsed += _filesystem.getBigBlockSizeDetails().getBATEntriesPerBlock();
} else { } else {
blocksUsed += sbat.getOccupiedSize(); blocksUsed += sbat.getOccupiedSize();
} }
} }
// Set the size on the root in terms of the number of SBAT blocks // Set the size on the root in terms of the number of SBAT blocks
// RootProperty.setSize does the sbat -> bytes conversion for us // RootProperty.setSize does the sbat -> bytes conversion for us
_filesystem._get_property_table().getRoot().setSize(blocksUsed); _filesystem._get_property_table().getRoot().setSize(blocksUsed);
} }
@Override @Override