Author: tv Date: Mon Aug 15 21:31:15 2011 New Revision: 1158024 URL: http://svn.apache.org/viewvc?rev=1158024&view=rev Log: Update IndexDisk and BlockDisk to use NIO in an attempt to fix a timing-dependent test failure. All test pass now.
Modified: commons/proper/jcs/trunk/src/changes/changes.xml commons/proper/jcs/trunk/src/java/org/apache/jcs/auxiliary/disk/block/BlockDisk.java commons/proper/jcs/trunk/src/java/org/apache/jcs/auxiliary/disk/block/BlockDiskCache.java commons/proper/jcs/trunk/src/java/org/apache/jcs/auxiliary/disk/block/BlockDiskKeyStore.java commons/proper/jcs/trunk/src/java/org/apache/jcs/auxiliary/disk/indexed/IndexedDisk.java commons/proper/jcs/trunk/src/java/org/apache/jcs/auxiliary/disk/indexed/IndexedDiskCache.java commons/proper/jcs/trunk/src/java/org/apache/jcs/utils/struct/SingleLinkedList.java commons/proper/jcs/trunk/src/test/org/apache/jcs/auxiliary/disk/block/BlockDiskCacheSameRegionConcurrentUnitTest.java commons/proper/jcs/trunk/src/test/org/apache/jcs/auxiliary/disk/indexed/DiskTestObjectUtil.java commons/proper/jcs/trunk/src/test/org/apache/jcs/auxiliary/disk/indexed/IndexDiskCacheUnitTest.java commons/proper/jcs/trunk/src/test/org/apache/jcs/utils/struct/SingleLinkedListUnitTest.java Modified: commons/proper/jcs/trunk/src/changes/changes.xml URL: http://svn.apache.org/viewvc/commons/proper/jcs/trunk/src/changes/changes.xml?rev=1158024&r1=1158023&r2=1158024&view=diff ============================================================================== --- commons/proper/jcs/trunk/src/changes/changes.xml (original) +++ commons/proper/jcs/trunk/src/changes/changes.xml Mon Aug 15 21:31:15 2011 @@ -21,8 +21,11 @@ <body> <release version="2.0" date="unreleased" description="JDK 1.5 based major release"> <action dev="tv" type="update"> - Update build files to make JCS a true Apache Commons component. The - artifact id is now commons-jcs + Update IndexDisk and BlockDisk to use NIO in an attempt to fix + a timing-dependent test failure. + </action> + <action dev="tv" type="update"> + Update build files to make JCS a true Apache Commons component. </action> <action dev="tv" type="update"> Set UTF-8 encoding on all source files Modified: commons/proper/jcs/trunk/src/java/org/apache/jcs/auxiliary/disk/block/BlockDisk.java URL: http://svn.apache.org/viewvc/commons/proper/jcs/trunk/src/java/org/apache/jcs/auxiliary/disk/block/BlockDisk.java?rev=1158024&r1=1158023&r2=1158024&view=diff ============================================================================== --- commons/proper/jcs/trunk/src/java/org/apache/jcs/auxiliary/disk/block/BlockDisk.java (original) +++ commons/proper/jcs/trunk/src/java/org/apache/jcs/auxiliary/disk/block/BlockDisk.java Mon Aug 15 21:31:15 2011 @@ -19,11 +19,16 @@ package org.apache.jcs.auxiliary.disk.bl * under the License. */ +import java.io.ByteArrayOutputStream; import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; import java.io.RandomAccessFile; import java.io.Serializable; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -55,10 +60,10 @@ public class BlockDisk * the total number of blocks that have been used. If there are no free, we will use this to * calculate the position of the next block. */ - private int numberOfBlocks = 0; + private final AtomicInteger numberOfBlocks = new AtomicInteger(0); /** Empty blocks that can be reused. */ - private final SingleLinkedList emptyBlocks = new SingleLinkedList(); + private final SingleLinkedList<Integer> emptyBlocks = new SingleLinkedList<Integer>(); /** The serializer. Uses a standard serializer by default. */ protected IElementSerializer elementSerializer = new StandardSerializer(); @@ -66,14 +71,14 @@ public class BlockDisk /** Location of the spot on disk */ private final String filepath; - /** The file handle. */ - private final RandomAccessFile raf; + /** File channel for multiple concurrent reads and writes */ + private final FileChannel fc; /** How many bytes have we put to disk */ - private long putBytes = 0; + private final AtomicLong putBytes = new AtomicLong(0); /** How many items have we put to disk */ - private long putCount = 0; + private final AtomicLong putCount = new AtomicLong(0); /** * Constructor for the Disk object @@ -85,12 +90,7 @@ public class BlockDisk public BlockDisk( File file, IElementSerializer elementSerializer ) throws FileNotFoundException { - this( file, DEFAULT_BLOCK_SIZE_BYTES ); - if ( log.isInfoEnabled() ) - { - log.info( "Used default block size [" + DEFAULT_BLOCK_SIZE_BYTES + "]" ); - } - this.elementSerializer = elementSerializer; + this( file, DEFAULT_BLOCK_SIZE_BYTES, elementSerializer ); } /** @@ -103,14 +103,7 @@ public class BlockDisk public BlockDisk( File file, int blockSizeBytes ) throws FileNotFoundException { - this.filepath = file.getAbsolutePath(); - raf = new RandomAccessFile( filepath, "rw" ); - - if ( log.isInfoEnabled() ) - { - log.info( "Constructing BlockDisk, blockSizeBytes [" + blockSizeBytes + "]" ); - } - this.blockSizeBytes = blockSizeBytes; + this( file, blockSizeBytes, new StandardSerializer() ); } /** @@ -125,14 +118,14 @@ public class BlockDisk throws FileNotFoundException { this.filepath = file.getAbsolutePath(); - raf = new RandomAccessFile( filepath, "rw" ); + RandomAccessFile raf = new RandomAccessFile( filepath, "rw" ); + this.fc = raf.getChannel(); if ( log.isInfoEnabled() ) { log.info( "Constructing BlockDisk, blockSizeBytes [" + blockSizeBytes + "]" ); } this.blockSizeBytes = blockSizeBytes; - this.elementSerializer = elementSerializer; } @@ -164,8 +157,8 @@ public class BlockDisk log.debug( "write, total pre-chunking data.length = " + data.length ); } - this.addToPutBytes( data.length ); - this.incrementPutCount(); + this.putBytes.addAndGet(data.length); + this.putCount.incrementAndGet(); // figure out how many blocks we need. int numBlocksNeeded = calculateTheNumberOfBlocksNeeded( data ); @@ -179,14 +172,14 @@ public class BlockDisk // get them from the empty list or take the next one for ( int i = 0; i < numBlocksNeeded; i++ ) { - Integer emptyBlock = (Integer) emptyBlocks.takeFirst(); + Integer emptyBlock = emptyBlocks.takeFirst(); if ( emptyBlock != null ) { blocks[i] = emptyBlock.intValue(); } else { - blocks[i] = takeNextBlock(); + blocks[i] = this.numberOfBlocks.getAndIncrement(); } } @@ -251,13 +244,13 @@ public class BlockDisk private boolean write( long position, byte[] data ) throws IOException { - synchronized ( this ) - { - raf.seek( position ); - raf.writeInt( data.length ); - raf.write( data, 0, data.length ); - } - return true; + ByteBuffer buffer = ByteBuffer.allocate(HEADER_SIZE_BYTES + data.length); + buffer.putInt(data.length); + buffer.put(data); + buffer.flip(); + int written = fc.write(buffer, position); + + return written == data.length; } /** @@ -279,19 +272,16 @@ public class BlockDisk } else { - data = new byte[0]; + ByteArrayOutputStream baos = new ByteArrayOutputStream(getBlockSizeBytes()); // get all the blocks into data for ( short i = 0; i < blockNumbers.length; i++ ) { byte[] chunk = readBlock( blockNumbers[i] ); - byte[] newTotal = new byte[data.length + chunk.length]; - // copy data into the new array - System.arraycopy( data, 0, newTotal, 0, data.length ); - // copy the chunk into the new array - System.arraycopy( chunk, 0, newTotal, data.length, chunk.length ); - // swap the new and old. - data = newTotal; + baos.write(chunk); } + + data = baos.toByteArray(); + baos.close(); } if ( log.isDebugEnabled() ) @@ -315,40 +305,42 @@ public class BlockDisk private byte[] readBlock( int block ) throws IOException { - byte[] data = null; int datalen = 0; - synchronized ( this ) - { - String message = null; - boolean corrupted = false; - long fileLength = raf.length(); - int position = calculateByteOffsetForBlock( block ); - if ( position > fileLength ) - { - corrupted = true; - message = "Record " + position + " starts past EOF."; - } - else - { - raf.seek( position ); - datalen = raf.readInt(); - if ( position + datalen > fileLength ) - { - corrupted = true; - message = "Record " + position + " exceeds file length."; - } - } + String message = null; + boolean corrupted = false; + long fileLength = fc.size(); - if ( corrupted ) + int position = calculateByteOffsetForBlock( block ); + if ( position > fileLength ) + { + corrupted = true; + message = "Record " + position + " starts past EOF."; + } + else + { + ByteBuffer datalength = ByteBuffer.allocate(HEADER_SIZE_BYTES); + fc.read(datalength, position); + datalength.flip(); + datalen = datalength.getInt(); + if ( position + datalen > fileLength ) { - log.warn( "\n The file is corrupt: " + "\n " + message ); - throw new IOException( "The File Is Corrupt, need to reset" ); + corrupted = true; + message = "Record " + position + " exceeds file length."; } + } - raf.readFully( data = new byte[datalen] ); + if ( corrupted ) + { + log.warn( "\n The file is corrupt: " + "\n " + message ); + throw new IOException( "The File Is Corrupt, need to reset" ); } - return data; + + ByteBuffer data = ByteBuffer.allocate(datalen); + fc.read(data, position + HEADER_SIZE_BYTES); + data.flip(); + + return data.array(); } /** @@ -368,35 +360,7 @@ public class BlockDisk } /** - * Add to to total put size. - * <p> - * @param length - */ - private synchronized void addToPutBytes( long length ) - { - this.putBytes += length; - } - - /** - * Thread safe increment. - */ - private synchronized void incrementPutCount() - { - this.putCount++; - } - - /** - * Returns the current number and adds one. - * <p> - * @return the block number to use. - */ - private synchronized int takeNextBlock() - { - return this.numberOfBlocks++; - } - - /** - * Calcuates the file offset for a particular block. + * Calculates the file offset for a particular block. * <p> * @param block * @return the offset for this block @@ -434,7 +398,7 @@ public class BlockDisk } /** - * Returns the raf length. + * Returns the file length. * <p> * @return the size of the file. * @exception IOException @@ -442,21 +406,32 @@ public class BlockDisk protected long length() throws IOException { - synchronized ( this ) - { - return raf.length(); - } + return fc.size(); } /** - * Closes the raf. + * Closes the file. * <p> * @exception IOException */ - protected synchronized void close() + protected void close() throws IOException { - raf.close(); + fc.close(); + } + + /** + * Resets the file. + * <p> + * @exception IOException + */ + protected void reset() + throws IOException + { + this.numberOfBlocks.set(0); + this.emptyBlocks.clear(); + fc.truncate(0); + fc.force(true); } /** @@ -464,7 +439,7 @@ public class BlockDisk */ protected int getNumberOfBlocks() { - return numberOfBlocks; + return numberOfBlocks.get(); } /** @@ -478,13 +453,15 @@ public class BlockDisk /** * @return Returns the average size of the an element inserted. */ - protected synchronized long getAveragePutSizeBytes() + protected long getAveragePutSizeBytes() { - if ( this.putCount == 0 ) + long count = this.putCount.get(); + + if (count == 0 ) { return 0; } - return this.putBytes / this.putCount; + return this.putBytes.get() / count; } /** @@ -506,8 +483,8 @@ public class BlockDisk StringBuffer buf = new StringBuffer(); buf.append( "\nBlock Disk " ); buf.append( "\n Filepath [" + filepath + "]" ); - buf.append( "\n NumberOfBlocks [" + getNumberOfBlocks() + "]" ); - buf.append( "\n BlockSizeBytes [" + getBlockSizeBytes() + "]" ); + buf.append( "\n NumberOfBlocks [" + this.numberOfBlocks.get() + "]" ); + buf.append( "\n BlockSizeBytes [" + this.blockSizeBytes + "]" ); buf.append( "\n Put Bytes [" + this.putBytes + "]" ); buf.append( "\n Put Count [" + this.putCount + "]" ); buf.append( "\n Average Size [" + getAveragePutSizeBytes() + "]" ); Modified: commons/proper/jcs/trunk/src/java/org/apache/jcs/auxiliary/disk/block/BlockDiskCache.java URL: http://svn.apache.org/viewvc/commons/proper/jcs/trunk/src/java/org/apache/jcs/auxiliary/disk/block/BlockDiskCache.java?rev=1158024&r1=1158023&r2=1158024&view=diff ============================================================================== --- commons/proper/jcs/trunk/src/java/org/apache/jcs/auxiliary/disk/block/BlockDiskCache.java (original) +++ commons/proper/jcs/trunk/src/java/org/apache/jcs/auxiliary/disk/block/BlockDiskCache.java Mon Aug 15 21:31:15 2011 @@ -79,10 +79,10 @@ public class BlockDiskCache private BlockDiskKeyStore keyStore; /** - * Use this lock to synchronize reads and writes to the underlying storage mechansism. We don't + * Use this lock to synchronize reads and writes to the underlying storage mechanism. We don't * need a reentrant lock, since we only lock one level. */ - private final ReentrantReadWriteLock storageLock = new ReentrantReadWriteLock(); + private final ReentrantReadWriteLock storageLock = new ReentrantReadWriteLock(true); /** * Constructs the BlockDisk after setting up the root directory. @@ -151,7 +151,7 @@ public class BlockDiskCache log.info( logCacheName + "Block Disk Cache is alive." ); } } - catch ( Exception e ) + catch ( IOException e ) { log.error( logCacheName + "Failure initializing for fileName: " + fileName + " and root directory: " + rootDirName, e ); @@ -169,6 +169,9 @@ public class BlockDiskCache boolean alright = false; // simply try to read a few. If it works, then the file is probably ok. // TODO add more. + + storageLock.readLock().lock(); + try { int maxToTest = 100; @@ -191,6 +194,11 @@ public class BlockDiskCache log.warn( logCacheName + "Problem verifying disk. Message [" + e.getMessage() + "]" ); alright = false; } + finally + { + storageLock.readLock().unlock(); + } + return alright; } @@ -205,10 +213,11 @@ public class BlockDiskCache { GroupId groupId = new GroupId( cacheName, groupName ); HashSet<Serializable> keys = new HashSet<Serializable>(); + + storageLock.readLock().lock(); + try { - storageLock.readLock().lock(); - for ( Serializable key : this.keyStore.keySet()) { if ( key instanceof GroupAttrName && ( (GroupAttrName) key ).groupId.equals( groupId ) ) @@ -217,10 +226,6 @@ public class BlockDiskCache } } } - catch ( Exception e ) - { - log.error( logCacheName + "Failure getting from disk, group = " + groupName, e ); - } finally { storageLock.readLock().unlock(); @@ -240,34 +245,29 @@ public class BlockDiskCache public Map<Serializable, ICacheElement> processGetMatching( String pattern ) { Map<Serializable, ICacheElement> elements = new HashMap<Serializable, ICacheElement>(); + + Object[] keyArray = null; + storageLock.readLock().lock(); try { - Object[] keyArray = null; - storageLock.readLock().lock(); - try - { - keyArray = this.keyStore.keySet().toArray(); - } - finally - { - storageLock.readLock().unlock(); - } + keyArray = this.keyStore.keySet().toArray(); + } + finally + { + storageLock.readLock().unlock(); + } - Set<Serializable> matchingKeys = getKeyMatcher().getMatchingKeysFromArray( pattern, keyArray ); + Set<Serializable> matchingKeys = getKeyMatcher().getMatchingKeysFromArray( pattern, keyArray ); - for (Serializable key : matchingKeys) + for (Serializable key : matchingKeys) + { + ICacheElement element = processGet( key ); + if ( element != null ) { - ICacheElement element = processGet( key ); - if ( element != null ) - { - elements.put( key, element ); - } + elements.put( key, element ); } } - catch ( Exception e ) - { - log.error( logCacheName + "Failure getting matching from disk, pattern = " + pattern, e ); - } + return elements; } @@ -313,20 +313,14 @@ public class BlockDiskCache } ICacheElement object = null; + storageLock.readLock().lock(); + try { - storageLock.readLock().lock(); - try + int[] ded = this.keyStore.get( key ); + if ( ded != null ) { - int[] ded = this.keyStore.get( key ); - if ( ded != null ) - { - object = (ICacheElement) this.dataFile.read( ded ); - } - } - finally - { - storageLock.readLock().unlock(); + object = (ICacheElement) this.dataFile.read( ded ); } } catch ( IOException ioe ) @@ -338,6 +332,10 @@ public class BlockDiskCache { log.error( logCacheName + "Failure getting from disk, key = " + key, e ); } + finally + { + storageLock.readLock().unlock(); + } return object; } @@ -345,12 +343,11 @@ public class BlockDiskCache /** * Writes an element to disk. The program flow is as follows: * <ol> - * <li>Aquire write lock.</li> <li>See id an item exists for this key.</li> <li>If an itme + * <li>Acquire write lock.</li> <li>See id an item exists for this key.</li> <li>If an item * already exists, add its blocks to the remove list.</li> <li>Have the Block disk write the * item.</li> <li>Create a descriptor and add it to the key map.</li> <li>Release the write * lock.</li> * </ol> - * (non-Javadoc) * @param element * @see org.apache.jcs.auxiliary.disk.AbstractDiskCache#doUpdate(org.apache.jcs.engine.behavior.ICacheElement) */ @@ -367,37 +364,37 @@ public class BlockDiskCache } int[] old = null; - try - { - // make sure this only locks for one particular cache region - storageLock.writeLock().lock(); - try - { - old = this.keyStore.get( element.getKey() ); - if ( old != null ) - { - this.dataFile.freeBlocks( old ); - } + // make sure this only locks for one particular cache region + storageLock.writeLock().lock(); - int[] blocks = this.dataFile.write( element ); + try + { + old = this.keyStore.get( element.getKey() ); - this.keyStore.put( element.getKey(), blocks ); - } - finally + if ( old != null ) { - storageLock.writeLock().unlock(); + this.dataFile.freeBlocks( old ); } + int[] blocks = this.dataFile.write( element ); + + this.keyStore.put( element.getKey(), blocks ); + if ( log.isDebugEnabled() ) { log.debug( logCacheName + "Put to file [" + fileName + "] key [" + element.getKey() + "]" ); } } - catch ( Exception e ) + catch ( IOException e ) { log.error( logCacheName + "Failure updating element, key: " + element.getKey() + " old: " + Arrays.toString(old), e ); } + finally + { + storageLock.writeLock().unlock(); + } + if ( log.isDebugEnabled() ) { log.debug( logCacheName + "Storing element on disk, key: " + element.getKey() ); @@ -427,14 +424,14 @@ public class BlockDiskCache boolean reset = false; boolean removed = false; + + storageLock.writeLock().lock(); + try { - storageLock.writeLock().lock(); - if ( key instanceof String && key.toString().endsWith( CacheConstants.NAME_COMPONENT_DELIMITER ) ) { // remove all keys of the same name group. - Iterator<Map.Entry<Serializable, int[]>> iter = this.keyStore.entrySet().iterator(); while ( iter.hasNext() ) @@ -514,15 +511,7 @@ public class BlockDiskCache @Override protected void processRemoveAll() { - try - { - reset(); - } - catch ( Exception e ) - { - log.error( logCacheName + "Problem removing all.", e ); - reset(); - } + reset(); } /** @@ -630,7 +619,7 @@ public class BlockDiskCache { if ( log.isWarnEnabled() ) { - log.warn( logCacheName + "Reseting cache" ); + log.warn( logCacheName + "Resetting cache" ); } try @@ -639,27 +628,14 @@ public class BlockDiskCache if ( dataFile != null ) { - dataFile.close(); - } - // TODO have the BlockDisk do this itself - File dataFileTemp = new File( this.rootDirectory, fileName + ".data" ); - dataFileTemp.delete(); - - if ( this.blockDiskCacheAttributes.getBlockSizeBytes() > 0 ) - { - this.dataFile = new BlockDisk( new File( rootDirectory, fileName + ".data" ), - this.blockDiskCacheAttributes.getBlockSizeBytes() ); - } - else - { - this.dataFile = new BlockDisk( new File( rootDirectory, fileName + ".data" ), getElementSerializer() ); + dataFile.reset(); } this.keyStore.reset(); } - catch ( Exception e ) + catch ( IOException e ) { - log.error( logCacheName + "Failure reseting state", e ); + log.error( logCacheName + "Failure resetting state", e ); } finally { Modified: commons/proper/jcs/trunk/src/java/org/apache/jcs/auxiliary/disk/block/BlockDiskKeyStore.java URL: http://svn.apache.org/viewvc/commons/proper/jcs/trunk/src/java/org/apache/jcs/auxiliary/disk/block/BlockDiskKeyStore.java?rev=1158024&r1=1158023&r2=1158024&view=diff ============================================================================== --- commons/proper/jcs/trunk/src/java/org/apache/jcs/auxiliary/disk/block/BlockDiskKeyStore.java (original) +++ commons/proper/jcs/trunk/src/java/org/apache/jcs/auxiliary/disk/block/BlockDiskKeyStore.java Mon Aug 15 21:31:15 2011 @@ -59,7 +59,7 @@ public class BlockDiskKeyStore private Map<Serializable, int[]> keyHash; /** The file where we persist the keys */ - private File keyFile; + private final File keyFile; /** The name to prefix log messages with. */ protected final String logCacheName; @@ -73,9 +73,6 @@ public class BlockDiskKeyStore /** we need this so we can communicate free blocks to the data store when keys fall off the LRU */ protected final BlockDiskCache blockDiskCache; - /** The root directory in which the keyFile lives */ - private final File rootDirectory; - /** * The background key persister, one for all regions. */ @@ -86,10 +83,8 @@ public class BlockDiskKeyStore * <p> * @param cacheAttributes * @param blockDiskCache used for freeing - * @throws Exception */ public BlockDiskKeyStore( BlockDiskCacheAttributes cacheAttributes, BlockDiskCache blockDiskCache ) - throws Exception { this.blockDiskCacheAttributes = cacheAttributes; this.logCacheName = "Region [" + this.blockDiskCacheAttributes.getCacheName() + "] "; @@ -98,8 +93,8 @@ public class BlockDiskKeyStore this.blockDiskCache = blockDiskCache; String rootDirName = cacheAttributes.getDiskPath(); - this.rootDirectory = new File( rootDirName ); - this.rootDirectory.mkdirs(); + File rootDirectory = new File( rootDirName ); + rootDirectory.mkdirs(); if ( log.isInfoEnabled() ) { @@ -162,28 +157,28 @@ public class BlockDiskKeyStore + numKeys + "]" ); } - keyFile.delete(); - - keyFile = new File( rootDirectory, fileName + ".key" ); - FileOutputStream fos = new FileOutputStream( keyFile ); - BufferedOutputStream bos = new BufferedOutputStream( fos, 1024 ); - ObjectOutputStream oos = new ObjectOutputStream( bos ); - try + synchronized (keyFile) { - // don't need to synchronize, since the underlying collection makes a copy - for (Map.Entry<Serializable, int[]> entry : keyHash.entrySet()) + FileOutputStream fos = new FileOutputStream( keyFile ); + BufferedOutputStream bos = new BufferedOutputStream( fos, 65536 ); + ObjectOutputStream oos = new ObjectOutputStream( bos ); + try { - BlockDiskElementDescriptor descriptor = new BlockDiskElementDescriptor(); - descriptor.setKey( entry.getKey() ); - descriptor.setBlocks( entry.getValue() ); - // stream these out in the loop. - oos.writeObject( descriptor ); + // don't need to synchronize, since the underlying collection makes a copy + for (Map.Entry<Serializable, int[]> entry : keyHash.entrySet()) + { + BlockDiskElementDescriptor descriptor = new BlockDiskElementDescriptor(); + descriptor.setKey( entry.getKey() ); + descriptor.setBlocks( entry.getValue() ); + // stream these out in the loop. + oos.writeObject( descriptor ); + } + } + finally + { + oos.flush(); + oos.close(); } - } - finally - { - oos.flush(); - oos.close(); } if ( log.isInfoEnabled() ) @@ -203,12 +198,12 @@ public class BlockDiskKeyStore */ protected void reset() { - File keyFileTemp = new File( this.rootDirectory, fileName + ".key" ); - keyFileTemp.delete(); - - keyFile = new File( this.rootDirectory, fileName + ".key" ); + synchronized (keyFile) + { + clearMemoryMap(); + saveKeys(); + } - initKeyMap(); } /** @@ -248,11 +243,8 @@ public class BlockDiskKeyStore /** * Loads the keys from the .key file. The keys are stored individually on disk. They are added * one by one to an LRUMap.. - * <p> - * @throws InterruptedException */ protected void loadKeys() - throws InterruptedException { if ( log.isInfoEnabled() ) { @@ -266,38 +258,41 @@ public class BlockDiskKeyStore HashMap<Serializable, int[]> keys = new HashMap<Serializable, int[]>(); - FileInputStream fis = new FileInputStream( keyFile ); - BufferedInputStream bis = new BufferedInputStream( fis ); - ObjectInputStream ois = new ObjectInputStream( bis ); - try + synchronized (keyFile) { - while ( true ) + FileInputStream fis = new FileInputStream( keyFile ); + BufferedInputStream bis = new BufferedInputStream( fis ); + ObjectInputStream ois = new ObjectInputStream( bis ); + try { - BlockDiskElementDescriptor descriptor = (BlockDiskElementDescriptor) ois.readObject(); - if ( descriptor != null ) + while ( true ) { - keys.put( descriptor.getKey(), descriptor.getBlocks() ); + BlockDiskElementDescriptor descriptor = (BlockDiskElementDescriptor) ois.readObject(); + if ( descriptor != null ) + { + keys.put( descriptor.getKey(), descriptor.getBlocks() ); + } } } - } - catch ( EOFException eof ) - { - // nothing - } - finally - { - ois.close(); + catch ( EOFException eof ) + { + // nothing + } + finally + { + ois.close(); + } } if ( !keys.isEmpty() ) { + keyHash.putAll( keys ); + if ( log.isDebugEnabled() ) { log.debug( logCacheName + "Found " + keys.size() + " in keys file." ); } - keyHash.putAll( keys ); - if ( log.isInfoEnabled() ) { log.info( logCacheName + "Loaded keys from [" + fileName + "], key count: " + keyHash.size() @@ -369,7 +364,7 @@ public class BlockDiskKeyStore * @param key * @return BlockDiskElementDescriptor if it was present, else null */ - public int[] remove( Object key ) + public int[] remove( Serializable key ) { return this.keyHash.remove( key ); } Modified: commons/proper/jcs/trunk/src/java/org/apache/jcs/auxiliary/disk/indexed/IndexedDisk.java URL: http://svn.apache.org/viewvc/commons/proper/jcs/trunk/src/java/org/apache/jcs/auxiliary/disk/indexed/IndexedDisk.java?rev=1158024&r1=1158023&r2=1158024&view=diff ============================================================================== --- commons/proper/jcs/trunk/src/java/org/apache/jcs/auxiliary/disk/indexed/IndexedDisk.java (original) +++ commons/proper/jcs/trunk/src/java/org/apache/jcs/auxiliary/disk/indexed/IndexedDisk.java Mon Aug 15 21:31:15 2011 @@ -24,6 +24,8 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.io.RandomAccessFile; import java.io.Serializable; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -33,8 +35,8 @@ import org.apache.jcs.utils.serializatio /** Provides thread safe access to the underlying random access file. */ class IndexedDisk { - /** The size of the header in bytes. The header describes the length of the entry. */ - public static final int RECORD_HEADER = 4; + /** The size of the header that indicates the amount of data stored in an occupied block. */ + public static final byte HEADER_SIZE_BYTES = 4; /** The serializer. Uses a standard serializer by default. */ protected IElementSerializer elementSerializer = new StandardSerializer(); @@ -46,10 +48,7 @@ class IndexedDisk private final String filepath; /** The data file. */ - private final RandomAccessFile raf; - - /** read buffer */ - private final byte[] buffer = new byte[16384]; // 16K + private final FileChannel fc; /** * Constructor for the Disk object @@ -63,7 +62,8 @@ class IndexedDisk { this.filepath = file.getAbsolutePath(); this.elementSerializer = elementSerializer; - raf = new RandomAccessFile( filepath, "rw" ); + RandomAccessFile raf = new RandomAccessFile( filepath, "rw" ); + this.fc = raf.getChannel(); } /** @@ -80,43 +80,43 @@ class IndexedDisk protected Serializable readObject( IndexedDiskElementDescriptor ded ) throws IOException, ClassNotFoundException { - byte[] data = null; - synchronized ( this ) + String message = null; + boolean corrupted = false; + long fileLength = fc.size(); + if ( ded.pos > fileLength ) + { + corrupted = true; + message = "Record " + ded + " starts past EOF."; + } + else { - String message = null; - boolean corrupted = false; - long fileLength = raf.length(); - if ( ded.pos > fileLength ) + ByteBuffer datalength = ByteBuffer.allocate(HEADER_SIZE_BYTES); + fc.read(datalength, ded.pos); + datalength.flip(); + int datalen = datalength.getInt(); + if ( ded.len != datalen ) { corrupted = true; - message = "Record " + ded + " starts past EOF."; - } - else - { - raf.seek( ded.pos ); - int datalen = raf.readInt(); - if ( ded.len != datalen ) - { - corrupted = true; - message = "Record " + ded + " does not match data length on disk (" + datalen + ")"; - } - else if ( ded.pos + ded.len > fileLength ) - { - corrupted = true; - message = "Record " + ded + " exceeds file length."; - } + message = "Record " + ded + " does not match data length on disk (" + datalen + ")"; } - - if ( corrupted ) + else if ( ded.pos + ded.len > fileLength ) { - log.warn( "\n The file is corrupt: " + "\n " + message ); - throw new IOException( "The File Is Corrupt, need to reset" ); + corrupted = true; + message = "Record " + ded + " exceeds file length."; } + } - raf.readFully( data = new byte[ded.len] ); + if ( corrupted ) + { + log.warn( "\n The file is corrupt: " + "\n " + message ); + throw new IOException( "The File Is Corrupt, need to reset" ); } - return (Serializable) elementSerializer.deSerialize( data ); + ByteBuffer data = ByteBuffer.allocate(ded.len); + fc.read(data, ded.pos + HEADER_SIZE_BYTES); + data.flip(); + + return (Serializable) elementSerializer.deSerialize( data.array() ); } /** @@ -129,41 +129,40 @@ class IndexedDisk protected void move( final IndexedDiskElementDescriptor ded, final long newPosition ) throws IOException { - synchronized ( this ) - { - raf.seek( ded.pos ); - int length = raf.readInt(); + ByteBuffer datalength = ByteBuffer.allocate(HEADER_SIZE_BYTES); + fc.read(datalength, ded.pos); + datalength.flip(); + int length = datalength.getInt(); - if ( length != ded.len ) - { - throw new IOException( "Mismatched memory and disk length (" + length + ") for " + ded ); - } - - // TODO: more checks? + if ( length != ded.len ) + { + throw new IOException( "Mismatched memory and disk length (" + length + ") for " + ded ); + } - long readPos = ded.pos; - long writePos = newPosition; + // TODO: more checks? - // header len + data len - int remaining = RECORD_HEADER + length; + long readPos = ded.pos; + long writePos = newPosition; - while ( remaining > 0 ) - { - // chunk it - int chunkSize = Math.min( remaining, buffer.length ); - raf.seek( readPos ); - raf.readFully( buffer, 0, chunkSize ); - - raf.seek( writePos ); - raf.write( buffer, 0, chunkSize ); - - writePos += chunkSize; - readPos += chunkSize; - remaining -= chunkSize; - } + // header len + data len + int remaining = HEADER_SIZE_BYTES + length; + ByteBuffer buffer = ByteBuffer.allocate(16384); - ded.pos = newPosition; + while ( remaining > 0 ) + { + // chunk it + int chunkSize = Math.min( remaining, buffer.capacity() ); + fc.read(buffer, readPos); + buffer.flip(); + fc.write(buffer, writePos); + buffer.clear(); + + writePos += chunkSize; + readPos += chunkSize; + remaining -= chunkSize; } + + ded.pos = newPosition; } /** @@ -181,11 +180,7 @@ class IndexedDisk if ( log.isTraceEnabled() ) { log.trace( "write> pos=" + pos ); - - synchronized (this) - { - log.trace( raf + " -- data.length = " + data.length ); - } + log.trace( fc + " -- data.length = " + data.length ); } if ( data.length != ded.len ) @@ -193,13 +188,13 @@ class IndexedDisk throw new IOException( "Mismatched descriptor and data lengths" ); } - synchronized ( this ) - { - raf.seek( pos ); - raf.writeInt( data.length ); - raf.write( data, 0, ded.len ); - } - return true; + ByteBuffer buffer = ByteBuffer.allocate(HEADER_SIZE_BYTES + data.length); + buffer.putInt(data.length); + buffer.put(data); + buffer.flip(); + int written = fc.write(buffer, pos); + + return written == data.length; } /** @@ -228,10 +223,7 @@ class IndexedDisk protected long length() throws IOException { - synchronized ( this ) - { - return raf.length(); - } + return fc.size(); } /** @@ -239,10 +231,10 @@ class IndexedDisk * <p> * @exception IOException */ - protected synchronized void close() + protected void close() throws IOException { - raf.close(); + fc.close(); } /** @@ -257,8 +249,8 @@ class IndexedDisk { log.debug( "Resetting Indexed File [" + filepath + "]" ); } - raf.setLength(0); - raf.seek(0); + fc.truncate(0); + fc.force(true); } /** @@ -267,14 +259,14 @@ class IndexedDisk * @param length the new length of the file * @throws IOException */ - protected synchronized void truncate( long length ) + protected void truncate( long length ) throws IOException { if ( log.isInfoEnabled() ) { log.info( "Trucating file [" + filepath + "] to " + length ); } - raf.setLength( length ); + fc.truncate( length ); } /** Modified: commons/proper/jcs/trunk/src/java/org/apache/jcs/auxiliary/disk/indexed/IndexedDiskCache.java URL: http://svn.apache.org/viewvc/commons/proper/jcs/trunk/src/java/org/apache/jcs/auxiliary/disk/indexed/IndexedDiskCache.java?rev=1158024&r1=1158023&r2=1158024&view=diff ============================================================================== --- commons/proper/jcs/trunk/src/java/org/apache/jcs/auxiliary/disk/indexed/IndexedDiskCache.java (original) +++ commons/proper/jcs/trunk/src/java/org/apache/jcs/auxiliary/disk/indexed/IndexedDiskCache.java Mon Aug 15 21:31:15 2011 @@ -327,7 +327,7 @@ public class IndexedDiskCache HashMap<Serializable, IndexedDiskElementDescriptor> keys = (HashMap<Serializable, IndexedDiskElementDescriptor>) keyFile.readObject( new IndexedDiskElementDescriptor( 0, (int) keyFile.length() - - IndexedDisk.RECORD_HEADER ) ); + - IndexedDisk.HEADER_SIZE_BYTES ) ); if ( keys != null ) { @@ -384,7 +384,7 @@ public class IndexedDiskCache { IndexedDiskElementDescriptor ded = e.getValue(); - isOk = ( ded.pos + IndexedDisk.RECORD_HEADER + ded.len <= fileLength ); + isOk = ( ded.pos + IndexedDisk.HEADER_SIZE_BYTES + ded.len <= fileLength ); if ( !isOk ) { @@ -438,7 +438,7 @@ public class IndexedDiskCache } else { - expectedNextPos = ded.pos + IndexedDisk.RECORD_HEADER + ded.len; + expectedNextPos = ded.pos + IndexedDisk.HEADER_SIZE_BYTES + ded.len; } } long end = System.currentTimeMillis(); @@ -1335,7 +1335,7 @@ public class IndexedDiskCache { dataFile.move( defragList[i], expectedNextPos ); } - expectedNextPos = defragList[i].pos + IndexedDisk.RECORD_HEADER + defragList[i].len; + expectedNextPos = defragList[i].pos + IndexedDisk.HEADER_SIZE_BYTES + defragList[i].len; } finally { @@ -1440,7 +1440,7 @@ public class IndexedDiskCache { if ( ded != null ) { - int amount = ded.len + IndexedDisk.RECORD_HEADER; + int amount = ded.len + IndexedDisk.HEADER_SIZE_BYTES; if ( add ) { Modified: commons/proper/jcs/trunk/src/java/org/apache/jcs/utils/struct/SingleLinkedList.java URL: http://svn.apache.org/viewvc/commons/proper/jcs/trunk/src/java/org/apache/jcs/utils/struct/SingleLinkedList.java?rev=1158024&r1=1158023&r2=1158024&view=diff ============================================================================== --- commons/proper/jcs/trunk/src/java/org/apache/jcs/utils/struct/SingleLinkedList.java (original) +++ commons/proper/jcs/trunk/src/java/org/apache/jcs/utils/struct/SingleLinkedList.java Mon Aug 15 21:31:15 2011 @@ -28,7 +28,7 @@ import org.apache.commons.logging.LogFac * <p> * @author Aaron Smuts */ -public class SingleLinkedList +public class SingleLinkedList<T> { /** The logger */ private static final Log log = LogFactory.getLog( SingleLinkedList.class ); @@ -37,10 +37,10 @@ public class SingleLinkedList private final Object lock = new Object(); /** the head of the queue */ - private Node head = new Node(); + private Node<T> head = new Node<T>(); /** the end of the queue */ - private Node tail = head; + private Node<T> tail = head; /** The size of the list */ private int size = 0; @@ -50,7 +50,7 @@ public class SingleLinkedList * <p> * @return null if the list is empty. */ - public Object takeFirst() + public T takeFirst() { synchronized ( lock ) { @@ -60,9 +60,9 @@ public class SingleLinkedList return null; } - Node node = head.next; + Node<T> node = head.next; - Object value = node.payload; + T value = node.payload; if ( log.isDebugEnabled() ) { @@ -85,9 +85,9 @@ public class SingleLinkedList * <p> * @param payload */ - public void addLast( Object payload ) + public void addLast( T payload ) { - Node newNode = new Node(); + Node<T> newNode = new Node<T>(); newNode.payload = payload; @@ -116,13 +116,13 @@ public class SingleLinkedList * <p> * @author Aaron Smuts */ - protected static class Node + protected static class Node<T> { /** next in the list */ - Node next = null; + Node<T> next = null; /** The data in this node */ - Object payload; + T payload; } /** Modified: commons/proper/jcs/trunk/src/test/org/apache/jcs/auxiliary/disk/block/BlockDiskCacheSameRegionConcurrentUnitTest.java URL: http://svn.apache.org/viewvc/commons/proper/jcs/trunk/src/test/org/apache/jcs/auxiliary/disk/block/BlockDiskCacheSameRegionConcurrentUnitTest.java?rev=1158024&r1=1158023&r2=1158024&view=diff ============================================================================== --- commons/proper/jcs/trunk/src/test/org/apache/jcs/auxiliary/disk/block/BlockDiskCacheSameRegionConcurrentUnitTest.java (original) +++ commons/proper/jcs/trunk/src/test/org/apache/jcs/auxiliary/disk/block/BlockDiskCacheSameRegionConcurrentUnitTest.java Mon Aug 15 21:31:15 2011 @@ -67,6 +67,7 @@ public class BlockDiskCacheSameRegionCon suite.addTest( new BlockDiskCacheSameRegionConcurrentUnitTest( "testBlockDiskCache1" ) { + @Override public void runTest() throws Exception { @@ -76,6 +77,7 @@ public class BlockDiskCacheSameRegionCon suite.addTest( new BlockDiskCacheSameRegionConcurrentUnitTest( "testBlockDiskCache2" ) { + @Override public void runTest() throws Exception { @@ -85,6 +87,7 @@ public class BlockDiskCacheSameRegionCon suite.addTest( new BlockDiskCacheSameRegionConcurrentUnitTest( "testBlockDiskCache3" ) { + @Override public void runTest() throws Exception { @@ -94,6 +97,7 @@ public class BlockDiskCacheSameRegionCon suite.addTest( new BlockDiskCacheSameRegionConcurrentUnitTest( "testBlockDiskCache4" ) { + @Override public void runTest() throws Exception { @@ -109,10 +113,14 @@ public class BlockDiskCacheSameRegionCon * <p> * @throws Exception */ + @Override public void setUp() throws Exception { JCS.setConfigFilename( "/TestBlockDiskCacheCon.ccf" ); + + // tv: For some reason this cleanup does not work as expected so the test fails if + // the data files already exist JCS.getInstance( "blockRegion4" ).clear(); } Modified: commons/proper/jcs/trunk/src/test/org/apache/jcs/auxiliary/disk/indexed/DiskTestObjectUtil.java URL: http://svn.apache.org/viewvc/commons/proper/jcs/trunk/src/test/org/apache/jcs/auxiliary/disk/indexed/DiskTestObjectUtil.java?rev=1158024&r1=1158023&r2=1158024&view=diff ============================================================================== --- commons/proper/jcs/trunk/src/test/org/apache/jcs/auxiliary/disk/indexed/DiskTestObjectUtil.java (original) +++ commons/proper/jcs/trunk/src/test/org/apache/jcs/auxiliary/disk/indexed/DiskTestObjectUtil.java Mon Aug 15 21:31:15 2011 @@ -49,7 +49,7 @@ public class DiskTestObjectUtil long total = 0; for ( int i = 0; i < endPosition; i++ ) { - int tileSize = serializer.serialize( testObjects[i] ).length + IndexedDisk.RECORD_HEADER; + int tileSize = serializer.serialize( testObjects[i] ).length + IndexedDisk.HEADER_SIZE_BYTES; total += tileSize; } return total; @@ -85,7 +85,7 @@ public class DiskTestObjectUtil long total = 0; for ( int i = startPosition; i < endPosition; i++ ) { - int tileSize = serializer.serialize( elements[i] ).length + IndexedDisk.RECORD_HEADER; + int tileSize = serializer.serialize( elements[i] ).length + IndexedDisk.HEADER_SIZE_BYTES; total += tileSize; } return total; Modified: commons/proper/jcs/trunk/src/test/org/apache/jcs/auxiliary/disk/indexed/IndexDiskCacheUnitTest.java URL: http://svn.apache.org/viewvc/commons/proper/jcs/trunk/src/test/org/apache/jcs/auxiliary/disk/indexed/IndexDiskCacheUnitTest.java?rev=1158024&r1=1158023&r2=1158024&view=diff ============================================================================== --- commons/proper/jcs/trunk/src/test/org/apache/jcs/auxiliary/disk/indexed/IndexDiskCacheUnitTest.java (original) +++ commons/proper/jcs/trunk/src/test/org/apache/jcs/auxiliary/disk/indexed/IndexDiskCacheUnitTest.java Mon Aug 15 21:31:15 2011 @@ -223,7 +223,7 @@ public class IndexDiskCacheUnitTest for ( int i = 0; i < numDescriptors; i++ ) { IndexedDiskElementDescriptor descriptor = new IndexedDiskElementDescriptor( pos, i * 2 ); - pos = pos + ( i * 2 ) + IndexedDisk.RECORD_HEADER; + pos = pos + ( i * 2 ) + IndexedDisk.HEADER_SIZE_BYTES; sortedDescriptors[i] = descriptor; } Modified: commons/proper/jcs/trunk/src/test/org/apache/jcs/utils/struct/SingleLinkedListUnitTest.java URL: http://svn.apache.org/viewvc/commons/proper/jcs/trunk/src/test/org/apache/jcs/utils/struct/SingleLinkedListUnitTest.java?rev=1158024&r1=1158023&r2=1158024&view=diff ============================================================================== --- commons/proper/jcs/trunk/src/test/org/apache/jcs/utils/struct/SingleLinkedListUnitTest.java (original) +++ commons/proper/jcs/trunk/src/test/org/apache/jcs/utils/struct/SingleLinkedListUnitTest.java Mon Aug 15 21:31:15 2011 @@ -35,13 +35,13 @@ public class SingleLinkedListUnitTest public void testTakeFromEmptyList() { // SETUP - SingleLinkedList list = new SingleLinkedList(); + SingleLinkedList<Object> list = new SingleLinkedList<Object>(); // DO WORK Object result = list.takeFirst(); // VERIFY - assertNull( "Shounldn't have anything.", result ); + assertNull( "Shouldn't have anything.", result ); } /** @@ -50,7 +50,7 @@ public class SingleLinkedListUnitTest public void testAddABunchAndTakeFromList() { // SETUP - SingleLinkedList list = new SingleLinkedList(); + SingleLinkedList<Integer> list = new SingleLinkedList<Integer>(); // DO WORK int numToPut = 100; @@ -60,19 +60,19 @@ public class SingleLinkedListUnitTest } // VERIFY - assertEquals( "Wrong nubmer in list.", numToPut, list.size() ); + assertEquals( "Wrong number in list.", numToPut, list.size() ); for ( int i = 0; i < numToPut; i++ ) { - Object result = list.takeFirst(); + Integer result = list.takeFirst(); assertEquals( "Wrong value returned.", Integer.valueOf( i ), result ); } // DO WORK - Object result = list.takeFirst(); + Integer result = list.takeFirst(); // VERIFY - assertNull( "Shounldn't have anything left.", result ); + assertNull( "Shouldn't have anything left.", result ); } /** @@ -81,7 +81,7 @@ public class SingleLinkedListUnitTest public void testAddABunchAndClear() { // SETUP - SingleLinkedList list = new SingleLinkedList(); + SingleLinkedList<Integer> list = new SingleLinkedList<Integer>(); // DO WORK int numToPut = 100; @@ -91,14 +91,14 @@ public class SingleLinkedListUnitTest } // VERIFY - assertEquals( "Wrong nubmer in list.", numToPut, list.size() ); + assertEquals( "Wrong number in list.", numToPut, list.size() ); // DO WORK list.clear(); - Object result = list.takeFirst(); + Integer result = list.takeFirst(); // VERIFY - assertEquals( "Wrong nubmer in list.", 0, list.size() ); - assertNull( "Shounldn't have anything left.", result ); + assertEquals( "Wrong number in list.", 0, list.size() ); + assertNull( "Shouldn't have anything left.", result ); } }