StackOverflow2013

Note that there are some explanatory texts on larger screens.

plurals

PO
text
Body
copied!<p>A memory mapped file will be fastest... something like this:</p> <pre><code> final File file; final FileChannel channel; final MappedByteBuffer buffer; file = new File(fileName); fin = new FileInputStream(file); channel = fin.getChannel(); buffer = channel.map(MapMode.READ_ONLY, 0, file.length()); </code></pre> <p>and then proceed to read from the byte buffer.</p> <p>This will be significantly faster than <code>FileInputStream</code> or <code>FileReader</code>.</p> <p>EDIT: </p> <p>After a bit of investigation with this it turns out that, depending on your OS, you might be better off using a new <code>BufferedInputStream(new FileInputStream(file))</code> instead. However reading the whole thing all at once into a char[] the size of the file sounds like the worst way.</p> <p>So <code>BufferedInputStream</code> should give roughly consistent performance on all platforms, while the memory mapped file may be slow or fast depending on the underlying OS. As with everything that is performance critical you should test your code and see what works best.</p> <p>EDIT:</p> <p>Ok here are some tests (the first one is done twice to get the files into the disk cache).</p> <p>I ran it on the rt.jar class files, extracted to the hard drive, this is under Windows 7 beta x64. That is 16784 files with a total of 94,706,637 bytes.</p> <p>First the results...</p> <p>(remember the first is repeated to get the disk cache setup)</p> <ul> <li><p>ArrayTest</p> <ul> <li>time = 83016</li> <li>bytes = 118641472</li> </ul></li> <li><p>ArrayTest </p> <ul> <li>time = 46570</li> <li>bytes = 118641472</li> </ul></li> <li><p>DataInputByteAtATime</p> <ul> <li>time = 74735</li> <li>bytes = 118641472</li> </ul></li> <li><p>DataInputReadFully</p> <ul> <li>time = 8953</li> <li>bytes = 118641472</li> </ul></li> <li><p>MemoryMapped</p> <ul> <li>time = 2320</li> <li>bytes = 118641472</li> </ul></li> </ul> <p>Here is the code...</p> <pre><code>import java.io.BufferedInputStream; import java.io.DataInputStream; import java.io.File; import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; import java.nio.MappedByteBuffer; import java.nio.channels.FileChannel; import java.nio.channels.FileChannel.MapMode; import java.util.HashSet; import java.util.Set; public class Main { public static void main(final String[] argv) { ArrayTest.main(argv); ArrayTest.main(argv); DataInputByteAtATime.main(argv); DataInputReadFully.main(argv); MemoryMapped.main(argv); } } abstract class Test { public final void run(final File root) { final Set<File> files; final long size; final long start; final long end; final long total; files = new HashSet<File>(); getFiles(root, files); start = System.currentTimeMillis(); size = readFiles(files); end = System.currentTimeMillis(); total = end - start; System.out.println(getClass().getName()); System.out.println("time = " + total); System.out.println("bytes = " + size); } private void getFiles(final File dir, final Set<File> files) { final File[] childeren; childeren = dir.listFiles(); for(final File child : childeren) { if(child.isFile()) { files.add(child); } else { getFiles(child, files); } } } private long readFiles(final Set<File> files) { long size; size = 0; for(final File file : files) { size += readFile(file); } return (size); } protected abstract long readFile(File file); } class ArrayTest extends Test { public static void main(final String[] argv) { final Test test; test = new ArrayTest(); test.run(new File(argv[0])); } protected long readFile(final File file) { InputStream stream; stream = null; try { final byte[] data; int soFar; int sum; stream = new BufferedInputStream(new FileInputStream(file)); data = new byte[(int)file.length()]; soFar = 0; do { soFar += stream.read(data, soFar, data.length - soFar); } while(soFar != data.length); sum = 0; for(final byte b : data) { sum += b; } return (sum); } catch(final IOException ex) { ex.printStackTrace(); } finally { if(stream != null) { try { stream.close(); } catch(final IOException ex) { ex.printStackTrace(); } } } return (0); } } class DataInputByteAtATime extends Test { public static void main(final String[] argv) { final Test test; test = new DataInputByteAtATime(); test.run(new File(argv[0])); } protected long readFile(final File file) { DataInputStream stream; stream = null; try { final int fileSize; int sum; stream = new DataInputStream(new BufferedInputStream(new FileInputStream(file))); fileSize = (int)file.length(); sum = 0; for(int i = 0; i < fileSize; i++) { sum += stream.readByte(); } return (sum); } catch(final IOException ex) { ex.printStackTrace(); } finally { if(stream != null) { try { stream.close(); } catch(final IOException ex) { ex.printStackTrace(); } } } return (0); } } class DataInputReadFully extends Test { public static void main(final String[] argv) { final Test test; test = new DataInputReadFully(); test.run(new File(argv[0])); } protected long readFile(final File file) { DataInputStream stream; stream = null; try { final byte[] data; int sum; stream = new DataInputStream(new BufferedInputStream(new FileInputStream(file))); data = new byte[(int)file.length()]; stream.readFully(data); sum = 0; for(final byte b : data) { sum += b; } return (sum); } catch(final IOException ex) { ex.printStackTrace(); } finally { if(stream != null) { try { stream.close(); } catch(final IOException ex) { ex.printStackTrace(); } } } return (0); } } class DataInputReadInChunks extends Test { public static void main(final String[] argv) { final Test test; test = new DataInputReadInChunks(); test.run(new File(argv[0])); } protected long readFile(final File file) { DataInputStream stream; stream = null; try { final byte[] data; int size; final int fileSize; int sum; stream = new DataInputStream(new BufferedInputStream(new FileInputStream(file))); fileSize = (int)file.length(); data = new byte[512]; size = 0; sum = 0; do { size += stream.read(data); sum = 0; for(int i = 0; i < size; i++) { sum += data[i]; } } while(size != fileSize); return (sum); } catch(final IOException ex) { ex.printStackTrace(); } finally { if(stream != null) { try { stream.close(); } catch(final IOException ex) { ex.printStackTrace(); } } } return (0); } } class MemoryMapped extends Test { public static void main(final String[] argv) { final Test test; test = new MemoryMapped(); test.run(new File(argv[0])); } protected long readFile(final File file) { FileInputStream stream; stream = null; try { final FileChannel channel; final MappedByteBuffer buffer; final int fileSize; int sum; stream = new FileInputStream(file); channel = stream.getChannel(); buffer = channel.map(MapMode.READ_ONLY, 0, file.length()); fileSize = (int)file.length(); sum = 0; for(int i = 0; i < fileSize; i++) { sum += buffer.get(); } return (sum); } catch(final IOException ex) { ex.printStackTrace(); } finally { if(stream != null) { try { stream.close(); } catch(final IOException ex) { ex.printStackTrace(); } } } return (0); } } </code></pre>

Querying!

Guidance

An individual column

Larger individual text columns get their own page to allow for proper reading.

SQuiL has stopped working due to an internal error.

If you are curious you may find further information in the browser console, which is accessible through the devtools (F12).

Reload