eu.stratosphere.nephele.services.iomanager.IOManagerPerformanceBenchmark.java Source code

Java tutorial

Introduction

Here is the source code for eu.stratosphere.nephele.services.iomanager.IOManagerPerformanceBenchmark.java

Source

/***********************************************************************************************************************
 * Copyright (C) 2010-2013 by the Stratosphere project (http://stratosphere.eu)
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
 * the License. You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
 * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
 * specific language governing permissions and limitations under the License.
 **********************************************************************************************************************/

package eu.stratosphere.nephele.services.iomanager;

import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.util.List;

import junit.framework.Assert;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;

import eu.stratosphere.core.memory.MemorySegment;
import eu.stratosphere.nephele.services.memorymanager.DefaultMemoryManagerTest;
import eu.stratosphere.nephele.services.memorymanager.spi.DefaultMemoryManager;
import eu.stratosphere.nephele.template.AbstractInvokable;
import eu.stratosphere.nephele.types.IntegerRecord;

/**
 *
 *
 */
public class IOManagerPerformanceBenchmark {
    private static final Log LOG = LogFactory.getLog(IOManagerPerformanceBenchmark.class);

    private static final int[] SEGMENT_SIZES_ALIGNED = { 4096, 16384, 524288 };

    private static final int[] SEGMENT_SIZES_UNALIGNED = { 3862, 16895, 500481 };

    private static final int[] NUM_SEGMENTS = { 1, 2, 4, 6 };

    private static final long MEMORY_SIZE = 32 * 1024 * 1024;

    private static final int NUM_INTS_WRITTEN = 100000000;

    private static final AbstractInvokable memoryOwner = new DefaultMemoryManagerTest.DummyInvokable();

    private DefaultMemoryManager memManager;

    private IOManager ioManager;

    @Before
    public void startup() {
        memManager = new DefaultMemoryManager(MEMORY_SIZE);
        ioManager = new IOManager();
    }

    @After
    public void afterTest() throws Exception {
        ioManager.shutdown();
        Assert.assertTrue("IO Manager has not properly shut down.", ioManager.isProperlyShutDown());

        Assert.assertTrue("Not all memory was returned to the memory manager in the test.",
                memManager.verifyEmpty());
        memManager.shutdown();
        memManager = null;
    }

    // ------------------------------------------------------------------------

    @Test
    public void speedTestIOManager() throws Exception {
        LOG.info("Starting speed test with IO Manager...");

        for (int num : NUM_SEGMENTS) {
            testChannelWithSegments(num);
        }
    }

    private final void testChannelWithSegments(int numSegments) throws Exception {
        final List<MemorySegment> memory = this.memManager.allocatePages(memoryOwner, numSegments);
        final Channel.ID channel = this.ioManager.createChannel();

        BlockChannelWriter writer = null;
        BlockChannelReader reader = null;

        try {
            writer = this.ioManager.createBlockChannelWriter(channel);
            final ChannelWriterOutputView out = new ChannelWriterOutputView(writer, memory,
                    this.memManager.getPageSize());

            long writeStart = System.currentTimeMillis();

            int valsLeft = NUM_INTS_WRITTEN;
            while (valsLeft-- > 0) {
                out.writeInt(valsLeft);
            }

            out.close();
            final int numBlocks = out.getBlockCount();
            writer.close();
            writer = null;

            long writeElapsed = System.currentTimeMillis() - writeStart;

            // ----------------------------------------------------------------

            reader = ioManager.createBlockChannelReader(channel);
            final ChannelReaderInputView in = new ChannelReaderInputView(reader, memory, numBlocks, false);

            long readStart = System.currentTimeMillis();

            valsLeft = NUM_INTS_WRITTEN;
            while (valsLeft-- > 0) {
                in.readInt();
                //            Assert.assertTrue(rec.getValue() == valsLeft);
            }

            in.close();
            reader.close();

            long readElapsed = System.currentTimeMillis() - readStart;

            reader.deleteChannel();
            reader = null;

            LOG.info("IOManager with " + numSegments + " mem segments: write " + writeElapsed + " msecs, read "
                    + readElapsed + " msecs.");

            memManager.release(memory);
        } finally {
            if (reader != null) {
                reader.closeAndDelete();
            }
            if (writer != null) {
                writer.closeAndDelete();
            }
        }
    }

    //   @Test
    //   public void speedTestRandomAccessFile() throws IOException {
    //      LOG.info("Starting speed test with java random access file ...");
    //      
    //      Channel.ID tmpChannel = ioManager.createChannel();
    //      File tempFile = null;
    //      RandomAccessFile raf = null;
    //      
    //      try {
    //         tempFile = new File(tmpChannel.getPath()); 
    //         raf = new RandomAccessFile(tempFile, "rw");
    //         
    //         IntegerRecord rec = new IntegerRecord(0);
    //         
    //         long writeStart = System.currentTimeMillis();
    //         
    //         int valsLeft = NUM_INTS_WRITTEN;
    //         while (valsLeft-- > 0) {
    //            rec.setValue(valsLeft);
    //            rec.write(raf);
    //         }
    //         raf.close();
    //         raf = null;
    //         
    //         long writeElapsed = System.currentTimeMillis() - writeStart;
    //         
    //         // ----------------------------------------------------------------
    //         
    //         raf = new RandomAccessFile(tempFile, "r");
    //         
    //         long readStart = System.currentTimeMillis();
    //         
    //         valsLeft = NUM_INTS_WRITTEN;
    //         while (valsLeft-- > 0) {
    //            rec.read(raf);
    //         }
    //         raf.close();
    //         raf = null;
    //         
    //         long readElapsed = System.currentTimeMillis() - readStart;
    //         
    //         
    //         LOG.info("Random Access File: write " + (writeElapsed / 1000) + " secs, read " + (readElapsed / 1000) + " secs.");
    //      }
    //      finally {
    //         // close if possible
    //         if (raf != null) {
    //            raf.close();
    //         }
    //         
    //         // try to delete the file
    //         if (tempFile != null) {
    //            tempFile.delete();
    //         }
    //      }
    //   }

    @Test
    public void speedTestFileStream() throws Exception {
        LOG.info("Starting speed test with java io file stream and ALIGNED buffer sizes ...");

        for (int bufferSize : SEGMENT_SIZES_ALIGNED) {
            speedTestStream(bufferSize);
        }

        LOG.info("Starting speed test with java io file stream and UNALIGNED buffer sizes ...");

        for (int bufferSize : SEGMENT_SIZES_UNALIGNED) {
            speedTestStream(bufferSize);
        }

    }

    private final void speedTestStream(int bufferSize) throws IOException {
        final Channel.ID tmpChannel = ioManager.createChannel();
        final IntegerRecord rec = new IntegerRecord(0);

        File tempFile = null;
        DataOutputStream daos = null;
        DataInputStream dais = null;

        try {
            tempFile = new File(tmpChannel.getPath());

            FileOutputStream fos = new FileOutputStream(tempFile);
            daos = new DataOutputStream(new BufferedOutputStream(fos, bufferSize));

            long writeStart = System.currentTimeMillis();

            int valsLeft = NUM_INTS_WRITTEN;
            while (valsLeft-- > 0) {
                rec.setValue(valsLeft);
                rec.write(daos);
            }
            daos.close();
            daos = null;

            long writeElapsed = System.currentTimeMillis() - writeStart;

            // ----------------------------------------------------------------

            FileInputStream fis = new FileInputStream(tempFile);
            dais = new DataInputStream(new BufferedInputStream(fis, bufferSize));

            long readStart = System.currentTimeMillis();

            valsLeft = NUM_INTS_WRITTEN;
            while (valsLeft-- > 0) {
                rec.read(dais);
            }
            dais.close();
            dais = null;

            long readElapsed = System.currentTimeMillis() - readStart;

            LOG.info("File-Stream with buffer " + bufferSize + ": write " + writeElapsed + " msecs, read "
                    + readElapsed + " msecs.");
        } finally {
            // close if possible
            if (daos != null) {
                daos.close();
            }
            if (dais != null) {
                dais.close();
            }
            // try to delete the file
            if (tempFile != null) {
                tempFile.delete();
            }
        }
    }

    // ------------------------------------------------------------------------

    @Test
    public void speedTestNIO() throws Exception {
        LOG.info("Starting speed test with java NIO heap buffers and ALIGNED buffer sizes ...");

        for (int bufferSize : SEGMENT_SIZES_ALIGNED) {
            speedTestNIO(bufferSize, false);
        }

        LOG.info("Starting speed test with java NIO heap buffers and UNALIGNED buffer sizes ...");

        for (int bufferSize : SEGMENT_SIZES_UNALIGNED) {
            speedTestNIO(bufferSize, false);
        }

        LOG.info("Starting speed test with java NIO direct buffers and ALIGNED buffer sizes ...");

        for (int bufferSize : SEGMENT_SIZES_ALIGNED) {
            speedTestNIO(bufferSize, true);
        }

        LOG.info("Starting speed test with java NIO direct buffers and UNALIGNED buffer sizes ...");

        for (int bufferSize : SEGMENT_SIZES_UNALIGNED) {
            speedTestNIO(bufferSize, true);
        }

    }

    @SuppressWarnings("resource")
    private final void speedTestNIO(int bufferSize, boolean direct) throws IOException {
        final Channel.ID tmpChannel = ioManager.createChannel();

        File tempFile = null;
        FileChannel fs = null;

        try {
            tempFile = new File(tmpChannel.getPath());

            RandomAccessFile raf = new RandomAccessFile(tempFile, "rw");
            fs = raf.getChannel();

            ByteBuffer buf = direct ? ByteBuffer.allocateDirect(bufferSize) : ByteBuffer.allocate(bufferSize);

            long writeStart = System.currentTimeMillis();

            int valsLeft = NUM_INTS_WRITTEN;
            while (valsLeft-- > 0) {
                if (buf.remaining() < 4) {
                    buf.flip();
                    fs.write(buf);
                    buf.clear();
                }
                buf.putInt(valsLeft);
            }

            if (buf.position() > 0) {
                buf.flip();
                fs.write(buf);
            }

            fs.close();
            raf.close();
            fs = null;

            long writeElapsed = System.currentTimeMillis() - writeStart;

            // ----------------------------------------------------------------

            raf = new RandomAccessFile(tempFile, "r");
            fs = raf.getChannel();
            buf.clear();

            long readStart = System.currentTimeMillis();

            fs.read(buf);
            buf.flip();

            valsLeft = NUM_INTS_WRITTEN;
            while (valsLeft-- > 0) {
                if (buf.remaining() < 4) {
                    buf.compact();
                    fs.read(buf);
                    buf.flip();
                }
                if (buf.getInt() != valsLeft) {
                    throw new IOException();
                }
            }

            fs.close();
            raf.close();

            long readElapsed = System.currentTimeMillis() - readStart;

            LOG.info("NIO Channel with buffer " + bufferSize + ": write " + writeElapsed + " msecs, read "
                    + readElapsed + " msecs.");
        } finally {
            // close if possible
            if (fs != null) {
                fs.close();
                fs = null;
            }
            // try to delete the file
            if (tempFile != null) {
                tempFile.delete();
            }
        }
    }

}