Example usage for java.nio ByteBuffer clear

List of usage examples for java.nio ByteBuffer clear

Introduction

In this page you can find the example usage for java.nio ByteBuffer clear.

Prototype

public final Buffer clear() 

Source Link

Document

Clears this buffer.

Usage

From source file:org.lnicholls.galleon.togo.ToGo.java

public boolean Download(Video video, CancelDownload cancelDownload) {

    ServerConfiguration serverConfiguration = Server.getServer().getServerConfiguration();
    GoBackConfiguration goBackConfiguration = Server.getServer().getGoBackConfiguration();

    ArrayList videos = new ArrayList();

    GetMethod get = null;//ww w  . j av  a2  s  .c  o m

    try {

        URL url = new URL(video.getUrl());

        Protocol protocol = new Protocol("https", new TiVoSSLProtocolSocketFactory(), 443);

        HttpClient client = new HttpClient();

        // TODO How to get TiVo address??

        client.getHostConfiguration().setHost(url.getHost(), 443, protocol);

        String password = Tools.decrypt(serverConfiguration.getMediaAccessKey());

        if (video.getParentalControls() != null && video.getParentalControls().booleanValue())

        {

            if (serverConfiguration.getPassword() == null)

                throw new NullPointerException("Parental Controls Password is null");

            password = password + Tools.decrypt(serverConfiguration.getPassword());

        }

        Credentials credentials = new UsernamePasswordCredentials("tivo", password);

        //client.getState().setCredentials("TiVo DVR", url.getHost(), credentials);

        client.getState().setCredentials(null, url.getHost(), credentials);

        get = new GetMethod(video.getUrl());

        client.executeMethod(get);

        if (get.getStatusCode() != 200)

        {

            log.debug("Status code: " + get.getStatusCode());

            return false;

        }

        InputStream input = get.getResponseBodyAsStream();

        String path = serverConfiguration.getRecordingsPath();

        File dir = new File(path);

        if (!dir.exists()) {

            dir.mkdirs();

        }

        String name = getFilename(video);
        File file = null;
        if (goBackConfiguration.isGroupByShow()) {
            if (video.getSeriesTitle() != null && video.getSeriesTitle().trim().length() > 0) {
                path = path + File.separator + clean(video.getSeriesTitle());
                File filePath = new File(path);
                if (!filePath.exists())
                    filePath.mkdirs();
                file = new File(path + File.separator + name);
            } else
                file = new File(path + File.separator + name);
        } else {
            file = new File(path + File.separator + name);
        }

        // TODO Handle retransfers

        /*
                
        if (file.exists() && video.getStatus()!=Video.STATUS_DOWNLOADING)
                
        {
                
           log.debug("duplicate file: "+file);
                
           try {
                
              List list = VideoManager.findByPath(file.getCanonicalPath());
                
              if (list!=null && list.size()>0)
                
              {
                
          video.setDownloadSize(file.length());
                
          video.setDownloadTime(0);
                
          video.setPath(file.getCanonicalPath());
                
          video.setStatus(Video.STATUS_DELETED);
                
          VideoManager.updateVideo(video);
                
          return true;
                
              }
                
           } catch (HibernateException ex) {
                
              log.error("Video update failed", ex);
                
           }
                
        }
                
        */

        log.info("Downloading: " + name);

        WritableByteChannel channel = new FileOutputStream(file, false).getChannel();

        long total = 0;

        double diff = 0.0;

        ByteBuffer buf = ByteBuffer.allocateDirect(1024 * 4);

        byte[] bytes = new byte[1024 * 4];

        int amount = 0;

        int index = 0;

        long target = video.getSize();

        long start = System.currentTimeMillis();

        long last = start;

        while (amount == 0 && total < target) {

            while (amount >= 0 && !cancelDownload.cancel()) {

                if (index == amount) {

                    amount = input.read(bytes);

                    index = 0;

                    total = total + amount;

                }

                while (index < amount && buf.hasRemaining()) {

                    buf.put(bytes[index++]);

                }

                buf.flip();

                int numWritten = channel.write(buf);

                if (buf.hasRemaining()) {

                    buf.compact();

                } else {

                    buf.clear();

                }

                if ((System.currentTimeMillis() - last > 10000) && (total > 0)) {

                    try {

                        video = VideoManager.retrieveVideo(video.getId());

                        if (video.getStatus() == Video.STATUS_DOWNLOADING) {

                            diff = (System.currentTimeMillis() - start) / 1000.0;

                            if (diff > 0) {

                                video.setDownloadSize(total);

                                video.setDownloadTime((int) diff);

                                VideoManager.updateVideo(video);

                            }

                        }

                    } catch (HibernateException ex) {

                        log.error("Video update failed", ex);

                    }

                    last = System.currentTimeMillis();

                }

            }

            if (cancelDownload.cancel()) {

                channel.close();

                return false;

            }

        }

        diff = (System.currentTimeMillis() - start) / 1000.0;

        channel.close();

        if (diff != 0)

            log.info("Download rate=" + (total / 1024) / diff + " KBps");

        try {

            video.setPath(file.getCanonicalPath());

            VideoManager.updateVideo(video);

        } catch (HibernateException ex) {

            log.error("Video update failed", ex);

        }

    } catch (MalformedURLException ex) {

        Tools.logException(ToGo.class, ex, video.getUrl());

        return false;

    } catch (Exception ex) {

        Tools.logException(ToGo.class, ex, video.getUrl());

        return false;

    } finally {

        if (get != null)

            get.releaseConnection();

    }

    return true;

}

From source file:org.wso2.andes.tools.messagestore.commands.Dump.java

protected List<List> createMessageData(java.util.List<Long> msgids, List<QueueEntry> messages,
        boolean showHeaders, boolean showRouting, boolean showMessageHeaders) {

    List<List> display = new LinkedList<List>();

    List<String> hex = new LinkedList<String>();
    List<String> ascii = new LinkedList<String>();
    display.add(hex);/*from w  ww .  j  a v  a  2s .co  m*/
    display.add(ascii);

    for (QueueEntry entry : messages) {
        ServerMessage msg = entry.getMessage();
        if (!includeMsg(msg, msgids)) {
            continue;
        }

        //Add divider between messages
        hex.add(Console.ROW_DIVIDER);
        ascii.add(Console.ROW_DIVIDER);

        // Show general message information
        hex.add(Show.Columns.ID.name());
        ascii.add(msg.getMessageNumber().toString());

        hex.add(Console.ROW_DIVIDER);
        ascii.add(Console.ROW_DIVIDER);

        if (showRouting) {
            addShowInformation(hex, ascii, msg, "Routing Details", true, false, false);
        }
        if (showHeaders) {
            addShowInformation(hex, ascii, msg, "Headers", false, true, false);
        }
        if (showMessageHeaders) {
            addShowInformation(hex, ascii, msg, null, false, false, true);
        }

        // Add Content Body section
        hex.add("Content Body");
        ascii.add("");
        hex.add(Console.ROW_DIVIDER);
        ascii.add(Console.ROW_DIVIDER);

        final int messageSize = (int) msg.getSize();
        if (messageSize != 0) {
            hex.add("Hex");
            hex.add(Console.ROW_DIVIDER);

            ascii.add("ASCII");
            ascii.add(Console.ROW_DIVIDER);

            java.nio.ByteBuffer buf = java.nio.ByteBuffer.allocate(64 * 1024);

            int position = 0;

            while (position < messageSize) {

                position += msg.getContent(buf, position);
                buf.flip();
                //Duplicate so we don't destroy original data :)
                java.nio.ByteBuffer hexBuffer = buf;

                java.nio.ByteBuffer charBuffer = hexBuffer.duplicate();

                Hex hexencoder = new Hex();

                while (hexBuffer.hasRemaining()) {
                    byte[] line = new byte[LINE_SIZE];

                    int bufsize = hexBuffer.remaining();
                    if (bufsize < LINE_SIZE) {
                        hexBuffer.get(line, 0, bufsize);
                    } else {
                        bufsize = line.length;
                        hexBuffer.get(line);
                    }

                    byte[] encoded = hexencoder.encode(line);

                    try {
                        String encStr = new String(encoded, 0, bufsize * 2, DEFAULT_ENCODING);
                        String hexLine = "";

                        int strLength = encStr.length();
                        for (int c = 0; c < strLength; c++) {
                            hexLine += encStr.charAt(c);

                            if ((c & 1) == 1 && SPACE_BYTES) {
                                hexLine += BYTE_SPACER;
                            }
                        }

                        hex.add(hexLine);
                    } catch (UnsupportedEncodingException e) {
                        _console.println(e.getMessage());
                        return null;
                    }
                }

                while (charBuffer.hasRemaining()) {
                    String asciiLine = "";

                    for (int pos = 0; pos < LINE_SIZE; pos++) {
                        if (charBuffer.hasRemaining()) {
                            byte ch = charBuffer.get();

                            if (isPrintable(ch)) {
                                asciiLine += (char) ch;
                            } else {
                                asciiLine += NON_PRINTING_ASCII_CHAR;
                            }

                            if (SPACE_BYTES) {
                                asciiLine += BYTE_SPACER;
                            }
                        } else {
                            break;
                        }
                    }

                    ascii.add(asciiLine);
                }
                buf.clear();
            }
        } else {
            List<String> result = new LinkedList<String>();

            display.add(result);
            result.add("No ContentBodies");
        }
    }

    // if hex is empty then we have no data to display
    if (hex.size() == 0) {
        return null;
    }

    return display;
}

From source file:jp.queuelinker.system.net.SelectorThread.java

@Override
public void run() {
    ByteBuffer buffer = ByteBuffer.allocate(BUFFER_SIZE);
    // I believe the inner try-catches does not cause overhead.
    // http://stackoverflow.com/questions/141560/
    long sendCount = 0;

    SocketChannel currentChannel;
    SelectionKey key = null;//from   w  w w.  java 2 s  .c om
    while (true) {
        try {
            selector.select();
            // selector.selectNow();
        } catch (ClosedSelectorException e) {
            logger.fatal("BUG: The selector is closed.");
            return;
        } catch (IOException e) {
            logger.fatal("An IOException occured while calling select().");
            // Fatal Error. Notify the error to the users and leave the matter to them.
            for (ChannelState state : channels) {
                state.callBack.fatalError();
            }
            return;
        }

        if (!requests.isEmpty()) {
            handleRequest();
            continue;
        }

        if (stopRequested) {
            return;
        }

        Set<SelectionKey> keys = selector.selectedKeys();

        Iterator<SelectionKey> iter = keys.iterator();
        iter_loop: while (iter.hasNext()) {
            key = iter.next();
            iter.remove(); // Required. Don't remove.

            if (key.isReadable()) {
                currentChannel = (SocketChannel) key.channel();
                final ChannelState state = (ChannelState) key.attachment();

                int valid;
                try {
                    valid = currentChannel.read(buffer);
                } catch (IOException e) {
                    logger.warn("An IOException happened while reading from a channel.");
                    state.callBack.exceptionOccured(state.channelId, state.attachment);
                    key.cancel();
                    continue;
                }
                if (valid == -1) {
                    // Normal socket close?
                    state.callBack.exceptionOccured(state.channelId, state.attachment);
                    // cleanUpChannel(state.channelId);
                    key.cancel();
                    continue;
                }

                buffer.rewind();
                if (state.callBack.receive(state.channelId, buffer, valid, state.attachment) == false) {
                    state.key.interestOps(state.key.interestOps() & ~SelectionKey.OP_READ);
                }
                buffer.clear();
            } else if (key.isWritable()) {
                currentChannel = (SocketChannel) key.channel();
                final ChannelState state = (ChannelState) key.attachment();

                while (state.sendBuffer.readableSize() < WRITE_SIZE) {
                    ByteBuffer newBuffer = state.callBack.send(state.channelId, state.attachment);
                    if (newBuffer != null) {
                        state.sendBuffer.write(newBuffer);
                        if (++sendCount % 50000 == 0) {
                            logger.info("Send Count: " + sendCount);
                        }
                    } else if (state.sendBuffer.readableSize() == 0) {
                        key.interestOps(key.interestOps() & ~SelectionKey.OP_WRITE);
                        continue iter_loop;
                    } else {
                        break;
                    }
                }

                final int available = state.sendBuffer.readableSize();
                if (available >= WRITE_SIZE || ++state.noopCount >= HEURISTIC_WAIT) {
                    int done;
                    try {
                        done = currentChannel.write(state.sendBuffer.getByteBuffer());
                    } catch (IOException e) {
                        logger.warn("An IOException occured while writing to a channel.");
                        state.callBack.exceptionOccured(state.channelId, state.attachment);
                        key.cancel();
                        continue;
                    }
                    if (done < available) {
                        state.sendBuffer.rollback(available - done);
                    }
                    state.sendBuffer.compact();
                    state.noopCount = 0;
                }
            } else if (key.isAcceptable()) {
                ServerSocketChannel channel = (ServerSocketChannel) key.channel();
                ChannelState state = (ChannelState) key.attachment();
                SocketChannel socketChannel;
                try {
                    socketChannel = channel.accept();
                    socketChannel.configureBlocking(false);
                } catch (IOException e) {
                    continue; // Do nothing.
                }
                state.callBack.newConnection(state.channelId, socketChannel, state.attachment);
            }
        }
    }
}

From source file:com.healthmarketscience.jackcess.Column.java

/**
 * @param lvalDefinition Column value that points to an LVAL record
 * @return The LVAL data//from   w  w  w. jav a2  s .  c o m
 */
private byte[] readLongValue(byte[] lvalDefinition) throws IOException {
    ByteBuffer def = ByteBuffer.wrap(lvalDefinition).order(PageChannel.DEFAULT_BYTE_ORDER);
    int lengthWithFlags = def.getInt();
    int length = lengthWithFlags & (~LONG_VALUE_TYPE_MASK);

    byte[] rtn = new byte[length];
    byte type = (byte) ((lengthWithFlags & LONG_VALUE_TYPE_MASK) >>> 24);

    if (type == LONG_VALUE_TYPE_THIS_PAGE) {

        // inline long value
        def.getInt(); //Skip over lval_dp
        def.getInt(); //Skip over unknown
        def.get(rtn);

    } else {

        // long value on other page(s)
        if (lvalDefinition.length != getFormat().SIZE_LONG_VALUE_DEF) {
            throw new IOException("Expected " + getFormat().SIZE_LONG_VALUE_DEF
                    + " bytes in long value definition, but found " + lvalDefinition.length);
        }

        int rowNum = ByteUtil.getUnsignedByte(def);
        int pageNum = ByteUtil.get3ByteInt(def, def.position());
        ByteBuffer lvalPage = getPageChannel().createPageBuffer();

        switch (type) {
        case LONG_VALUE_TYPE_OTHER_PAGE: {
            getPageChannel().readPage(lvalPage, pageNum);

            short rowStart = Table.findRowStart(lvalPage, rowNum, getFormat());
            short rowEnd = Table.findRowEnd(lvalPage, rowNum, getFormat());

            if ((rowEnd - rowStart) != length) {
                throw new IOException("Unexpected lval row length");
            }

            lvalPage.position(rowStart);
            lvalPage.get(rtn);
        }
            break;

        case LONG_VALUE_TYPE_OTHER_PAGES:

            ByteBuffer rtnBuf = ByteBuffer.wrap(rtn);
            int remainingLen = length;
            while (remainingLen > 0) {
                lvalPage.clear();
                getPageChannel().readPage(lvalPage, pageNum);

                short rowStart = Table.findRowStart(lvalPage, rowNum, getFormat());
                short rowEnd = Table.findRowEnd(lvalPage, rowNum, getFormat());

                // read next page information
                lvalPage.position(rowStart);
                rowNum = ByteUtil.getUnsignedByte(lvalPage);
                pageNum = ByteUtil.get3ByteInt(lvalPage);

                // update rowEnd and remainingLen based on chunkLength
                int chunkLength = (rowEnd - rowStart) - 4;
                if (chunkLength > remainingLen) {
                    rowEnd = (short) (rowEnd - (chunkLength - remainingLen));
                    chunkLength = remainingLen;
                }
                remainingLen -= chunkLength;

                lvalPage.limit(rowEnd);
                rtnBuf.put(lvalPage);
            }

            break;

        default:
            throw new IOException("Unrecognized long value type: " + type);
        }
    }

    return rtn;
}

From source file:edu.hawaii.soest.hioos.storx.StorXParser.java

/**
 * Parses the binary STOR-X file.  The binary file format is a sequence of
 * 'frames' that all begin with 'SAT'.  The parser creates a list with the
 * individual frames.  Some frames are StorX frames (SATSTX), some are from 
 * external sensors (ISUS: 'SATNLB', 'SATNDB'; SBE CTD: 'SATSBE')
 *
 * @param fileBuffer - the binary data file as a ByteBuffer
 *///w  w  w  .  j  a v a  2 s  .  co  m
public void parse(ByteBuffer fileBuffer) throws Exception {

    logger.debug("StorXParser.parse() called.");

    this.fileBuffer = fileBuffer;
    //logger.debug(this.fileBuffer.toString());

    try {

        // Create a buffer that will store a single frame of the file
        ByteBuffer frameBuffer = ByteBuffer.allocate(1024);

        // create four byte placeholders used to evaluate up to a four-byte 
        // window.  The FIFO layout looks like:
        //           ---------------------------
        //   in ---> | Four | Three | Two | One |  ---> out
        //           ---------------------------
        byte byteOne = 0x00, // set initial placeholder values
                byteTwo = 0x00, byteThree = 0x00, byteFour = 0x00;

        int frameByteCount = 0; // keep track of bytes per frame
        int frameCount = 0; // keep track of frames

        this.fileBuffer.position(0);
        this.fileBuffer.limit(this.fileBuffer.capacity());

        while (this.fileBuffer.hasRemaining()) {

            // load the next byte into the FIFO window
            byteOne = fileBuffer.get();

            // show the byte stream coming in
            //logger.debug("b1: " + new String(Hex.encodeHex(new byte[]{byteOne}))   + "\t" +
            //             "b2: " + new String(Hex.encodeHex(new byte[]{byteTwo}))   + "\t" +
            //             "b3: " + new String(Hex.encodeHex(new byte[]{byteThree})) + "\t" +
            //             "b4: " + new String(Hex.encodeHex(new byte[]{byteFour}))  + "\t" +
            //             "st: " + Integer.toString(this.state)                     + "\t" +
            //             "po: " + this.fileBuffer.position()                       + "\t" +
            //             "cp: " + this.fileBuffer.capacity()
            //             );

            // evaluate the bytes, separate the file frame by frame (SAT ...)
            switch (this.state) {

            case 0: // find a frame beginning (SAT) 53 41 54

                if (byteOne == 0x54 && byteTwo == 0x41 && byteThree == 0x53) {

                    // found a line, add the beginning to the line buffer 
                    frameBuffer.put(byteThree);
                    frameBuffer.put(byteTwo);
                    frameBuffer.put(byteOne);

                    frameByteCount = frameByteCount + 3;

                    this.state = 1;
                    break;

                } else {
                    break;

                }

            case 1: // find the next frame beginning (SAT) 53 41 54

                if ((byteOne == 0x54 && byteTwo == 0x41 && byteThree == 0x53)
                        || fileBuffer.position() == fileBuffer.capacity()) {

                    // we have a line ending. store the line in the arrayList
                    frameBuffer.put(byteOne);
                    frameByteCount++;
                    frameBuffer.flip();
                    byte[] frameArray = frameBuffer.array();
                    ByteBuffer currentFrameBuffer;

                    if (fileBuffer.position() == fileBuffer.capacity()) {

                        // create a true copy of the byte array subset (no trailing 'SAT')
                        byte[] frameCopy = new byte[frameByteCount];
                        System.arraycopy(frameArray, 0, frameCopy, 0, frameByteCount);
                        currentFrameBuffer = ByteBuffer.wrap(frameCopy);

                    } else {

                        // create a true copy of the byte array subset (less the 'SAT')
                        byte[] frameCopy = new byte[frameByteCount - 3];
                        System.arraycopy(frameArray, 0, frameCopy, 0, frameByteCount - 3);
                        currentFrameBuffer = ByteBuffer.wrap(frameCopy);

                    }

                    // parse the current frame and add it to the frameMap

                    frameCount++;

                    // create a map to store frames as they are encountered
                    BasicHierarchicalMap frameMap = new BasicHierarchicalMap();

                    // peek at the first six header bytes as a string
                    byte[] sixBytes = new byte[6];
                    currentFrameBuffer.get(sixBytes);
                    currentFrameBuffer.position(0);
                    String frameHeader = new String(sixBytes, "US-ASCII");

                    // determine the frame type based on the header
                    if (frameHeader.matches(this.STOR_X_HEADER_ID)) {
                        frameMap.put("rawFrame", currentFrameBuffer);
                        frameMap.put("id", frameHeader);
                        frameMap.put("type", frameHeader.substring(3, 6));
                        frameMap.put("serialNumber", null);
                        frameMap.put("date", null);
                        String headerString = new String(currentFrameBuffer.array());
                        // trim trailing null characters and line endings
                        int nullIndex = headerString.indexOf(0);
                        headerString = headerString.substring(0, nullIndex).trim();
                        frameMap.put("parsedFrameObject", headerString);

                        // Add the frame to the frames map
                        this.framesMap.add("/frames/frame", (BasicHierarchicalMap) frameMap.clone());

                        frameMap.removeAll("frame");
                        currentFrameBuffer.clear();

                    } else if (frameHeader.matches(this.STOR_X_FRAME_ID)) {

                        // test if the frame is complete
                        if (currentFrameBuffer.capacity() == this.STOR_X_FRAME_SIZE) {

                            // convert the frame buffer to a StorXFrame
                            StorXFrame storXFrame = new StorXFrame(currentFrameBuffer);

                            frameMap.put("rawFrame", currentFrameBuffer);
                            frameMap.put("id", frameHeader);
                            frameMap.put("type", frameHeader.substring(3, 6));
                            frameMap.put("serialNumber", storXFrame.getSerialNumber());
                            frameMap.put("date", parseTimestamp(storXFrame.getTimestamp()));
                            frameMap.put("parsedFrameObject", storXFrame);

                            // Add the frame to the frames map
                            this.framesMap.add("/frames/frame", (BasicHierarchicalMap) frameMap.clone());

                            frameMap.removeAll("frame");
                            currentFrameBuffer.clear();

                        } else {
                            logger.debug(frameHeader + " frame " + frameCount + " length is "
                                    + currentFrameBuffer.capacity() + " not " + this.STOR_X_FRAME_SIZE);
                        }

                    } else if (frameHeader.matches(this.SBE_CTD_FRAME_ID)) {

                        // convert the frame buffer to a CTDFrame
                        CTDFrame ctdFrame = new CTDFrame(currentFrameBuffer);

                        // add in a sample if it matches a general data sample pattern
                        if (ctdFrame.getSample().matches(" [0-9].*[0-9]\r\n")) {

                            // extract the sample bytes from the frame
                            frameMap.put("rawFrame", currentFrameBuffer);
                            frameMap.put("id", frameHeader);
                            frameMap.put("type", frameHeader.substring(3, 6));
                            frameMap.put("serialNumber", ctdFrame.getSerialNumber());
                            frameMap.put("date", parseTimestamp(ctdFrame.getTimestamp()));
                            frameMap.put("parsedFrameObject", ctdFrame);

                            // Add the frame to the frames map
                            this.framesMap.add("/frames/frame", (BasicHierarchicalMap) frameMap.clone());

                        } else {
                            logger.debug("This CTD frame is not a data sample."
                                    + " Skipping it. The string is: " + ctdFrame.getSample());
                        }

                        frameMap.removeAll("frame");
                        currentFrameBuffer.clear();

                    } else if (frameHeader.matches(this.ISUS_DARK_FRAME_ID)) {

                        // test if the frame is complete
                        if (currentFrameBuffer.capacity() == this.ISUS_FRAME_SIZE) {

                            // convert the frame buffer to a ISUSFrame
                            ISUSFrame isusFrame = new ISUSFrame(currentFrameBuffer);

                            frameMap.put("rawFrame", currentFrameBuffer);
                            frameMap.put("id", frameHeader);
                            frameMap.put("type", frameHeader.substring(3, 6));
                            frameMap.put("serialNumber", isusFrame.getSerialNumber());
                            frameMap.put("date", parseTimestamp(isusFrame.getTimestamp()));
                            frameMap.put("parsedFrameObject", isusFrame);

                            // Add the frame to the frames map
                            this.framesMap.add("/frames/frame", (BasicHierarchicalMap) frameMap.clone());

                            frameMap.removeAll("frame");
                            currentFrameBuffer.clear();

                        } else {
                            logger.debug(frameHeader + " frame " + frameCount + " length is "
                                    + currentFrameBuffer.capacity() + " not " + this.ISUS_FRAME_SIZE);
                        }

                        currentFrameBuffer.clear();

                    } else if (frameHeader.matches(this.ISUS_LIGHT_FRAME_ID)) {

                        // test if the frame is complete
                        if (currentFrameBuffer.capacity() == this.ISUS_FRAME_SIZE) {

                            // convert the frame buffer to a ISUSFrame
                            ISUSFrame isusFrame = new ISUSFrame(currentFrameBuffer);

                            frameMap.put("rawFrame", currentFrameBuffer);
                            frameMap.put("id", frameHeader);
                            frameMap.put("type", frameHeader.substring(3, 6));
                            frameMap.put("serialNumber", isusFrame.getSerialNumber());
                            frameMap.put("date", parseTimestamp(isusFrame.getTimestamp()));
                            frameMap.put("parsedFrameObject", isusFrame);

                            // Add the frame to the frames map
                            this.framesMap.add("/frames/frame", (BasicHierarchicalMap) frameMap.clone());

                            frameMap.removeAll("frame");
                            currentFrameBuffer.clear();

                        } else {
                            logger.debug(frameHeader + " frame " + frameCount + " length is "
                                    + currentFrameBuffer.capacity() + " not " + this.ISUS_FRAME_SIZE);
                        }

                        currentFrameBuffer.clear();

                    } else {
                        logger.info("The current frame type is not recognized. "
                                + "Discarding it.  The header was: " + frameHeader);
                        currentFrameBuffer.clear();

                    }

                    // reset the frame buffer for the next frame, but add the 'SAT'
                    // bytes already encountered
                    frameBuffer.clear();
                    frameByteCount = 0;
                    this.fileBuffer.position(this.fileBuffer.position() - 3);
                    this.state = 0;
                    break;

                } else {

                    // no full line yet, keep adding bytes
                    frameBuffer.put(byteOne);
                    frameByteCount++;
                    break;

                }

            } // end switch()

            // shift the bytes in the FIFO window
            byteFour = byteThree;
            byteThree = byteTwo;
            byteTwo = byteOne;

        } // end while()

        logger.debug(this.framesMap.toXMLString(1000));

    } catch (Exception e) {
        logger.debug("Failed to parse the data file.  The error message was:" + e.getMessage());
        e.printStackTrace();

    }

}

From source file:net.librec.data.convertor.TextDataConvertor.java

/**
 * Read data from the data file. Note that we didn't take care of the
 * duplicated lines.//from  www  .  j a va2  s  .co m
 *
 * @param dataColumnFormat
 *            the format of input data file
 * @param inputDataPath
 *            the path of input data file
 * @param binThold
 *            the threshold to binarize a rating. If a rating is greater
 *            than the threshold, the value will be 1; otherwise 0. To
 *            disable this appender, i.e., keep the original rating value,
 *            set the threshold a negative value
 * @throws IOException
 *            if the <code>inputDataPath</code> is not valid.
 */
private void readData(String dataColumnFormat, String inputDataPath, double binThold) throws IOException {
    LOG.info(String.format("Dataset: %s", StringUtil.last(inputDataPath, 38)));
    // Table {row-id, col-id, rate}
    Table<Integer, Integer, Double> dataTable = HashBasedTable.create();
    // Table {row-id, col-id, timestamp}
    Table<Integer, Integer, Long> timeTable = null;
    // Map {col-id, multiple row-id}: used to fast build a rating matrix
    Multimap<Integer, Integer> colMap = HashMultimap.create();
    // BiMap {raw id, inner id} userIds, itemIds
    if (this.userIds == null) {
        this.userIds = HashBiMap.create();
    }
    if (this.itemIds == null) {
        this.itemIds = HashBiMap.create();
    }
    final List<File> files = new ArrayList<File>();
    final ArrayList<Long> fileSizeList = new ArrayList<Long>();
    SimpleFileVisitor<Path> finder = new SimpleFileVisitor<Path>() {
        @Override
        public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
            fileSizeList.add(file.toFile().length());
            files.add(file.toFile());
            return super.visitFile(file, attrs);
        }
    };
    Files.walkFileTree(Paths.get(inputDataPath), finder);
    LOG.info("All dataset files " + files.toString());
    long allFileSize = 0;
    for (Long everyFileSize : fileSizeList) {
        allFileSize = allFileSize + everyFileSize.longValue();
    }
    LOG.info("All dataset files size " + Long.toString(allFileSize));
    int readingFileCount = 0;
    long loadAllFileByte = 0;
    // loop every dataFile collecting from walkFileTree
    for (File dataFile : files) {
        LOG.info("Now loading dataset file " + dataFile.toString().substring(
                dataFile.toString().lastIndexOf(File.separator) + 1, dataFile.toString().lastIndexOf(".")));
        readingFileCount += 1;
        loadFilePathRate = readingFileCount / (float) files.size();
        long readingOneFileByte = 0;
        FileInputStream fis = new FileInputStream(dataFile);
        FileChannel fileRead = fis.getChannel();
        ByteBuffer buffer = ByteBuffer.allocate(BSIZE);
        int len;
        String bufferLine = new String();
        byte[] bytes = new byte[BSIZE];
        while ((len = fileRead.read(buffer)) != -1) {
            readingOneFileByte += len;
            loadDataFileRate = readingOneFileByte / (float) fileRead.size();
            loadAllFileByte += len;
            loadAllFileRate = loadAllFileByte / (float) allFileSize;
            buffer.flip();
            buffer.get(bytes, 0, len);
            bufferLine = bufferLine.concat(new String(bytes, 0, len));
            bufferLine = bufferLine.replaceAll("\r", "\n");
            String[] bufferData = bufferLine.split("(\n)+");
            boolean isComplete = bufferLine.endsWith("\n");
            int loopLength = isComplete ? bufferData.length : bufferData.length - 1;
            for (int i = 0; i < loopLength; i++) {
                String line = new String(bufferData[i]);
                String[] data = line.trim().split("[ \t,]+");
                String user = data[0];
                String item = data[1];
                Double rate = ((dataColumnFormat.equals("UIR") || dataColumnFormat.equals("UIRT"))
                        && data.length >= 3) ? Double.valueOf(data[2]) : 1.0;

                // binarize the rating for item recommendation task
                if (binThold >= 0) {
                    rate = rate > binThold ? 1.0 : 0.0;
                }

                // inner id starting from 0
                int row = userIds.containsKey(user) ? userIds.get(user) : userIds.size();
                userIds.put(user, row);

                int col = itemIds.containsKey(item) ? itemIds.get(item) : itemIds.size();
                itemIds.put(item, col);

                dataTable.put(row, col, rate);
                colMap.put(col, row);
                // record rating's issuing time
                if (StringUtils.equals(dataColumnFormat, "UIRT") && data.length >= 4) {
                    if (timeTable == null) {
                        timeTable = HashBasedTable.create();
                    }
                    // convert to million-seconds
                    long mms = 0L;
                    try {
                        mms = Long.parseLong(data[3]); // cannot format
                        // 9.7323480e+008
                    } catch (NumberFormatException e) {
                        mms = (long) Double.parseDouble(data[3]);
                    }
                    long timestamp = timeUnit.toMillis(mms);
                    timeTable.put(row, col, timestamp);
                }
            }
            if (!isComplete) {
                bufferLine = bufferData[bufferData.length - 1];
            }
            buffer.clear();
        }
        fileRead.close();
        fis.close();
    }
    int numRows = numUsers(), numCols = numItems();
    // build rating matrix
    preferenceMatrix = new SparseMatrix(numRows, numCols, dataTable, colMap);
    if (timeTable != null)
        datetimeMatrix = new SparseMatrix(numRows, numCols, timeTable, colMap);
    // release memory of data table
    dataTable = null;
    timeTable = null;
}

From source file:hivemall.recommend.SlimUDTF.java

private void runIterativeTraining() throws HiveException {
    final ByteBuffer buf = this._inputBuf;
    final NioStatefulSegment dst = this._fileIO;
    assert (buf != null);
    assert (dst != null);

    final Reporter reporter = getReporter();
    final Counters.Counter iterCounter = (reporter == null) ? null
            : reporter.getCounter("hivemall.recommend.slim$Counter", "iteration");

    try {/*from w  ww .jav a 2s  .c  om*/
        if (dst.getPosition() == 0L) {// run iterations w/o temporary file
            if (buf.position() == 0) {
                return; // no training example
            }
            buf.flip();
            for (int iter = 2; iter < numIterations; iter++) {
                _cvState.next();
                reportProgress(reporter);
                setCounterValue(iterCounter, iter);

                while (buf.remaining() > 0) {
                    int recordBytes = buf.getInt();
                    assert (recordBytes > 0) : recordBytes;
                    replayTrain(buf);
                }
                buf.rewind();
                if (_cvState.isConverged(_observedTrainingExamples)) {
                    break;
                }
            }
            logger.info("Performed " + _cvState.getCurrentIteration() + " iterations of "
                    + NumberUtils.formatNumber(_observedTrainingExamples)
                    + " training examples on memory (thus "
                    + NumberUtils.formatNumber(_observedTrainingExamples * _cvState.getCurrentIteration())
                    + " training updates in total) ");

        } else { // read training examples in the temporary file and invoke train for each example
            // write KNNi in buffer to a temporary file
            if (buf.remaining() > 0) {
                writeBuffer(buf, dst);
            }

            try {
                dst.flush();
            } catch (IOException e) {
                throw new HiveException("Failed to flush a file: " + dst.getFile().getAbsolutePath(), e);
            }

            if (logger.isInfoEnabled()) {
                File tmpFile = dst.getFile();
                logger.info("Wrote KNN entries of axis items to a temporary file for iterative training: "
                        + tmpFile.getAbsolutePath() + " (" + FileUtils.prettyFileSize(tmpFile) + ")");
            }

            // run iterations
            for (int iter = 2; iter < numIterations; iter++) {
                _cvState.next();
                setCounterValue(iterCounter, iter);

                buf.clear();
                dst.resetPosition();
                while (true) {
                    reportProgress(reporter);
                    // load a KNNi to a buffer in the temporary file
                    final int bytesRead;
                    try {
                        bytesRead = dst.read(buf);
                    } catch (IOException e) {
                        throw new HiveException("Failed to read a file: " + dst.getFile().getAbsolutePath(), e);
                    }
                    if (bytesRead == 0) { // reached file EOF
                        break;
                    }
                    assert (bytesRead > 0) : bytesRead;

                    // reads training examples from a buffer
                    buf.flip();
                    int remain = buf.remaining();
                    if (remain < SizeOf.INT) {
                        throw new HiveException("Illegal file format was detected");
                    }
                    while (remain >= SizeOf.INT) {
                        int pos = buf.position();
                        int recordBytes = buf.getInt();
                        remain -= SizeOf.INT;
                        if (remain < recordBytes) {
                            buf.position(pos);
                            break;
                        }

                        replayTrain(buf);
                        remain -= recordBytes;
                    }
                    buf.compact();
                }
                if (_cvState.isConverged(_observedTrainingExamples)) {
                    break;
                }
            }
            logger.info("Performed " + _cvState.getCurrentIteration() + " iterations of "
                    + NumberUtils.formatNumber(_observedTrainingExamples)
                    + " training examples on memory and KNNi data on secondary storage (thus "
                    + NumberUtils.formatNumber(_observedTrainingExamples * _cvState.getCurrentIteration())
                    + " training updates in total) ");

        }
    } catch (Throwable e) {
        throw new HiveException("Exception caused in the iterative training", e);
    } finally {
        // delete the temporary file and release resources
        try {
            dst.close(true);
        } catch (IOException e) {
            throw new HiveException("Failed to close a file: " + dst.getFile().getAbsolutePath(), e);
        }
        this._inputBuf = null;
        this._fileIO = null;
    }
}

From source file:com.healthmarketscience.jackcess.Column.java

/**
 * Write an LVAL column into a ByteBuffer inline if it fits, otherwise in
 * other data page(s).//from   w  w  w  .  j av  a 2 s  .  c  om
 * @param value Value of the LVAL column
 * @return A buffer containing the LVAL definition and (possibly) the column
 *         value (unless written to other pages)
 * @usage _advanced_method_
 */
public ByteBuffer writeLongValue(byte[] value, int remainingRowLength) throws IOException {
    if (value.length > getType().getMaxSize()) {
        throw new IOException(
                "value too big for column, max " + getType().getMaxSize() + ", got " + value.length);
    }

    // determine which type to write
    byte type = 0;
    int lvalDefLen = getFormat().SIZE_LONG_VALUE_DEF;
    if (((getFormat().SIZE_LONG_VALUE_DEF + value.length) <= remainingRowLength)
            && (value.length <= getFormat().MAX_INLINE_LONG_VALUE_SIZE)) {
        type = LONG_VALUE_TYPE_THIS_PAGE;
        lvalDefLen += value.length;
    } else if (value.length <= getFormat().MAX_LONG_VALUE_ROW_SIZE) {
        type = LONG_VALUE_TYPE_OTHER_PAGE;
    } else {
        type = LONG_VALUE_TYPE_OTHER_PAGES;
    }

    ByteBuffer def = getPageChannel().createBuffer(lvalDefLen);
    // take length and apply type to first byte
    int lengthWithFlags = value.length | (type << 24);
    def.putInt(lengthWithFlags);

    if (type == LONG_VALUE_TYPE_THIS_PAGE) {
        // write long value inline
        def.putInt(0);
        def.putInt(0); //Unknown
        def.put(value);
    } else {

        TempPageHolder lvalBufferH = getTable().getLongValueBuffer();
        ByteBuffer lvalPage = null;
        int firstLvalPageNum = PageChannel.INVALID_PAGE_NUMBER;
        byte firstLvalRow = 0;

        // write other page(s)
        switch (type) {
        case LONG_VALUE_TYPE_OTHER_PAGE:
            lvalPage = getLongValuePage(value.length, lvalBufferH);
            firstLvalPageNum = lvalBufferH.getPageNumber();
            firstLvalRow = (byte) Table.addDataPageRow(lvalPage, value.length, getFormat(), 0);
            lvalPage.put(value);
            getPageChannel().writePage(lvalPage, firstLvalPageNum);
            break;

        case LONG_VALUE_TYPE_OTHER_PAGES:

            ByteBuffer buffer = ByteBuffer.wrap(value);
            int remainingLen = buffer.remaining();
            buffer.limit(0);
            lvalPage = getLongValuePage(getFormat().MAX_LONG_VALUE_ROW_SIZE, lvalBufferH);
            firstLvalPageNum = lvalBufferH.getPageNumber();
            int lvalPageNum = firstLvalPageNum;
            ByteBuffer nextLvalPage = null;
            int nextLvalPageNum = 0;
            while (remainingLen > 0) {
                lvalPage.clear();

                // figure out how much we will put in this page (we need 4 bytes for
                // the next page pointer)
                int chunkLength = Math.min(getFormat().MAX_LONG_VALUE_ROW_SIZE - 4, remainingLen);

                // figure out if we will need another page, and if so, allocate it
                if (chunkLength < remainingLen) {
                    // force a new page to be allocated
                    lvalBufferH.clear();
                    nextLvalPage = getLongValuePage(getFormat().MAX_LONG_VALUE_ROW_SIZE, lvalBufferH);
                    nextLvalPageNum = lvalBufferH.getPageNumber();
                } else {
                    nextLvalPage = null;
                    nextLvalPageNum = 0;
                }

                // add row to this page
                byte lvalRow = (byte) Table.addDataPageRow(lvalPage, chunkLength + 4, getFormat(), 0);

                // write next page info (we'll always be writing into row 0 for
                // newly created pages)
                lvalPage.put((byte) 0); // row number
                ByteUtil.put3ByteInt(lvalPage, nextLvalPageNum); // page number

                // write this page's chunk of data
                buffer.limit(buffer.limit() + chunkLength);
                lvalPage.put(buffer);
                remainingLen -= chunkLength;

                // write new page to database
                getPageChannel().writePage(lvalPage, lvalPageNum);

                if (lvalPageNum == firstLvalPageNum) {
                    // save initial row info
                    firstLvalRow = lvalRow;
                } else {
                    // check assertion that we wrote to row 0 for all subsequent pages
                    if (lvalRow != (byte) 0) {
                        throw new IllegalStateException("Expected row 0, but was " + lvalRow);
                    }
                }

                // move to next page
                lvalPage = nextLvalPage;
                lvalPageNum = nextLvalPageNum;
            }
            break;

        default:
            throw new IOException("Unrecognized long value type: " + type);
        }

        // update def
        def.put(firstLvalRow);
        ByteUtil.put3ByteInt(def, firstLvalPageNum);
        def.putInt(0); //Unknown

    }

    def.flip();
    return def;
}

From source file:com.castis.sysComp.PoisConverterSysComp.java

public void parseRegionFile(File file) throws Exception {
    String line = "";
    FileInputStream in = null;//from  w  ww . ja  v  a2s. c  o m
    Reader isReader = null;
    LineNumberReader bufReader = null;

    FileOutputStream fos = null;
    String fileName = file.getName();

    int index = fileName.indexOf("-");
    if (index != -1) {
        fileName = fileName.substring(index + 1, fileName.length());
    }

    String dir = filePolling.getValidFileDirectory(resultDir);

    String tempDir = dir + "/temp/";
    File targetDirectory = new File(CiFileUtil.getReplaceFullPath(tempDir));
    if (!targetDirectory.isDirectory()) {
        CiFileUtil.createDirectory(tempDir);
    }

    fos = new FileOutputStream(tempDir + fileName);

    int byteSize = 2048;
    ByteBuffer byteBuffer = ByteBuffer.allocateDirect(byteSize);
    GatheringByteChannel outByteCh = fos.getChannel();

    try {
        in = new FileInputStream(file);
        isReader = new InputStreamReader(in, "UTF-16LE");
        bufReader = new LineNumberReader(isReader);

        boolean first = true;
        while ((line = bufReader.readLine()) != null) {

            byte[] utf8 = line.getBytes("UTF-8");
            String string = new String(utf8, "UTF-8");

            String data[] = string.split("\t");

            if (first == true) {
                first = false;

                if (data[0] == null || data[0].contains("region") == false) {
                    throw new DataParsingException("data parsing error(not formatted)");
                }
                continue;
            }

            if (data[0] == null || data[0].equals("")) {
                throw new DataParsingException("data parsing error(region id)");
            }
            if (data[1] == null || data[1].equals("")) {
                throw new DataParsingException("data parsing error(region name)");
            }
            if (data[2] == null || data[2].equals("")) {
                throw new DataParsingException("data parsing error(parent id)");
            }

            StringBuffer strBuffer = new StringBuffer();
            strBuffer.append(data[0]);
            strBuffer.append("\t");
            strBuffer.append(data[1]);
            strBuffer.append("\t");
            strBuffer.append(data[2]);

            strBuffer.append("\r\n");

            byte[] outByte = null;
            try {
                outByte = strBuffer.toString().getBytes("UTF-8");
            } catch (UnsupportedEncodingException e2) {
                e2.printStackTrace();
            }
            byteBuffer.put(outByte);
            byteBuffer.flip();
            try {
                outByteCh.write(byteBuffer);
            } catch (IOException e) {
            }
            byteBuffer.clear();
        }

        fos.close();

        index = fileName.indexOf("_");

        String targetDir = resultDir;
        File sourceFile = new File(tempDir + fileName);
        if (index != -1) {
            String directory = fileName.substring(0, index);
            targetDir += "/" + directory;
        }

        try {

            File resultTargetDir = new File(CiFileUtil.getReplaceFullPath(targetDir));
            if (!resultTargetDir.isDirectory()) {
                CiFileUtil.createDirectory(targetDir);
            }

            CiFileUtil.renameFile(sourceFile, targetDir, fileName);
        } catch (Exception e) {
            log.error(e.getMessage());
        }

    } catch (Exception e) {
        String errorMsg = "Fail to parsing Line.[current line(" + bufReader.getLineNumber() + ") :" + line
                + "] : ";
        log.error(errorMsg, e);
        throw new DataParsingException(errorMsg, e); //throw(e);

    } finally {
        if (in != null)
            in.close();
        if (isReader != null)
            isReader.close();
        if (bufReader != null)
            bufReader.close();
    }
}