Example usage for java.util.concurrent LinkedBlockingDeque poll

Introduction

In this page you can find the example usage for java.util.concurrent LinkedBlockingDeque poll.

Prototype

public E poll() 

Source Link

Usage

From source file:org.commoncrawl.service.listcrawler.DataTransferAgent.java

static int uploadSingeFile(CCBridgeServerMapping mapping, FileSystem fs, Configuration conf, Path hdfsFilePath,
        String uploadName, EventLoop eventLoop) throws IOException {

    final FileStatus fileStatus = fs.getFileStatus(hdfsFilePath);
    LOG.info("Uploading:" + uploadName + " size:" + fileStatus.getLen() + " to:" + mapping._internalName);

    {/* w ww  . j av a  2 s.co  m*/
        // construct url 
        URL fePostURL = new URL("http://" + mapping._externalName + ":8090/");
        LOG.info("POST URL IS:" + fePostURL.toString());

        // open input stream 
        final FSDataInputStream is = fs.open(hdfsFilePath);
        final Semaphore blockingSemaphore = new Semaphore(0);
        NIOHttpConnection connection = null;
        try {
            // create connection 
            connection = new NIOHttpConnection(fePostURL, eventLoop.getSelector(), eventLoop.getResolver(),
                    null);
            // set listener 
            connection.setListener(new Listener() {

                @Override
                public void HttpConnectionStateChanged(NIOHttpConnection theConnection, State oldState,
                        State state) {
                    LOG.info("Connection State Changed to:" + state.toString());
                    if (state == State.DONE || state == State.ERROR) {
                        //LOG.info("Connection Transition to Done or Error");
                        //LOG.info("Response Headers:" + theConnection.getResponseHeaders().toString());
                        blockingSemaphore.release();
                    }
                }

                @Override
                public void HttpContentAvailable(NIOHttpConnection theConnection, NIOBufferList contentBuffer) {
                    // TODO Auto-generated method stub

                }
            });
            // set headers 
            connection.getRequestHeaders().reset();
            connection.getRequestHeaders().prepend("PUT /put?src=" + uploadName + " HTTP/1.1", null);
            connection.getRequestHeaders().set("Host", mapping._internalName + ":8090");
            connection.getRequestHeaders().set("Content-Length", Long.toString(fileStatus.getLen()));
            connection.getRequestHeaders().set("Connection", "keep-alive");
            connection.setPopulateDefaultHeaderItems(false);

            final LinkedBlockingDeque<BufferStruct> _loaderQueue = new LinkedBlockingDeque<BufferStruct>(20);
            final AtomicBoolean eof = new AtomicBoolean();
            final ByteBuffer sentinel = ByteBuffer.allocate(4096);
            sentinel.position(sentinel.position());
            final Thread loaderThread = new Thread(new Runnable() {

                int _id = 0;

                @Override
                public void run() {
                    int bytesRead;
                    byte incomingBuffer[] = new byte[4096 * 10];
                    try {
                        while ((bytesRead = is.read(incomingBuffer)) != -1) {
                            ByteBuffer buffer = ByteBuffer.wrap(incomingBuffer, 0, bytesRead);
                            buffer.position(bytesRead);

                            //LOG.info("Loader Thread Read:"+ bytesRead + " Buffer:" + ++_id);
                            try {
                                _loaderQueue.put(new BufferStruct(buffer, _id));
                            } catch (InterruptedException e) {
                                LOG.error(CCStringUtils.stringifyException(e));
                                break;
                            }
                            incomingBuffer = new byte[4096 * 10];
                        }
                        try {
                            _loaderQueue.put(new BufferStruct(sentinel, ++_id));
                        } catch (InterruptedException e) {
                        }
                    } catch (IOException e) {
                        LOG.error(CCStringUtils.stringifyException(e));
                        return;
                    }
                }

            });

            loaderThread.start();

            // set data source ... 
            connection.setDataSource(new DataSource() {

                int bytesTransferred = 0;

                @Override
                public boolean read(NIOBufferList dataBuffer) throws IOException {
                    if (eof.get())
                        return true;
                    //LOG.info("Connect read callback triggered");
                    BufferStruct buffer = _loaderQueue.poll();
                    if (buffer != null) {
                        if (buffer._buffer != sentinel) {
                            //LOG.info("Got Buffer:"+ buffer._id);
                            if (buffer._id == 1) {
                                //LOG.info("Inital Buffer Bytes:" + new String(buffer._buffer.array(),0,10).toString());
                            }
                            dataBuffer.write(buffer._buffer);
                            bytesTransferred += buffer._buffer.limit();
                            //LOG.info("Read:" + buffer._buffer.limit() + " Transfered:" + bytesTransferred);
                            return false;
                        } else {
                            //LOG.info("EOF Condition");
                            dataBuffer.write(sentinel);
                            eof.set(true);
                            return true;
                        }
                    }
                    return false;
                }
            });

            // open connection 
            connection.open();
            // wait for connection to complete ... 
            blockingSemaphore.acquireUninterruptibly();
            // kill loader thread 
            loaderThread.interrupt();
            try {
                LOG.info("Waiting for Loader Thread");
                loaderThread.join();
                LOG.info("Done Waiting for Loader Thread");
            } catch (InterruptedException e) {
            }
        } finally {
            is.close();
            if (connection != null) {
                connection.close();
                LOG.info("Response Code for File:" + uploadName + "to Host: " + mapping._internalName + " is:"
                        + connection.getResponseHeaders().getHttpResponseCode());
                return connection.getResponseHeaders().getHttpResponseCode();
                /*
                if (connection.getResponseHeaders().getHttpResponseCode() != 200) { 
                  throw new IOException("Failed to upload file:" + dataFile.getName() + " responseCode:" + connection.getResponseHeaders().getHttpResponseCode());
                }
                */
            }
        }
    }
    // something went wrong ??? 
    LOG.error("Failed to upload file:" + uploadName + " unknown response code");
    return 500;
}

---