Example usage for java.util.concurrent LinkedBlockingQueue add

List of usage examples for java.util.concurrent LinkedBlockingQueue add

  1. HOME
  2. Java
  3. java
  4. java.util.concurrent.*
  5. LinkedBlockingQueue
  6. add

Introduction

In this page you can find the example usage for java.util.concurrent LinkedBlockingQueue add.

Prototype

boolean add(E e);

Source Link

Document

Inserts the specified element into this queue if it is possible to do so immediately without violating capacity restrictions, returning true upon success and throwing an IllegalStateException if no space is currently available.

Usage

From source file:org.commoncrawl.service.listcrawler.CrawlHistoryManager.java

private static void launchInTestMode() {

    File baseTestDir = new File("/tmp/logManagerTest");
    FileUtils.recursivelyDeleteFile(baseTestDir);
    baseTestDir.mkdir();//from  w  w  w  . ja va  2  s . co m
    File remoteDir = new File(baseTestDir, "remote");
    File localDir = new File(baseTestDir, "local");
    remoteDir.mkdir();
    localDir.mkdir();

    final TreeMap<String, URLFP> urlToFPMap = new TreeMap<String, URLFP>();
    final TreeMap<URLFP, String> urlFPToString = new TreeMap<URLFP, String>();

    Set<String> list1 = Sets.newHashSet(urlList1);
    Set<String> list2 = Sets.newHashSet(urlList2);
    final Set<String> combined = Sets.union(list1, list2);
    Set<String> difference = Sets.difference(list1, list2);
    final Set<String> completedURLS = new HashSet<String>();
    for (String url : combined) {
        URLFP fingerprint = URLUtils.getURLFPFromURL(url, true);
        urlToFPMap.put(url, fingerprint);
        urlFPToString.put(fingerprint, url);
    }

    File testInputFile1 = new File(localDir, "INPUT_LIST-" + System.currentTimeMillis());
    File testInputFile2 = new File(localDir, "INPUT_LIST-" + (System.currentTimeMillis() + 1));

    try {

        generateTestURLFile(testInputFile1, urlList1);
        generateTestURLFile(testInputFile2, urlList2);

        FileSystem localFileSystem = FileSystem.getLocal(CrawlEnvironment.getHadoopConfig());

        EventLoop eventLoop = new EventLoop();
        eventLoop.start();

        final CrawlHistoryManager logManager = new CrawlHistoryManager(localFileSystem,
                new Path(remoteDir.getAbsolutePath()), localDir, eventLoop, 0);

        final LinkedBlockingQueue<ProxyCrawlHistoryItem> queue = new LinkedBlockingQueue<ProxyCrawlHistoryItem>();

        final Semaphore initialListComplete = new Semaphore(0);

        logManager.startQueueLoaderThread(new CrawlQueueLoader() {

            @Override
            public void queueURL(URLFP urlfp, String url) {
                ProxyCrawlHistoryItem item = new ProxyCrawlHistoryItem();
                item.setOriginalURL(url);
                queue.add(item);
            }

            @Override
            public void flush() {
                // TODO Auto-generated method stub

            }
        });

        Thread queueTestThread = new Thread(new Runnable() {

            @Override
            public void run() {
                while (true) {
                    try {
                        ProxyCrawlHistoryItem item = queue.take();

                        if (item.getOriginalURL().length() == 0) {
                            break;
                        } else {

                            System.out.println("Got:" + item.getOriginalURL());

                            CrawlURL urlObject = new CrawlURL();

                            Assert.assertTrue(!completedURLS.contains(item.getOriginalURL()));
                            completedURLS.add(item.getOriginalURL());

                            urlObject.setLastAttemptResult((byte) CrawlURL.CrawlResult.SUCCESS);
                            urlObject.setUrl(item.getOriginalURL());
                            urlObject.setResultCode(200);

                            logManager.crawlComplete(urlObject);

                            if (completedURLS.equals(combined)) {
                                System.out.println("Hit Trigger URL. Releasing InitialListComplete Sempahore");
                                initialListComplete.release(1);
                            }
                        }

                    } catch (InterruptedException e) {
                    }
                }
            }

        });

        queueTestThread.start();

        logManager.loadList(testInputFile1, 0);
        logManager.loadList(testInputFile2, 0);
        System.out.println("Waiting for Initial List to Complete");
        initialListComplete.acquireUninterruptibly();
        System.out.println("Woke Up");

        try {
            eventLoop.getEventThread().join();
        } catch (InterruptedException e) {
            e.printStackTrace();
        }

    } catch (IOException e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
    }
}

From source file:com.test.HibernateDerbyLockingTest.java

public void testJDBC() throws Exception {
    final DerbyTemplate template = new DerbyTemplate();
    try {/*from  w  w  w . ja va  2 s . c o m*/
        template.doWithStatement(new IStatementCallback() {
            public void execute(Statement statement) throws Exception {
                statement.execute(
                        "CREATE TABLE TEST(\n" + "ID CHAR(36) NOT NULL,\n" + "NAME VARCHAR(255)\n" + ")");
            }
        });
        template.doWithStatement(new IStatementCallback() {
            public void execute(Statement statement) throws Exception {
                statement.execute("INSERT INTO TEST(ID, NAME) VALUES('12345', 'Bob')");
            }
        });

        final LinkedBlockingQueue<String> queue = new LinkedBlockingQueue<String>();

        ExecutorService executorService = Executors.newCachedThreadPool();
        Future<?> submit = executorService.submit(new Callable<Object>() {

            public Object call() throws Exception {
                template.doWithStatement(new IStatementCallback() {
                    public void execute(Statement statement) throws Exception {
                        ResultSet resultSet = statement.executeQuery(
                                "SELECT ID, NAME FROM TEST WHERE ID = '12345' for update with rs");
                        while (resultSet.next()) {
                            String id = resultSet.getString("ID");
                            String name = resultSet.getString("NAME");
                        }
                        try {
                            Thread.sleep(2000);
                        } catch (Throwable t) {
                        }
                        System.out.println("one");
                        queue.add("one");
                        try {
                            Thread.sleep(500);
                        } catch (Throwable t) {
                        }
                    }
                });
                return null;
            }
        });
        Thread.sleep(500);
        Future<?> submit2 = executorService.submit(new Callable<Object>() {
            public Object call() throws Exception {
                template.doWithStatement(new IStatementCallback() {
                    public void execute(Statement statement) throws Exception {
                        ResultSet resultSet = statement.executeQuery(
                                "SELECT ID, NAME FROM TEST WHERE ID = '12345' for update with rr");
                        while (resultSet.next()) {
                            String id = resultSet.getString("ID");
                            String name = resultSet.getString("NAME");
                        }
                        queue.add("two");
                        System.out.println("two");
                    }
                });
                return null;
            }
        });
        submit.get();
        submit2.get();
        assertEquals("one", queue.poll(3, TimeUnit.SECONDS));
        assertEquals("two", queue.poll(3, TimeUnit.SECONDS));
    } finally {
        template.doWithStatement(new IStatementCallback() {
            public void execute(Statement statement) throws Exception {
                statement.execute("DROP TABLE TEST");
            }
        });
    }
}

From source file:com.ibm.crail.tools.CrailBenchmark.java

void getFileAsync(String filename, int loop, int batch) throws Exception, InterruptedException {
    System.out.println("getFileAsync, filename " + filename + ", loop " + loop + ", batch " + batch);

    //warmup/* ww w  . j  a va2  s .c om*/
    ConcurrentLinkedQueue<CrailBuffer> bufferQueue = new ConcurrentLinkedQueue<CrailBuffer>();
    CrailBuffer buf = fs.allocateBuffer();
    bufferQueue.add(buf);
    warmUp(filename, warmup, bufferQueue);
    fs.freeBuffer(buf);

    //benchmark
    System.out.println("starting benchmark...");
    fs.getStatistics().reset();
    LinkedBlockingQueue<Future<CrailNode>> fileQueue = new LinkedBlockingQueue<Future<CrailNode>>();
    long start = System.currentTimeMillis();
    for (int i = 0; i < loop; i++) {
        //single operation == loop
        for (int j = 0; j < batch; j++) {
            Future<CrailNode> future = fs.lookup(filename);
            fileQueue.add(future);
        }
        for (int j = 0; j < batch; j++) {
            Future<CrailNode> future = fileQueue.poll();
            future.get();
        }
    }
    long end = System.currentTimeMillis();
    double executionTime = ((double) (end - start));
    double latency = executionTime * 1000.0 / ((double) batch);
    System.out.println("execution time [ms] " + executionTime);
    System.out.println("latency [us] " + latency);

    fs.getStatistics().print("close");
}

From source file:com.ibm.crail.tools.CrailBenchmark.java

void createFileAsync(String filename, int loop, int batch) throws Exception, InterruptedException {
    System.out.println("createFileAsync, filename " + filename + ", loop " + loop + ", batch " + batch);

    //warmup//from w w  w .j  a v a  2  s . co  m
    ConcurrentLinkedQueue<CrailBuffer> bufferQueue = new ConcurrentLinkedQueue<CrailBuffer>();
    CrailBuffer buf = fs.allocateBuffer();
    bufferQueue.add(buf);
    warmUp(filename, warmup, bufferQueue);
    fs.freeBuffer(buf);

    //benchmark
    System.out.println("starting benchmark...");
    fs.getStatistics().reset();
    LinkedBlockingQueue<Future<CrailNode>> futureQueue = new LinkedBlockingQueue<Future<CrailNode>>();
    LinkedBlockingQueue<CrailFile> fileQueue = new LinkedBlockingQueue<CrailFile>();
    LinkedBlockingQueue<String> pathQueue = new LinkedBlockingQueue<String>();
    fs.create(filename, CrailNodeType.DIRECTORY, CrailStorageClass.DEFAULT, CrailLocationClass.DEFAULT).get()
            .syncDir();

    for (int i = 0; i < loop; i++) {
        String name = "/" + i;
        String f = filename + name;
        pathQueue.add(f);
    }

    long start = System.currentTimeMillis();
    for (int i = 0; i < loop; i += batch) {
        //single operation == loop
        for (int j = 0; j < batch; j++) {
            String path = pathQueue.poll();
            Future<CrailNode> future = fs.create(path, CrailNodeType.DATAFILE, CrailStorageClass.DEFAULT,
                    CrailLocationClass.DEFAULT);
            futureQueue.add(future);
        }
        for (int j = 0; j < batch; j++) {
            Future<CrailNode> future = futureQueue.poll();
            CrailFile file = future.get().asFile();
            fileQueue.add(file);
        }
        for (int j = 0; j < batch; j++) {
            CrailFile file = fileQueue.poll();
            file.syncDir();
        }
    }
    long end = System.currentTimeMillis();
    double executionTime = ((double) (end - start));
    double latency = executionTime * 1000.0 / ((double) loop);
    System.out.println("execution time [ms] " + executionTime);
    System.out.println("latency [us] " + latency);

    fs.delete(filename, true).get().syncDir();

    fs.getStatistics().print("close");

}

From source file:com.ibm.crail.tools.CrailBenchmark.java

void createFile(String filename, int loop) throws Exception, InterruptedException {
    System.out.println("createFile, filename " + filename + ", loop " + loop);

    //warmup//from   w  ww.  j a  v a2s  .  c  om
    ConcurrentLinkedQueue<CrailBuffer> bufferQueue = new ConcurrentLinkedQueue<CrailBuffer>();
    CrailBuffer buf = fs.allocateBuffer();
    bufferQueue.add(buf);
    warmUp(filename, warmup, bufferQueue);
    fs.freeBuffer(buf);

    //benchmark
    System.out.println("starting benchmark...");
    fs.getStatistics().reset();
    LinkedBlockingQueue<String> pathQueue = new LinkedBlockingQueue<String>();
    fs.create(filename, CrailNodeType.DIRECTORY, CrailStorageClass.DEFAULT, CrailLocationClass.DEFAULT).get()
            .syncDir();
    int filecounter = 0;
    for (int i = 0; i < loop; i++) {
        String name = "" + filecounter++;
        String f = filename + "/" + name;
        pathQueue.add(f);
    }

    double ops = 0;
    long start = System.currentTimeMillis();
    while (!pathQueue.isEmpty()) {
        String path = pathQueue.poll();
        fs.create(path, CrailNodeType.DATAFILE, CrailStorageClass.DEFAULT, CrailLocationClass.DEFAULT).get()
                .syncDir();
    }
    long end = System.currentTimeMillis();
    double executionTime = ((double) (end - start)) / 1000.0;
    double latency = 0.0;
    if (executionTime > 0) {
        latency = 1000000.0 * executionTime / ops;
    }

    System.out.println("execution time " + executionTime);
    System.out.println("ops " + ops);
    System.out.println("latency " + latency);

    fs.getStatistics().print("close");
}

From source file:com.ibm.crail.tools.CrailBenchmark.java

void readSequentialAsync(String filename, int size, int loop, int batch) throws Exception {
    System.out.println("readSequentialAsync, filename " + filename + ", size " + size + ", loop " + loop
            + ", batch " + batch);

    ConcurrentLinkedQueue<CrailBuffer> bufferQueue = new ConcurrentLinkedQueue<CrailBuffer>();
    for (int i = 0; i < batch; i++) {
        CrailBuffer buf = null;//from   w w  w. j a  v a  2 s. co  m
        if (size == CrailConstants.BUFFER_SIZE) {
            buf = fs.allocateBuffer();
        } else if (size < CrailConstants.BUFFER_SIZE) {
            CrailBuffer _buf = fs.allocateBuffer();
            _buf.clear().limit(size);
            buf = _buf.slice();
        } else {
            buf = OffHeapBuffer.wrap(ByteBuffer.allocateDirect(size));
        }
        bufferQueue.add(buf);
    }

    //warmup
    warmUp(filename, warmup, bufferQueue);

    //benchmark
    System.out.println("starting benchmark...");
    double sumbytes = 0;
    double ops = 0;
    fs.getStatistics().reset();
    CrailFile file = fs.lookup(filename).get().asFile();
    CrailInputStream directStream = file.getDirectInputStream(file.getCapacity());
    HashMap<Integer, CrailBuffer> futureMap = new HashMap<Integer, CrailBuffer>();
    LinkedBlockingQueue<Future<CrailResult>> futureQueue = new LinkedBlockingQueue<Future<CrailResult>>();
    long start = System.currentTimeMillis();
    for (int i = 0; i < batch - 1 && ops < loop; i++) {
        CrailBuffer buf = bufferQueue.poll();
        buf.clear();
        Future<CrailResult> future = directStream.read(buf);
        futureQueue.add(future);
        futureMap.put(future.hashCode(), buf);
        ops = ops + 1.0;
    }
    while (ops < loop) {
        CrailBuffer buf = bufferQueue.poll();
        buf.clear();
        Future<CrailResult> future = directStream.read(buf);
        futureQueue.add(future);
        futureMap.put(future.hashCode(), buf);

        future = futureQueue.poll();
        CrailResult result = future.get();
        buf = futureMap.get(future.hashCode());
        bufferQueue.add(buf);

        sumbytes = sumbytes + result.getLen();
        ops = ops + 1.0;
    }
    while (!futureQueue.isEmpty()) {
        Future<CrailResult> future = futureQueue.poll();
        CrailResult result = future.get();
        futureMap.get(future.hashCode());
        sumbytes = sumbytes + result.getLen();
        ops = ops + 1.0;
    }
    long end = System.currentTimeMillis();
    double executionTime = ((double) (end - start)) / 1000.0;
    double throughput = 0.0;
    double latency = 0.0;
    double sumbits = sumbytes * 8.0;
    if (executionTime > 0) {
        throughput = sumbits / executionTime / 1000.0 / 1000.0;
        latency = 1000000.0 * executionTime / ops;
    }
    directStream.close();

    System.out.println("execution time " + executionTime);
    System.out.println("ops " + ops);
    System.out.println("sumbytes " + sumbytes);
    System.out.println("throughput " + throughput);
    System.out.println("latency " + latency);

    fs.getStatistics().print("close");
}

From source file:com.ibm.crail.tools.CrailBenchmark.java

void writeAsync(String filename, int size, int loop, int batch, int storageClass, int locationClass)
        throws Exception {
    System.out.println("writeAsync, filename " + filename + ", size " + size + ", loop " + loop + ", batch "
            + batch + ", storageClass " + storageClass + ", locationClass " + locationClass);

    ConcurrentLinkedQueue<CrailBuffer> bufferQueue = new ConcurrentLinkedQueue<CrailBuffer>();
    for (int i = 0; i < batch; i++) {
        CrailBuffer buf = null;/*from w  w w.j a  v a 2 s .c o m*/
        if (size == CrailConstants.BUFFER_SIZE) {
            buf = fs.allocateBuffer();
        } else if (size < CrailConstants.BUFFER_SIZE) {
            CrailBuffer _buf = fs.allocateBuffer();
            _buf.clear().limit(size);
            buf = _buf.slice();
        } else {
            buf = OffHeapBuffer.wrap(ByteBuffer.allocateDirect(size));
        }
        bufferQueue.add(buf);
    }

    //warmup
    warmUp(filename, warmup, bufferQueue);

    //benchmark
    System.out.println("starting benchmark...");
    LinkedBlockingQueue<Future<CrailResult>> futureQueue = new LinkedBlockingQueue<Future<CrailResult>>();
    HashMap<Integer, CrailBuffer> futureMap = new HashMap<Integer, CrailBuffer>();
    fs.getStatistics().reset();
    long _loop = (long) loop;
    long _bufsize = (long) CrailConstants.BUFFER_SIZE;
    long _capacity = _loop * _bufsize;
    double sumbytes = 0;
    double ops = 0;
    CrailFile file = fs.create(filename, CrailNodeType.DATAFILE, CrailStorageClass.get(storageClass),
            CrailLocationClass.get(locationClass)).get().asFile();
    CrailOutputStream directStream = file.getDirectOutputStream(_capacity);
    long start = System.currentTimeMillis();
    for (int i = 0; i < batch - 1 && ops < loop; i++) {
        CrailBuffer buf = bufferQueue.poll();
        buf.clear();
        Future<CrailResult> future = directStream.write(buf);
        futureQueue.add(future);
        futureMap.put(future.hashCode(), buf);
        ops = ops + 1.0;
    }
    while (ops < loop) {
        CrailBuffer buf = bufferQueue.poll();
        buf.clear();
        Future<CrailResult> future = directStream.write(buf);
        futureQueue.add(future);
        futureMap.put(future.hashCode(), buf);

        future = futureQueue.poll();
        future.get();
        buf = futureMap.get(future.hashCode());
        bufferQueue.add(buf);

        sumbytes = sumbytes + buf.capacity();
        ops = ops + 1.0;
    }
    while (!futureQueue.isEmpty()) {
        Future<CrailResult> future = futureQueue.poll();
        future.get();
        CrailBuffer buf = futureMap.get(future.hashCode());
        sumbytes = sumbytes + buf.capacity();
        ops = ops + 1.0;
    }
    long end = System.currentTimeMillis();
    double executionTime = ((double) (end - start)) / 1000.0;
    double throughput = 0.0;
    double latency = 0.0;
    double sumbits = sumbytes * 8.0;
    if (executionTime > 0) {
        throughput = sumbits / executionTime / 1000.0 / 1000.0;
        latency = 1000000.0 * executionTime / ops;
    }
    directStream.close();

    System.out.println("execution time " + executionTime);
    System.out.println("ops " + ops);
    System.out.println("sumbytes " + sumbytes);
    System.out.println("throughput " + throughput);
    System.out.println("latency " + latency);

    fs.getStatistics().print("close");
}

From source file:com.ibm.crail.tools.CrailBenchmark.java

void collectionTest(int size, int loop) throws Exception {
    System.out.println("collectionTest, size " + size + ", loop " + loop);

    RingBuffer<Object> ringBuffer = new RingBuffer<Object>(10);
    ArrayBlockingQueue<Object> arrayQueue = new ArrayBlockingQueue<Object>(10);
    LinkedBlockingQueue<Object> listQueue = new LinkedBlockingQueue<Object>();

    Object obj = new Object();
    long start = System.currentTimeMillis();
    for (int i = 0; i < loop; i++) {
        for (int j = 0; j < size; j++) {
            ringBuffer.add(obj);/*from w  w w .  j a  v a  2  s .  c o  m*/
            Object tmp = ringBuffer.peek();
            tmp = ringBuffer.poll();
        }
    }
    long end = System.currentTimeMillis();
    double executionTime = ((double) (end - start));
    System.out.println("ringbuffer, execution time [ms] " + executionTime);

    start = System.currentTimeMillis();
    for (int i = 0; i < loop; i++) {
        for (int j = 0; j < size; j++) {
            arrayQueue.add(obj);
            Object tmp = arrayQueue.peek();
            tmp = arrayQueue.poll();
        }
    }
    end = System.currentTimeMillis();
    executionTime = ((double) (end - start));
    System.out.println("arrayQueue, execution time [ms] " + executionTime);

    start = System.currentTimeMillis();
    for (int i = 0; i < loop; i++) {
        for (int j = 0; j < size; j++) {
            listQueue.add(obj);
            Object tmp = listQueue.peek();
            tmp = listQueue.poll();
        }
    }
    end = System.currentTimeMillis();
    executionTime = ((double) (end - start));
    System.out.println("arrayQueue, execution time [ms] " + executionTime);
}

From source file:org.springframework.integration.jms.JmsOutboundGateway.java

private javax.jms.Message doSendAndReceiveAsyncDefaultCorrelation(Destination requestDestination,
        javax.jms.Message jmsRequest, Session session, int priority) throws JMSException {
    String correlationId = null;/*w w w.j av  a 2 s.co m*/
    MessageProducer messageProducer = null;

    try {
        messageProducer = session.createProducer(requestDestination);
        LinkedBlockingQueue<javax.jms.Message> replyQueue = new LinkedBlockingQueue<javax.jms.Message>(1);

        this.sendRequestMessage(jmsRequest, messageProducer, priority);

        correlationId = jmsRequest.getJMSMessageID();

        if (logger.isDebugEnabled()) {
            logger.debug(this.getComponentName() + " Sent message with correlationId " + correlationId);
        }
        this.replies.put(correlationId, replyQueue);

        /*
         * Check to see if the reply arrived before we obtained the correlationId
         */
        synchronized (this.earlyOrLateReplies) {
            TimedReply timedReply = this.earlyOrLateReplies.remove(correlationId);
            if (timedReply != null) {
                if (logger.isDebugEnabled()) {
                    logger.debug("Found early reply with correlationId " + correlationId);
                }
                replyQueue.add(timedReply.getReply());
            }
        }

        return obtainReplyFromContainer(correlationId, replyQueue);
    } finally {
        JmsUtils.closeMessageProducer(messageProducer);
        if (correlationId != null) {
            this.replies.remove(correlationId);
        }
    }
}

From source file:org.springframework.integration.jms.JmsOutboundGateway.java

private void onMessageSync(javax.jms.Message message, String correlationId) {
    try {//w  w w. j  ava 2 s  .c o m
        LinkedBlockingQueue<javax.jms.Message> queue = this.replies.get(correlationId);
        if (queue == null) {
            if (this.correlationKey != null) {
                Log debugLogger = LogFactory.getLog("si.jmsgateway.debug");
                if (debugLogger.isDebugEnabled()) {
                    Object siMessage = this.messageConverter.fromMessage(message);
                    debugLogger.debug("No pending reply for " + siMessage + " with correlationId: "
                            + correlationId + " pending replies: " + this.replies.keySet());
                }
                throw new RuntimeException("No sender waiting for reply");
            }
            synchronized (this.earlyOrLateReplies) {
                queue = this.replies.get(correlationId);
                if (queue == null) {
                    if (logger.isDebugEnabled()) {
                        logger.debug("Reply for correlationId " + correlationId + " received early or late");
                    }
                    this.earlyOrLateReplies.put(correlationId, new TimedReply(message));
                }
            }
        }
        if (queue != null) {
            if (logger.isDebugEnabled()) {
                logger.debug("Received reply with correlationId " + correlationId);
            }
            queue.add(message);
        }
    } catch (Exception e) {
        if (logger.isWarnEnabled()) {
            logger.warn("Failed to consume reply with correlationId " + correlationId, e);
        }
    }
}