Example usage for java.util Deque addAll

Introduction

In this page you can find the example usage for java.util Deque addAll.

Prototype

boolean addAll(Collection<? extends E> c);

Source Link

Document

Adds all of the elements in the specified collection at the end of this deque, as if by calling #addLast on each one, in the order that they are returned by the collection's iterator.

Usage

From source file:org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles.java

/**
 * @return A Multimap<startkey, LoadQueueItem> that groups LQI by likely
 * bulk load region targets./*from  w  ww.  j  a  v a 2s.  c om*/
 */
private Multimap<ByteBuffer, LoadQueueItem> groupOrSplitPhase(final HTable table, ExecutorService pool,
        Deque<LoadQueueItem> queue, final Pair<byte[][], byte[][]> startEndKeys) throws IOException {
    // <region start key, LQI> need synchronized only within this scope of this
    // phase because of the puts that happen in futures.
    Multimap<ByteBuffer, LoadQueueItem> rgs = HashMultimap.create();
    final Multimap<ByteBuffer, LoadQueueItem> regionGroups = Multimaps.synchronizedMultimap(rgs);

    // drain LQIs and figure out bulk load groups
    Set<Future<List<LoadQueueItem>>> splittingFutures = new HashSet<Future<List<LoadQueueItem>>>();
    while (!queue.isEmpty()) {
        final LoadQueueItem item = queue.remove();

        final Callable<List<LoadQueueItem>> call = new Callable<List<LoadQueueItem>>() {
            public List<LoadQueueItem> call() throws Exception {
                List<LoadQueueItem> splits = groupOrSplit(regionGroups, item, table, startEndKeys);
                return splits;
            }
        };
        splittingFutures.add(pool.submit(call));
    }
    // get all the results.  All grouping and splitting must finish before
    // we can attempt the atomic loads.
    for (Future<List<LoadQueueItem>> lqis : splittingFutures) {
        try {
            List<LoadQueueItem> splits = lqis.get();
            if (splits != null) {
                queue.addAll(splits);
            }
        } catch (ExecutionException e1) {
            Throwable t = e1.getCause();
            if (t instanceof IOException) {
                LOG.error("IOException during splitting", e1);
                throw (IOException) t; // would have been thrown if not parallelized,
            }
            LOG.error("Unexpected execution exception during splitting", e1);
            throw new IllegalStateException(t);
        } catch (InterruptedException e1) {
            LOG.error("Unexpected interrupted exception during splitting", e1);
            throw (InterruptedIOException) new InterruptedIOException().initCause(e1);
        }
    }
    return regionGroups;
}

---

From source file:org.apache.hadoop.hbase.tool.LoadIncrementalHFiles.java

/**
 * This takes the LQI's grouped by likely regions and attempts to bulk load them. Any failures are
 * re-queued for another pass with the groupOrSplitPhase.
 * <p>//from ww  w  .  j  ava 2 s .  co  m
 * protected for testing.
 */
@VisibleForTesting
protected void bulkLoadPhase(Table table, Connection conn, ExecutorService pool, Deque<LoadQueueItem> queue,
        Multimap<ByteBuffer, LoadQueueItem> regionGroups, boolean copyFile,
        Map<LoadQueueItem, ByteBuffer> item2RegionMap) throws IOException {
    // atomically bulk load the groups.
    Set<Future<List<LoadQueueItem>>> loadingFutures = new HashSet<>();
    for (Entry<ByteBuffer, ? extends Collection<LoadQueueItem>> e : regionGroups.asMap().entrySet()) {
        byte[] first = e.getKey().array();
        Collection<LoadQueueItem> lqis = e.getValue();

        ClientServiceCallable<byte[]> serviceCallable = buildClientServiceCallable(conn, table.getName(), first,
                lqis, copyFile);

        Callable<List<LoadQueueItem>> call = new Callable<List<LoadQueueItem>>() {
            @Override
            public List<LoadQueueItem> call() throws Exception {
                List<LoadQueueItem> toRetry = tryAtomicRegionLoad(serviceCallable, table.getName(), first,
                        lqis);
                return toRetry;
            }
        };
        if (item2RegionMap != null) {
            for (LoadQueueItem lqi : lqis) {
                item2RegionMap.put(lqi, e.getKey());
            }
        }
        loadingFutures.add(pool.submit(call));
    }

    // get all the results.
    for (Future<List<LoadQueueItem>> future : loadingFutures) {
        try {
            List<LoadQueueItem> toRetry = future.get();

            if (item2RegionMap != null) {
                for (LoadQueueItem lqi : toRetry) {
                    item2RegionMap.remove(lqi);
                }
            }
            // LQIs that are requeued to be regrouped.
            queue.addAll(toRetry);

        } catch (ExecutionException e1) {
            Throwable t = e1.getCause();
            if (t instanceof IOException) {
                // At this point something unrecoverable has happened.
                // TODO Implement bulk load recovery
                throw new IOException("BulkLoad encountered an unrecoverable problem", t);
            }
            LOG.error("Unexpected execution exception during bulk load", e1);
            throw new IllegalStateException(t);
        } catch (InterruptedException e1) {
            LOG.error("Unexpected interrupted exception during bulk load", e1);
            throw (InterruptedIOException) new InterruptedIOException().initCause(e1);
        }
    }
}

---

From source file:org.apache.hadoop.hbase.tool.LoadIncrementalHFiles.java

/**
 * @param table the table to load into// w w w .java 2s .  c  om
 * @param pool the ExecutorService
 * @param queue the queue for LoadQueueItem
 * @param startEndKeys start and end keys
 * @return A map that groups LQI by likely bulk load region targets and Set of missing hfiles.
 */
private Pair<Multimap<ByteBuffer, LoadQueueItem>, Set<String>> groupOrSplitPhase(final Table table,
        ExecutorService pool, Deque<LoadQueueItem> queue, final Pair<byte[][], byte[][]> startEndKeys)
        throws IOException {
    // <region start key, LQI> need synchronized only within this scope of this
    // phase because of the puts that happen in futures.
    Multimap<ByteBuffer, LoadQueueItem> rgs = HashMultimap.create();
    final Multimap<ByteBuffer, LoadQueueItem> regionGroups = Multimaps.synchronizedMultimap(rgs);
    Set<String> missingHFiles = new HashSet<>();
    Pair<Multimap<ByteBuffer, LoadQueueItem>, Set<String>> pair = new Pair<>(regionGroups, missingHFiles);

    // drain LQIs and figure out bulk load groups
    Set<Future<Pair<List<LoadQueueItem>, String>>> splittingFutures = new HashSet<>();
    while (!queue.isEmpty()) {
        final LoadQueueItem item = queue.remove();

        final Callable<Pair<List<LoadQueueItem>, String>> call = new Callable<Pair<List<LoadQueueItem>, String>>() {
            @Override
            public Pair<List<LoadQueueItem>, String> call() throws Exception {
                Pair<List<LoadQueueItem>, String> splits = groupOrSplit(regionGroups, item, table,
                        startEndKeys);
                return splits;
            }
        };
        splittingFutures.add(pool.submit(call));
    }
    // get all the results. All grouping and splitting must finish before
    // we can attempt the atomic loads.
    for (Future<Pair<List<LoadQueueItem>, String>> lqis : splittingFutures) {
        try {
            Pair<List<LoadQueueItem>, String> splits = lqis.get();
            if (splits != null) {
                if (splits.getFirst() != null) {
                    queue.addAll(splits.getFirst());
                } else {
                    missingHFiles.add(splits.getSecond());
                }
            }
        } catch (ExecutionException e1) {
            Throwable t = e1.getCause();
            if (t instanceof IOException) {
                LOG.error("IOException during splitting", e1);
                throw (IOException) t; // would have been thrown if not parallelized,
            }
            LOG.error("Unexpected execution exception during splitting", e1);
            throw new IllegalStateException(t);
        } catch (InterruptedException e1) {
            LOG.error("Unexpected interrupted exception during splitting", e1);
            throw (InterruptedIOException) new InterruptedIOException().initCause(e1);
        }
    }
    return pair;
}

---

From source file:org.apache.hadoop.hive.ql.parse.GenTezUtils.java

public static void removeUnionOperators(Configuration conf, GenTezProcContext context, BaseWork work)
        throws SemanticException {

    List<Operator<?>> roots = new ArrayList<Operator<?>>();
    roots.addAll(work.getAllRootOperators());
    if (work.getDummyOps() != null) {
        roots.addAll(work.getDummyOps());
    }/* w  w  w  .ja v  a 2 s  .c  om*/
    roots.addAll(context.eventOperatorSet);

    // need to clone the plan.
    List<Operator<?>> newRoots = Utilities.cloneOperatorTree(conf, roots);

    // we're cloning the operator plan but we're retaining the original work. That means
    // that root operators have to be replaced with the cloned ops. The replacement map
    // tells you what that mapping is.
    BiMap<Operator<?>, Operator<?>> replacementMap = HashBiMap.create();

    // there's some special handling for dummyOps required. Mapjoins won't be properly
    // initialized if their dummy parents aren't initialized. Since we cloned the plan
    // we need to replace the dummy operators in the work with the cloned ones.
    List<HashTableDummyOperator> dummyOps = new LinkedList<HashTableDummyOperator>();

    Iterator<Operator<?>> it = newRoots.iterator();
    for (Operator<?> orig : roots) {
        Set<FileSinkOperator> fsOpSet = OperatorUtils.findOperators(orig, FileSinkOperator.class);
        for (FileSinkOperator fsOp : fsOpSet) {
            context.fileSinkSet.remove(fsOp);
        }

        Operator<?> newRoot = it.next();

        replacementMap.put(orig, newRoot);

        if (newRoot instanceof HashTableDummyOperator) {
            // dummy ops need to be updated to the cloned ones.
            dummyOps.add((HashTableDummyOperator) newRoot);
            it.remove();
        } else if (newRoot instanceof AppMasterEventOperator) {
            // event operators point to table scan operators. When cloning these we
            // need to restore the original scan.
            if (newRoot.getConf() instanceof DynamicPruningEventDesc) {
                TableScanOperator ts = ((DynamicPruningEventDesc) orig.getConf()).getTableScan();
                if (ts == null) {
                    throw new AssertionError("No table scan associated with dynamic event pruning. " + orig);
                }
                ((DynamicPruningEventDesc) newRoot.getConf()).setTableScan(ts);
            }
            it.remove();
        } else {
            if (newRoot instanceof TableScanOperator) {
                if (context.tsToEventMap.containsKey(orig)) {
                    // we need to update event operators with the cloned table scan
                    for (AppMasterEventOperator event : context.tsToEventMap.get(orig)) {
                        ((DynamicPruningEventDesc) event.getConf()).setTableScan((TableScanOperator) newRoot);
                    }
                }
            }
            context.rootToWorkMap.remove(orig);
            context.rootToWorkMap.put(newRoot, work);
        }
    }

    // now we remove all the unions. we throw away any branch that's not reachable from
    // the current set of roots. The reason is that those branches will be handled in
    // different tasks.
    Deque<Operator<?>> operators = new LinkedList<Operator<?>>();
    operators.addAll(newRoots);

    Set<Operator<?>> seen = new HashSet<Operator<?>>();

    while (!operators.isEmpty()) {
        Operator<?> current = operators.pop();
        seen.add(current);

        if (current instanceof FileSinkOperator) {
            FileSinkOperator fileSink = (FileSinkOperator) current;

            // remember it for additional processing later
            context.fileSinkSet.add(fileSink);

            FileSinkDesc desc = fileSink.getConf();
            Path path = desc.getDirName();
            List<FileSinkDesc> linked;

            if (!context.linkedFileSinks.containsKey(path)) {
                linked = new ArrayList<FileSinkDesc>();
                context.linkedFileSinks.put(path, linked);
            }
            linked = context.linkedFileSinks.get(path);
            linked.add(desc);

            desc.setDirName(new Path(path, "" + linked.size()));
            desc.setLinkedFileSink(true);
            desc.setParentDir(path);
            desc.setLinkedFileSinkDesc(linked);
        }

        if (current instanceof AppMasterEventOperator) {
            // remember for additional processing later
            context.eventOperatorSet.add((AppMasterEventOperator) current);

            // mark the original as abandoned. Don't need it anymore.
            context.abandonedEventOperatorSet
                    .add((AppMasterEventOperator) replacementMap.inverse().get(current));
        }

        if (current instanceof UnionOperator) {
            Operator<?> parent = null;
            int count = 0;

            for (Operator<?> op : current.getParentOperators()) {
                if (seen.contains(op)) {
                    ++count;
                    parent = op;
                }
            }

            // we should have been able to reach the union from only one side.
            assert count <= 1;

            if (parent == null) {
                // root operator is union (can happen in reducers)
                replacementMap.put(current, current.getChildOperators().get(0));
            } else {
                parent.removeChildAndAdoptItsChildren(current);
            }
        }

        if (current instanceof FileSinkOperator || current instanceof ReduceSinkOperator) {
            current.setChildOperators(null);
        } else {
            operators.addAll(current.getChildOperators());
        }
    }
    work.setDummyOps(dummyOps);
    work.replaceRoots(replacementMap);
}

---

From source file:org.apache.hadoop.hive.ql.parse.mr3.GenMR3Utils.java

public void removeUnionOperators(Configuration conf, GenMR3ProcContext context, BaseWork work)
        throws SemanticException {

    List<Operator<?>> roots = new ArrayList<Operator<?>>();
    roots.addAll(work.getAllRootOperators());
    if (work.getDummyOps() != null) {
        roots.addAll(work.getDummyOps());
    }/*from   w w w  .j  av  a2  s. c om*/
    roots.addAll(context.eventOperatorSet);

    // need to clone the plan.
    List<Operator<?>> newRoots = Utilities.cloneOperatorTree(conf, roots);

    // we're cloning the operator plan but we're retaining the original work. That means
    // that root operators have to be replaced with the cloned ops. The replacement map
    // tells you what that mapping is.
    BiMap<Operator<?>, Operator<?>> replacementMap = HashBiMap.create();

    // there's some special handling for dummyOps required. Mapjoins won't be properly
    // initialized if their dummy parents aren't initialized. Since we cloned the plan
    // we need to replace the dummy operators in the work with the cloned ones.
    List<HashTableDummyOperator> dummyOps = new LinkedList<HashTableDummyOperator>();

    Iterator<Operator<?>> it = newRoots.iterator();
    for (Operator<?> orig : roots) {
        Operator<?> newRoot = it.next();

        replacementMap.put(orig, newRoot);

        if (newRoot instanceof HashTableDummyOperator) {
            // dummy ops need to be updated to the cloned ones.
            dummyOps.add((HashTableDummyOperator) newRoot);
            it.remove();
        } else if (newRoot instanceof AppMasterEventOperator) {
            // event operators point to table scan operators. When cloning these we
            // need to restore the original scan.
            if (newRoot.getConf() instanceof DynamicPruningEventDesc) {
                TableScanOperator ts = ((DynamicPruningEventDesc) orig.getConf()).getTableScan();
                if (ts == null) {
                    throw new AssertionError("No table scan associated with dynamic event pruning. " + orig);
                }
                ((DynamicPruningEventDesc) newRoot.getConf()).setTableScan(ts);
            }
            it.remove();
        } else {
            if (newRoot instanceof TableScanOperator) {
                if (context.tsToEventMap.containsKey(orig)) {
                    // we need to update event operators with the cloned table scan
                    for (AppMasterEventOperator event : context.tsToEventMap.get(orig)) {
                        ((DynamicPruningEventDesc) event.getConf()).setTableScan((TableScanOperator) newRoot);
                    }
                }
            }
            context.rootToWorkMap.remove(orig);
            context.rootToWorkMap.put(newRoot, work);
        }
    }

    // now we remove all the unions. we throw away any branch that's not reachable from
    // the current set of roots. The reason is that those branches will be handled in
    // different tasks.
    Deque<Operator<?>> operators = new LinkedList<Operator<?>>();
    operators.addAll(newRoots);

    Set<Operator<?>> seen = new HashSet<Operator<?>>();

    while (!operators.isEmpty()) {
        Operator<?> current = operators.pop();
        seen.add(current);

        if (current instanceof FileSinkOperator) {
            FileSinkOperator fileSink = (FileSinkOperator) current;

            // remember it for additional processing later
            context.fileSinkSet.add(fileSink);

            FileSinkDesc desc = fileSink.getConf();
            Path path = desc.getDirName();
            List<FileSinkDesc> linked;

            if (!context.linkedFileSinks.containsKey(path)) {
                linked = new ArrayList<FileSinkDesc>();
                context.linkedFileSinks.put(path, linked);
            }
            linked = context.linkedFileSinks.get(path);
            linked.add(desc);

            desc.setDirName(new Path(path, "" + linked.size()));
            desc.setLinkedFileSinkDesc(linked);
        }

        if (current instanceof AppMasterEventOperator) {
            // remember for additional processing later
            context.eventOperatorSet.add((AppMasterEventOperator) current);

            // mark the original as abandoned. Don't need it anymore.
            context.abandonedEventOperatorSet
                    .add((AppMasterEventOperator) replacementMap.inverse().get(current));
        }

        if (current instanceof UnionOperator) {
            Operator<?> parent = null;
            int count = 0;

            for (Operator<?> op : current.getParentOperators()) {
                if (seen.contains(op)) {
                    ++count;
                    parent = op;
                }
            }

            // we should have been able to reach the union from only one side.
            assert count <= 1;

            if (parent == null) {
                // root operator is union (can happen in reducers)
                replacementMap.put(current, current.getChildOperators().get(0));
            } else {
                parent.removeChildAndAdoptItsChildren(current);
            }
        }

        if (current instanceof FileSinkOperator || current instanceof ReduceSinkOperator) {
            current.setChildOperators(null);
        } else {
            operators.addAll(current.getChildOperators());
        }
    }
    work.setDummyOps(dummyOps);
    work.replaceRoots(replacementMap);
}

---

From source file:org.apache.hadoop.hive.ql.parse.mr3.MR3Compiler.java

@Override
protected void optimizeOperatorPlan(ParseContext pCtx, Set<ReadEntity> inputs, Set<WriteEntity> outputs)
        throws SemanticException {
    PERF_LOGGER.PerfLogBegin(CLASS_NAME, PerfLogger.MR3_OPTIMIZE_OPERATOR_TREE);

    // Sequence of TableScan operators to be walked
    Deque<Operator<? extends OperatorDesc>> deque = new LinkedList<Operator<? extends OperatorDesc>>();
    deque.addAll(pCtx.getTopOps().values());

    Set<String> keys = pCtx.getTopOps().keySet();
    for (String key : keys) {
        console.printInfo("bubu key " + key);
        console.printInfo("bubu operator " + pCtx.getTopOps().get(key).toString());
    }/*from  w w w.j  a va  2  s .c om*/

    //    // Create the context for the walker
    //    OptimizeMR3ProcContext procCtx = new OptimizeMR3ProcContext(conf, pCtx, inputs, outputs, deque);
    //
    //    // create a walker which walks the tree in a DFS manner while maintaining
    //    // the operator stack.
    //    Map<Rule, NodeProcessor> opRules = new LinkedHashMap<Rule, NodeProcessor>();
    //    opRules.put(new RuleRegExp("Set parallelism - ReduceSink", ReduceSinkOperator.getOperatorName()
    //        + "%"), new SetSparkReducerParallelism());
    //
    //    opRules.put(new TypeRule(JoinOperator.class), new SparkJoinOptimizer(pCtx));
    //
    //    opRules.put(new TypeRule(MapJoinOperator.class), new SparkJoinHintOptimizer(pCtx));
    //
    //    // The dispatcher fires the processor corresponding to the closest matching
    //    // rule and passes the context along
    //    Dispatcher disp = new DefaultRuleDispatcher(null, opRules, procCtx);
    //    GraphWalker ogw = new DefaultGraphWalker(disp);
    //
    //    // Create a list of topop nodes
    //    ArrayList<Node> topNodes = new ArrayList<Node>();
    //    topNodes.addAll(pCtx.getTopOps().values());
    //    ogw.startWalking(topNodes, null);

    PERF_LOGGER.PerfLogEnd(CLASS_NAME, PerfLogger.MR3_OPTIMIZE_OPERATOR_TREE);
}

---

From source file:org.apache.hadoop.hive.ql.parse.spark.GenSparkUtils.java

public void removeUnionOperators(Configuration conf, GenSparkProcContext context, BaseWork work)
        throws SemanticException {

    List<Operator<?>> roots = new ArrayList<Operator<?>>();

    // For MapWork, getAllRootOperators is not suitable, since it checks
    // getPathToAliases, and will return null if this is empty. Here we are
    // replacing getAliasToWork, so should use that information instead.
    if (work instanceof MapWork) {
        roots.addAll(((MapWork) work).getAliasToWork().values());
    } else {/*  w w  w  .j  ava 2  s.  c om*/
        roots.addAll(work.getAllRootOperators());
    }
    if (work.getDummyOps() != null) {
        roots.addAll(work.getDummyOps());
    }

    // need to clone the plan.
    List<Operator<?>> newRoots = Utilities.cloneOperatorTree(conf, roots);

    // Build a map to map the original FileSinkOperator and the cloned FileSinkOperators
    // This map is used for set the stats flag for the cloned FileSinkOperators in later process
    Iterator<Operator<?>> newRootsIt = newRoots.iterator();
    for (Operator<?> root : roots) {
        Operator<?> newRoot = newRootsIt.next();
        List<Operator<?>> newOpQueue = new LinkedList<Operator<?>>();
        collectOperators(newRoot, newOpQueue);
        List<Operator<?>> opQueue = new LinkedList<Operator<?>>();
        collectOperators(root, opQueue);
        Iterator<Operator<?>> newOpQueueIt = newOpQueue.iterator();
        for (Operator<?> op : opQueue) {
            Operator<?> newOp = newOpQueueIt.next();

            // We need to update rootToWorkMap in case the op is a key, since even
            // though we clone the op tree, we're still using the same MapWork/ReduceWork.
            if (context.rootToWorkMap.containsKey(op)) {
                context.rootToWorkMap.put(newOp, context.rootToWorkMap.get(op));
            }
            // Don't remove the old entry - in SparkPartitionPruningSink it still
            // refers to the old TS, and we need to lookup it later in
            // processPartitionPruningSink.

            if (op instanceof FileSinkOperator) {
                List<FileSinkOperator> fileSinkList = context.fileSinkMap.get(op);
                if (fileSinkList == null) {
                    fileSinkList = new LinkedList<FileSinkOperator>();
                }
                fileSinkList.add((FileSinkOperator) newOp);
                context.fileSinkMap.put((FileSinkOperator) op, fileSinkList);
            } else if (op instanceof SparkPartitionPruningSinkOperator) {
                SparkPartitionPruningSinkOperator oldPruningSink = (SparkPartitionPruningSinkOperator) op;
                SparkPartitionPruningSinkOperator newPruningSink = (SparkPartitionPruningSinkOperator) newOp;
                newPruningSink.getConf().setTableScan(oldPruningSink.getConf().getTableScan());
                context.pruningSinkSet.add(newPruningSink);
                context.pruningSinkSet.remove(oldPruningSink);
            }
        }
    }

    // we're cloning the operator plan but we're retaining the original work. That means
    // that root operators have to be replaced with the cloned ops. The replacement map
    // tells you what that mapping is.
    Map<Operator<?>, Operator<?>> replacementMap = new HashMap<Operator<?>, Operator<?>>();

    // there's some special handling for dummyOps required. Mapjoins won't be properly
    // initialized if their dummy parents aren't initialized. Since we cloned the plan
    // we need to replace the dummy operators in the work with the cloned ones.
    List<HashTableDummyOperator> dummyOps = new LinkedList<HashTableDummyOperator>();

    Iterator<Operator<?>> it = newRoots.iterator();
    for (Operator<?> orig : roots) {
        Operator<?> newRoot = it.next();
        if (newRoot instanceof HashTableDummyOperator) {
            dummyOps.add((HashTableDummyOperator) newRoot);
            it.remove();
        } else {
            replacementMap.put(orig, newRoot);
        }
    }

    // now we remove all the unions. we throw away any branch that's not reachable from
    // the current set of roots. The reason is that those branches will be handled in
    // different tasks.
    Deque<Operator<?>> operators = new LinkedList<Operator<?>>();
    operators.addAll(newRoots);

    Set<Operator<?>> seen = new HashSet<Operator<?>>();

    while (!operators.isEmpty()) {
        Operator<?> current = operators.pop();
        seen.add(current);

        if (current instanceof UnionOperator) {
            Operator<?> parent = null;
            int count = 0;

            for (Operator<?> op : current.getParentOperators()) {
                if (seen.contains(op)) {
                    ++count;
                    parent = op;
                }
            }

            // we should have been able to reach the union from only one side.
            Preconditions.checkArgument(count <= 1,
                    "AssertionError: expected count to be <= 1, but was " + count);

            if (parent == null) {
                // root operator is union (can happen in reducers)
                replacementMap.put(current, current.getChildOperators().get(0));
            } else {
                parent.removeChildAndAdoptItsChildren(current);
            }
        }

        if (current instanceof FileSinkOperator || current instanceof ReduceSinkOperator) {
            current.setChildOperators(null);
        } else {
            operators.addAll(current.getChildOperators());
        }
    }
    work.setDummyOps(dummyOps);
    work.replaceRoots(replacementMap);
}

---

From source file:org.apache.hadoop.hive.ql.parse.TezCompiler.java

private Set<Set<Operator<?>>> getComponents(OptimizeTezProcContext procCtx) {
    Deque<Operator<?>> deque = new LinkedList<Operator<?>>();
    deque.addAll(procCtx.parseContext.getTopOps().values());

    AtomicInteger index = new AtomicInteger();
    Map<Operator<?>, Integer> indexes = new HashMap<Operator<?>, Integer>();
    Map<Operator<?>, Integer> lowLinks = new HashMap<Operator<?>, Integer>();
    Stack<Operator<?>> nodes = new Stack<Operator<?>>();
    Set<Set<Operator<?>>> components = new HashSet<Set<Operator<?>>>();

    for (Operator<?> o : deque) {
        if (!indexes.containsKey(o)) {
            connect(o, index, nodes, indexes, lowLinks, components);
        }/*from www .j a  va  2  s . c  o  m*/
    }

    return components;
}

---

From source file:org.apache.hadoop.hive.ql.parse.TezCompiler.java

private void runStatsDependentOptimizations(OptimizeTezProcContext procCtx, Set<ReadEntity> inputs,
        Set<WriteEntity> outputs) throws SemanticException {

    // Sequence of TableScan operators to be walked
    Deque<Operator<?>> deque = new LinkedList<Operator<?>>();
    deque.addAll(procCtx.parseContext.getTopOps().values());

    // create a walker which walks the tree in a DFS manner while maintaining
    // the operator stack.
    Map<Rule, NodeProcessor> opRules = new LinkedHashMap<Rule, NodeProcessor>();
    opRules.put(new RuleRegExp("Set parallelism - ReduceSink", ReduceSinkOperator.getOperatorName() + "%"),
            new SetReducerParallelism());

    opRules.put(new RuleRegExp("Convert Join to Map-join", JoinOperator.getOperatorName() + "%"),
            new ConvertJoinMapJoin());

    opRules.put(/*www .j  av  a2  s  . c  o  m*/
            new RuleRegExp("Remove dynamic pruning by size", AppMasterEventOperator.getOperatorName() + "%"),
            new RemoveDynamicPruningBySize());

    // The dispatcher fires the processor corresponding to the closest matching
    // rule and passes the context along
    Dispatcher disp = new DefaultRuleDispatcher(null, opRules, procCtx);
    List<Node> topNodes = new ArrayList<Node>();
    topNodes.addAll(procCtx.parseContext.getTopOps().values());
    GraphWalker ogw = new ForwardWalker(disp);
    ogw.startWalking(topNodes, null);
}

---

From source file:org.apache.hadoop.hive.ql.parse.TezCompiler.java

private void runDynamicPartitionPruning(OptimizeTezProcContext procCtx, Set<ReadEntity> inputs,
        Set<WriteEntity> outputs) throws SemanticException {

    if (!procCtx.conf.getBoolVar(ConfVars.TEZ_DYNAMIC_PARTITION_PRUNING)) {
        return;/*w  w  w . j  a va2 s . c o  m*/
    }

    // Sequence of TableScan operators to be walked
    Deque<Operator<?>> deque = new LinkedList<Operator<?>>();
    deque.addAll(procCtx.parseContext.getTopOps().values());

    Map<Rule, NodeProcessor> opRules = new LinkedHashMap<Rule, NodeProcessor>();
    opRules.put(new RuleRegExp(new String("Dynamic Partition Pruning"), FilterOperator.getOperatorName() + "%"),
            new DynamicPartitionPruningOptimization());

    // The dispatcher fires the processor corresponding to the closest matching
    // rule and passes the context along
    Dispatcher disp = new DefaultRuleDispatcher(null, opRules, procCtx);
    List<Node> topNodes = new ArrayList<Node>();
    topNodes.addAll(procCtx.parseContext.getTopOps().values());
    GraphWalker ogw = new ForwardWalker(disp);
    ogw.startWalking(topNodes, null);

    // need a new run of the constant folding because we might have created lots
    // of "and true and true" conditions.
    // Rather than run the full constant folding just need to shortcut AND/OR expressions
    // involving constant true/false values.
    if (procCtx.conf.getBoolVar(ConfVars.HIVEOPTCONSTANTPROPAGATION)) {
        new ConstantPropagate(ConstantPropagateOption.SHORTCUT).transform(procCtx.parseContext);
    }
}

---

• «
• 1
• 2
• 3
• 4
• »