Java tutorial
/* * @(#)$Id$ * * Copyright 2006-2008 Makoto YUI * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. * * Contributors: * Makoto YUI - initial implementation */ package xbird.xquery.ext.grid; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.IdentityHashMap; import java.util.List; import java.util.Map; import java.util.Set; import javax.annotation.Nullable; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import gridool.GridConfiguration; import gridool.GridException; import gridool.GridNode; import gridool.GridTask; import gridool.GridTaskResult; import gridool.GridTaskResultPolicy; import gridool.annotation.DirectoryResource; import gridool.annotation.GridConfigResource; import gridool.construct.GridJobBase; import gridool.directory.LocalDirectory; import gridool.routing.GridNodeSelector; import gridool.routing.GridTaskRouter; import xbird.storage.DbException; import xbird.util.collections.CollectionUtils; import xbird.util.math.MathUtils; import xbird.xquery.dm.value.Item; import xbird.xquery.dm.value.Sequence; import xbird.xquery.dm.value.sequence.MarshalledSequence; import xbird.xquery.dm.value.sequence.PipelinedSequence; import xbird.xquery.dm.value.sequence.ValueSequence; import xbird.xquery.expr.XQExpression; import xbird.xquery.expr.ext.MapExpr; import xbird.xquery.expr.var.BindingVariable; import xbird.xquery.meta.DynamicContext; /** * * <DIV lang="en"></DIV> * <DIV lang="ja"></DIV> * * @author Makoto YUI (yuin405+xbird@gmail.com) */ public final class MapQueryJob extends GridJobBase<MapExpr, Sequence<? extends Item>> { private static final long serialVersionUID = -2810696094868588963L; private static final Log LOG = LogFactory.getLog(MapQueryJob.class); @Nullable private transient List<Sequence<? extends Item>> _results; // ====================================== // injected resources @GridConfigResource private transient GridConfiguration config; @DirectoryResource private transient LocalDirectory directory; // ====================================== public MapQueryJob() { super(); this._results = Collections.emptyList(); } @Override public boolean injectResources() { return true; } private void checkInjectedResouces() { if (config == null) { throw new IllegalStateException("GridConfiguration is not injected"); } if (directory == null) { throw new IllegalStateException("LocalDirectory is not injected"); } } @Override public Map<GridTask, GridNode> map(GridTaskRouter router, MapExpr mapExpr) throws GridException { checkInjectedResouces(); final String colPath = mapExpr.getCollectionPath(); final GridNode[] liveNodes = router.getAllNodes(); final int numLiveNodes = liveNodes.length; if (numLiveNodes == 0) {// no live node return Collections.emptyMap(); } final Set<GridNode> mapableNodes = CollectionUtils.asSet(liveNodes); final Map<String, List<GridNode>> mapping; try { mapping = directory.prefixSearch(colPath, mapableNodes); } catch (DbException e) { throw new GridException("prefixSearch failed: " + colPath, e); } if (mapping.isEmpty()) {// no mapping found return Collections.emptyMap(); } final BindingVariable bindingVar = mapExpr.getBindingVariable(); final XQExpression bodyExpr = mapExpr.getBodyExpression(); final GridNodeSelector selector = config.getNodeSelector(); assert (selector != null); final Map<GridNode, List<String>> nodeKeysMap = new HashMap<GridNode, List<String>>(numLiveNodes); final Map<GridTask, GridNode> map = new IdentityHashMap<GridTask, GridNode>(numLiveNodes); for (Map.Entry<String, List<GridNode>> entry : mapping.entrySet()) { final String key = entry.getKey(); final List<GridNode> candidateNodes = entry.getValue(); assert (!candidateNodes.isEmpty()); final GridNode node = selector.selectNode(candidateNodes, null, config); candidateNodes.clear(); // let GC do its work List<String> mappedKeys = nodeKeysMap.get(node); if (mappedKeys == null) { mappedKeys = new ArrayList<String>(128); nodeKeysMap.put(node, mappedKeys); // TODO REVIEWME DispatchQueryExecTask task = new DispatchQueryExecTask(this, bindingVar, bodyExpr, mappedKeys, false); map.put(task, node); } mappedKeys.add(key); } if (LOG.isInfoEnabled()) { logJobInformation(map, getJobId()); } this._results = new ArrayList<Sequence<? extends Item>>(map.size()); return map; } @Override public GridTaskResultPolicy result(GridTask task, GridTaskResult result) throws GridException { final Sequence<? extends Item> resultSeq = result.getResult(); if (resultSeq == null) { final Exception error = result.getException(); if (error != null) { LOG.error(error.getMessage()); } return GridTaskResultPolicy.FAILOVER; } else { _results.add(resultSeq); return GridTaskResultPolicy.CONTINUE; } } @SuppressWarnings("unchecked") @Override public Sequence<? extends Item> reduce() throws GridException { final int size = _results.size(); if (size == 0) { return ValueSequence.emptySequence(); } else if (size == 1) { Sequence<? extends Item> res = _results.get(0); return wrapResult(res); } else { PipelinedSequence piped = new PipelinedSequence(DynamicContext.DUMMY, _results); return wrapResult(piped); } } private static Sequence<? extends Item> wrapResult(final Sequence<? extends Item> seq) { final MarshalledSequence res = new MarshalledSequence(seq, DynamicContext.DUMMY); res.setRedirectable(true); return res; } private static void logJobInformation(final Map<GridTask, GridNode> map, final String jobId) { LOG.info("Mapped job [" + jobId + "] to " + map.size() + " nodes"); if (LOG.isDebugEnabled()) { final int[] numJobs = new int[map.size()]; int i = 0; for (Map.Entry<GridTask, GridNode> e : map.entrySet()) { DispatchQueryExecTask task = (DispatchQueryExecTask) e.getKey(); numJobs[i++] = task.getRelativePaths().size(); } float stddev = MathUtils.stddev(numJobs); LOG.debug("Standard deviation of the mapped task size for " + jobId + ": " + stddev); } } }