List of usage examples for org.apache.mahout.math Vector size
int size();
From source file:org.qcri.pca.CompositeJobTest.java
License:Apache License
private void verifyYtX(DummyRecordWriter<IntWritable, VectorWritable> writer) { Assert.assertEquals("The reducer should output " + cols + " keys!", cols, writer.getKeys().size()); for (IntWritable key : writer.getKeys()) { List<VectorWritable> list = writer.getValue(key); assertEquals("reducer produces more than one values per key!", 1, list.size()); Vector v = list.get(0).get(); assertEquals("reducer vector size must match the x size!", xsize, v.size()); for (int c = 0; c < xsize; c++) Assert.assertEquals("The ytx[" + key.get() + "][" + c + "] is incorrect: ", ytx[key.get()][c], v.get(c), EPSILON);/* ww w . j a va 2 s . com*/ } }
From source file:org.qcri.pca.MahoutCompatibilityTest.java
License:Apache License
@Test public void testMAHOUT_1238() throws IOException { Vector v = new SequentialAccessSparseVector(5); v.set(1, 3.0);// ww w .ja v a 2 s .c o m v.set(3, 5.0); Vector view = new VectorView(v, 0, v.size()); doTestVectorWritableEquals(view); }
From source file:org.qcri.pca.MeanAndSpanJob.java
/** * This method overrides the Vector.assign method to allow optimization for * ZeroIndifferent functions/* www . j av a 2s . c o m*/ * * @param vector * the vector to be updated * @param other * the other vector * @param function * the function that operates on elements of the two vectors * @return the modified vector */ static public Vector vectorAssign(Vector vector, Vector other, ZeroIndifferentFunc function) { if (vector.size() != other.size()) { throw new CardinalityException(vector.size(), other.size()); } // special case: iterate only over the non-zero elements of the vector to // add Iterator<Element> it = other.nonZeroes().iterator(); Element e; while (it.hasNext() && (e = it.next()) != null) { double val = vector.getQuick(e.index()); double newVal = function.apply(val, e.get()); vector.setQuick(e.index(), newVal); } return vector; }
From source file:org.qcri.pca.MeanAndSpanJobTest.java
License:Apache License
private void verifyMapperOutput(DummyRecordWriter<IntWritable, VectorWritable> writer) { Assert.assertEquals("Each mapper should output three keys!", 3, writer.getKeys().size()); for (IntWritable key : writer.getKeys()) { List<VectorWritable> list = writer.getValue(key); assertEquals("Mapper did not combine the results!", 1, list.size()); Vector v = list.get(0).get(); switch (key.get()) { case MeanAndSpanJob.MEANVECTOR: Assert.assertEquals("MeanVector size does not match!", v.size(), cols + 1); Assert.assertEquals("MeanVector count does not match!", rows, v.get(0), EPSILON); verifySum(inputVectors, v.viewPart(1, cols)); break; case MeanAndSpanJob.MINVECTOR: Assert.assertEquals("MinVector size does not match!", v.size(), cols); verifyMin(inputVectors, v);/*from w w w . ja va2s. c o m*/ break; case MeanAndSpanJob.MAXVECTOR: Assert.assertEquals("MaxVector size does not match!", v.size(), cols); verifyMax(inputVectors, v); break; default: Assert.fail("Unknown key from mapper"); } } }
From source file:org.qcri.pca.MeanAndSpanJobTest.java
License:Apache License
private void verifyReducerOutput(DummyRecordWriter<IntWritable, VectorWritable> writer) { Assert.assertEquals("The reducer should output two keys!", 2, writer.getKeys().size()); for (IntWritable key : writer.getKeys()) { List<VectorWritable> list = writer.getValue(key); assertEquals("Reducer did not combine the results!", 1, list.size()); Vector v = list.get(0).get(); switch (key.get()) { case MeanAndSpanJob.MEANVECTOR: Assert.assertEquals("MeanVector size does not match!", v.size(), cols); verifyMean(inputVectors, v); break; case MeanAndSpanJob.SPANVECTOR: Assert.assertEquals("SpanVector size does not match!", v.size(), cols); verifySpan(inputVectors, v); break; default://from ww w .j a va 2 s . com Assert.fail("Unknown key from mapper"); } } }
From source file:org.qcri.sparkpca.PCAUtils.java
/** * Subtract two dense vectors/*from w w w . j a v a 2 s. c o m*/ * @param vector1 * @param vector2 * @param resArray: result array * @return */ static double[] denseVectorMinusVector(org.apache.spark.mllib.linalg.Vector vector1, Vector vector2, double[] resArray) { for (int i = 0; i < vector2.size(); i++) { resArray[i] = vector1.apply(i) - vector2.getQuick(i); } return resArray; }
From source file:org.qcri.sparkpca.PCAUtils.java
/*** * Mi = (Yi-Ym)' x (Xi-Xm) = Yi' x (Xi-Xm) - Ym' x (Xi-Xm) * //from w w w. j av a 2 s .c o m * M = Sum(Mi) = Sum(Yi' x (Xi-Xm)) - Ym' x (Sum(Xi)-N*Xm) * * The second part is done in this function */ public static Matrix updateXtXAndYtx(Matrix realCentralYtx, Vector realCentralSumX, Vector ym, Vector xm, int nRows) { for (int yRow = 0; yRow < ym.size(); yRow++) { double scale = ym.getQuick(yRow); for (int xCol = 0; xCol < realCentralSumX.size(); xCol++) { double centeredValue = realCentralSumX.getQuick(xCol) - nRows * xm.getQuick(xCol); double currValue = realCentralYtx.getQuick(yRow, xCol); currValue -= centeredValue * scale; realCentralYtx.setQuick(yRow, xCol, currValue); } } return realCentralYtx; }
From source file:org.trustedanalytics.atk.giraph.aggregators.VectorSumAggregator.java
License:Apache License
@Override public void aggregate(VectorWritable value) { Vector currentValue = getAggregatedValue().get(); if (currentValue.size() == 0) { getAggregatedValue().set(value.get()); } else if (value.get().size() > 0) { getAggregatedValue().set(currentValue.plus(value.get())); }/*from ww w. ja v a 2 s . co m*/ }
From source file:org.trustedanalytics.atk.giraph.algorithms.lbp.LoopyBeliefPropagationComputation.java
License:Apache License
/** * Initialize vertex/* w ww . j a v a 2 s. c o m*/ * * @param vertex of the graph */ private void initializeVertex(Vertex<LongWritable, VertexData4LBPWritable, DoubleWritable> vertex) { // normalize prior and posterior Vector prior = vertex.getValue().getPriorVector(); Vector posterior = vertex.getValue().getPosteriorVector(); int nStates = prior.size(); double sum = 0d; for (int i = 0; i < nStates; i++) { double v = prior.getQuick(i); if (v < 0d) { throw new IllegalArgumentException("Vertex ID: " + vertex.getId() + " has negative prior value."); } else if (v < MIN_PRIOR_VALUE) { v = MIN_PRIOR_VALUE; prior.setQuick(i, v); } sum += v; } for (int i = 0; i < nStates; i++) { posterior.setQuick(i, prior.getQuick(i) / sum); prior.setQuick(i, Math.log(posterior.getQuick(i))); } // collect graph statistics VertexType vt = vertex.getValue().getType(); vt = ignoreVertexType ? VertexType.TRAIN : vt; switch (vt) { case TRAIN: aggregate(SUM_TRAIN_VERTICES, new LongWritable(1)); break; case VALIDATE: aggregate(SUM_VALIDATE_VERTICES, new LongWritable(1)); break; case TEST: aggregate(SUM_TEST_VERTICES, new LongWritable(1)); break; default: throw new IllegalArgumentException("Unknown vertex type: " + vt.toString()); } // if it's not a training vertex, use uniform posterior and don't send out messages if (vt != VertexType.TRAIN) { posterior.assign(1.0 / nStates); return; } // calculate messages IdWithVectorMessage newMessage = new IdWithVectorMessage(); newMessage.setData(vertex.getId().get()); // calculate initial belief Vector belief = prior.clone(); for (Edge<LongWritable, DoubleWritable> edge : vertex.getEdges()) { double weight = edge.getValue().get(); if (weight <= 0d) { throw new IllegalArgumentException("Vertex ID: " + vertex.getId() + " has an edge with negative or zero weight value " + weight); } for (int i = 0; i < nStates; i++) { sum = 0d; for (int j = 0; j < nStates; j++) { double msg = Math.exp( prior.getQuick(j) + edgePotential(Math.abs(i - j) / (double) (nStates - 1), weight)); if (maxProduct) { sum = sum > msg ? sum : msg; } else { sum += msg; } } belief.setQuick(i, sum > 0d ? Math.log(sum) : Double.MIN_VALUE); } belief = belief.plus(-belief.maxValue()); // send out messages newMessage.setVector(belief); sendMessage(edge.getTargetVertexId(), newMessage); } }
From source file:org.trustedanalytics.atk.giraph.algorithms.lbp.LoopyBeliefPropagationComputation.java
License:Apache License
@Override public void compute(Vertex<LongWritable, VertexData4LBPWritable, DoubleWritable> vertex, Iterable<IdWithVectorMessage> messages) throws IOException { long step = getSuperstep(); if (step == 0) { initializeVertex(vertex);/*from w w w .j a v a 2s .co m*/ return; } // collect messages sent to this vertex HashMap<Long, Vector> map = new HashMap<Long, Vector>(); for (IdWithVectorMessage message : messages) { map.put(message.getData(), message.getVector()); } // update posterior according to prior and messages VertexData4LBPWritable vertexValue = vertex.getValue(); VertexType vt = vertexValue.getType(); vt = ignoreVertexType ? VertexType.TRAIN : vt; Vector prior = vertexValue.getPriorVector(); double nStates = prior.size(); if (vt != VertexType.TRAIN) { // assign a uniform prior for validate/test vertex prior = prior.clone().assign(Math.log(1.0 / nStates)); } // sum of prior and messages Vector sumPosterior = prior; for (IdWithVectorMessage message : messages) { sumPosterior = sumPosterior.plus(message.getVector()); } sumPosterior = sumPosterior.plus(-sumPosterior.maxValue()); // update posterior if this isn't an anchor vertex if (prior.maxValue() < anchorThreshold) { // normalize posterior Vector posterior = sumPosterior.clone().assign(Functions.EXP); posterior = posterior.normalize(1d); Vector oldPosterior = vertexValue.getPosteriorVector(); double delta = posterior.minus(oldPosterior).norm(1d); // aggregate deltas switch (vt) { case TRAIN: aggregate(SUM_TRAIN_DELTA, new DoubleWritable(delta)); break; case VALIDATE: aggregate(SUM_VALIDATE_DELTA, new DoubleWritable(delta)); break; case TEST: aggregate(SUM_TEST_DELTA, new DoubleWritable(delta)); break; default: throw new IllegalArgumentException("Unknown vertex type: " + vt.toString()); } // update posterior vertexValue.setPosteriorVector(posterior); } if (step < maxSupersteps) { // if it's not a training vertex, don't send out messages if (vt != VertexType.TRAIN) { return; } IdWithVectorMessage newMessage = new IdWithVectorMessage(); newMessage.setData(vertex.getId().get()); // update belief Vector belief = prior.clone(); for (Edge<LongWritable, DoubleWritable> edge : vertex.getEdges()) { double weight = edge.getValue().get(); long id = edge.getTargetVertexId().get(); Vector tempVector = sumPosterior; if (map.containsKey(id)) { tempVector = sumPosterior.minus(map.get(id)); } for (int i = 0; i < nStates; i++) { double sum = 0d; for (int j = 0; j < nStates; j++) { double msg = Math.exp( tempVector.getQuick(j) + edgePotential(Math.abs(i - j) / (nStates - 1), weight)); if (maxProduct) { sum = sum > msg ? sum : msg; } else { sum += msg; } } belief.setQuick(i, sum > 0d ? Math.log(sum) : Double.MIN_VALUE); } belief = belief.plus(-belief.maxValue()); newMessage.setVector(belief); sendMessage(edge.getTargetVertexId(), newMessage); } } else { // convert prior back to regular scale before output prior = vertexValue.getPriorVector(); prior = prior.assign(Functions.EXP); vertexValue.setPriorVector(prior); vertex.voteToHalt(); } }