Java tutorial
/** * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.mahout.math.hadoop.stochasticsvd; import java.io.Closeable; import java.io.IOException; import java.text.NumberFormat; import java.util.ArrayDeque; import java.util.Arrays; import java.util.Deque; import java.util.Iterator; import java.util.regex.Matcher; import com.google.common.collect.Lists; import org.apache.commons.lang3.Validate; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.filecache.DistributedCache; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.SequenceFile; import org.apache.hadoop.io.SequenceFile.CompressionType; import org.apache.hadoop.io.Writable; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.OutputCollector; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.Mapper; import org.apache.hadoop.mapreduce.Reducer; import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat; import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat; import org.apache.mahout.common.HadoopUtil; import org.apache.mahout.common.IOUtils; import org.apache.mahout.common.Pair; import org.apache.mahout.common.iterator.sequencefile.PathType; import org.apache.mahout.common.iterator.sequencefile.SequenceFileDirIterator; import org.apache.mahout.math.DenseVector; import org.apache.mahout.math.SequentialAccessSparseVector; import org.apache.mahout.math.Vector; import org.apache.mahout.math.VectorWritable; import org.apache.mahout.math.function.Functions; import org.apache.mahout.math.hadoop.stochasticsvd.qr.QRFirstStep; /** * Computes ABt products, then first step of QR which is pushed down to the * reducer. */ @SuppressWarnings("deprecation") public final class ABtDenseOutJob { public static final String PROP_BT_PATH = "ssvd.Bt.path"; public static final String PROP_BT_BROADCAST = "ssvd.Bt.broadcast"; public static final String PROP_SB_PATH = "ssvdpca.sb.path"; public static final String PROP_SQ_PATH = "ssvdpca.sq.path"; public static final String PROP_XI_PATH = "ssvdpca.xi.path"; private ABtDenseOutJob() { } /** * So, here, i preload A block into memory. * <P> * * A sparse matrix seems to be ideal for that but there are two reasons why i * am not using it: * <UL> * <LI>1) I don't know the full block height. so i may need to reallocate it * from time to time. Although this probably not a showstopper. * <LI>2) I found that RandomAccessSparseVectors seem to take much more memory * than the SequentialAccessSparseVectors. * </UL> * <P> * */ public static class ABtMapper extends Mapper<Writable, VectorWritable, SplitPartitionedWritable, DenseBlockWritable> { private SplitPartitionedWritable outKey; private final Deque<Closeable> closeables = new ArrayDeque<Closeable>(); private SequenceFileDirIterator<IntWritable, VectorWritable> btInput; private Vector[] aCols; private double[][] yiCols; private int aRowCount; private int kp; private int blockHeight; private boolean distributedBt; private Path[] btLocalPath; private Configuration localFsConfig; /* * xi and s_q are PCA-related corrections, per MAHOUT-817 */ protected Vector xi; protected Vector sq; @Override protected void map(Writable key, VectorWritable value, Context context) throws IOException, InterruptedException { Vector vec = value.get(); int vecSize = vec.size(); if (aCols == null) { aCols = new Vector[vecSize]; } else if (aCols.length < vecSize) { aCols = Arrays.copyOf(aCols, vecSize); } if (vec.isDense()) { for (int i = 0; i < vecSize; i++) { extendAColIfNeeded(i, aRowCount + 1); aCols[i].setQuick(aRowCount, vec.getQuick(i)); } } else if (vec.size() > 0) { for (Vector.Element vecEl : vec.nonZeroes()) { int i = vecEl.index(); extendAColIfNeeded(i, aRowCount + 1); aCols[i].setQuick(aRowCount, vecEl.get()); } } aRowCount++; } private void extendAColIfNeeded(int col, int rowCount) { if (aCols[col] == null) { aCols[col] = new SequentialAccessSparseVector(rowCount < blockHeight ? blockHeight : rowCount, 1); } else if (aCols[col].size() < rowCount) { Vector newVec = new SequentialAccessSparseVector(rowCount + blockHeight, aCols[col].getNumNondefaultElements() << 1); newVec.viewPart(0, aCols[col].size()).assign(aCols[col]); aCols[col] = newVec; } } @Override protected void cleanup(Context context) throws IOException, InterruptedException { try { yiCols = new double[kp][]; for (int i = 0; i < kp; i++) { yiCols[i] = new double[Math.min(aRowCount, blockHeight)]; } int numPasses = (aRowCount - 1) / blockHeight + 1; String propBtPathStr = context.getConfiguration().get(PROP_BT_PATH); Validate.notNull(propBtPathStr, "Bt input is not set"); Path btPath = new Path(propBtPathStr); DenseBlockWritable dbw = new DenseBlockWritable(); /* * so it turns out that it may be much more efficient to do a few * independent passes over Bt accumulating the entire block in memory * than pass huge amount of blocks out to combiner. so we aim of course * to fit entire s x (k+p) dense block in memory where s is the number * of A rows in this split. If A is much sparser than (k+p) avg # of * elements per row then the block may exceed the split size. if this * happens, and if the given blockHeight is not high enough to * accomodate this (because of memory constraints), then we start * splitting s into several passes. since computation is cpu-bound * anyway, it should be o.k. for supersparse inputs. (as ok it can be * that projection is thicker than the original anyway, why would one * use that many k+p then). */ int lastRowIndex = -1; for (int pass = 0; pass < numPasses; pass++) { if (distributedBt) { btInput = new SequenceFileDirIterator<IntWritable, VectorWritable>(btLocalPath, true, localFsConfig); } else { btInput = new SequenceFileDirIterator<IntWritable, VectorWritable>(btPath, PathType.GLOB, null, null, true, context.getConfiguration()); } closeables.addFirst(btInput); Validate.isTrue(btInput.hasNext(), "Empty B' input!"); int aRowBegin = pass * blockHeight; int bh = Math.min(blockHeight, aRowCount - aRowBegin); /* * check if we need to trim block allocation */ if (pass > 0) { if (bh == blockHeight) { for (int i = 0; i < kp; i++) { Arrays.fill(yiCols[i], 0.0); } } else { for (int i = 0; i < kp; i++) { yiCols[i] = null; } for (int i = 0; i < kp; i++) { yiCols[i] = new double[bh]; } } } while (btInput.hasNext()) { Pair<IntWritable, VectorWritable> btRec = btInput.next(); int btIndex = btRec.getFirst().get(); Vector btVec = btRec.getSecond().get(); Vector aCol; if (btIndex > aCols.length || (aCol = aCols[btIndex]) == null || aCol.size() == 0) { /* 100% zero A column in the block, skip it as sparse */ continue; } int j = -1; for (Vector.Element aEl : aCol.nonZeroes()) { j = aEl.index(); /* * now we compute only swathes between aRowBegin..aRowBegin+bh * exclusive. it seems like a deficiency but in fact i think it * will balance itself out: either A is dense and then we * shouldn't have more than one pass and therefore filter * conditions will never kick in. Or, the only situation where we * can't fit Y_i block in memory is when A input is much sparser * than k+p per row. But if this is the case, then we'd be looking * at very few elements without engaging them in any operations so * even then it should be ok. */ if (j < aRowBegin) { continue; } if (j >= aRowBegin + bh) { break; } /* * assume btVec is dense */ if (xi != null) { /* * MAHOUT-817: PCA correction for B'. I rewrite the whole * computation loop so i don't have to check if PCA correction * is needed at individual element level. It looks bulkier this * way but perhaps less wasteful on cpu. */ for (int s = 0; s < kp; s++) { // code defensively against shortened xi double xii = xi.size() > btIndex ? xi.get(btIndex) : 0.0; yiCols[s][j - aRowBegin] += aEl.get() * (btVec.getQuick(s) - xii * sq.get(s)); } } else { /* * no PCA correction */ for (int s = 0; s < kp; s++) { yiCols[s][j - aRowBegin] += aEl.get() * btVec.getQuick(s); } } } if (lastRowIndex < j) { lastRowIndex = j; } } /* * so now we have stuff in yi */ dbw.setBlock(yiCols); outKey.setTaskItemOrdinal(pass); context.write(outKey, dbw); closeables.remove(btInput); btInput.close(); } } finally { IOUtils.close(closeables); } } @Override protected void setup(Context context) throws IOException, InterruptedException { Configuration conf = context.getConfiguration(); int k = Integer.parseInt(conf.get(QRFirstStep.PROP_K)); int p = Integer.parseInt(conf.get(QRFirstStep.PROP_P)); kp = k + p; outKey = new SplitPartitionedWritable(context); blockHeight = conf.getInt(BtJob.PROP_OUTER_PROD_BLOCK_HEIGHT, -1); distributedBt = conf.get(PROP_BT_BROADCAST) != null; if (distributedBt) { btLocalPath = HadoopUtil.getCachedFiles(conf); localFsConfig = new Configuration(); localFsConfig.set("fs.default.name", "file:///"); } /* * PCA -related corrections (MAHOUT-817) */ String xiPathStr = conf.get(PROP_XI_PATH); if (xiPathStr != null) { xi = SSVDHelper.loadAndSumUpVectors(new Path(xiPathStr), conf); sq = SSVDHelper.loadAndSumUpVectors(new Path(conf.get(PROP_SQ_PATH)), conf); } } } /** * QR first step pushed down to reducer. * */ public static class QRReducer extends Reducer<SplitPartitionedWritable, DenseBlockWritable, SplitPartitionedWritable, VectorWritable> { /* * HACK: partition number formats in hadoop, copied. this may stop working * if it gets out of sync with newer hadoop version. But unfortunately rules * of forming output file names are not sufficiently exposed so we need to * hack it if we write the same split output from either mapper or reducer. * alternatively, we probably can replace it by our own output file naming * management completely and bypass MultipleOutputs entirely. */ private static final NumberFormat NUMBER_FORMAT = NumberFormat.getInstance(); static { NUMBER_FORMAT.setMinimumIntegerDigits(5); NUMBER_FORMAT.setGroupingUsed(false); } private final Deque<Closeable> closeables = Lists.newLinkedList(); protected int blockHeight; protected int accumSize; protected int lastTaskId = -1; protected OutputCollector<Writable, DenseBlockWritable> qhatCollector; protected OutputCollector<Writable, VectorWritable> rhatCollector; protected QRFirstStep qr; protected Vector yiRow; protected Vector sb; @Override protected void setup(Context context) throws IOException, InterruptedException { Configuration conf = context.getConfiguration(); blockHeight = conf.getInt(BtJob.PROP_OUTER_PROD_BLOCK_HEIGHT, -1); String sbPathStr = conf.get(PROP_SB_PATH); /* * PCA -related corrections (MAHOUT-817) */ if (sbPathStr != null) { sb = SSVDHelper.loadAndSumUpVectors(new Path(sbPathStr), conf); } } protected void setupBlock(Context context, SplitPartitionedWritable spw) throws InterruptedException, IOException { IOUtils.close(closeables); qhatCollector = createOutputCollector(QJob.OUTPUT_QHAT, spw, context, DenseBlockWritable.class); rhatCollector = createOutputCollector(QJob.OUTPUT_RHAT, spw, context, VectorWritable.class); qr = new QRFirstStep(context.getConfiguration(), qhatCollector, rhatCollector); closeables.addFirst(qr); lastTaskId = spw.getTaskId(); } @Override protected void reduce(SplitPartitionedWritable key, Iterable<DenseBlockWritable> values, Context context) throws IOException, InterruptedException { if (key.getTaskId() != lastTaskId) { setupBlock(context, key); } Iterator<DenseBlockWritable> iter = values.iterator(); DenseBlockWritable dbw = iter.next(); double[][] yiCols = dbw.getBlock(); if (iter.hasNext()) { throw new IOException("Unexpected extra Y_i block in reducer input."); } long blockBase = key.getTaskItemOrdinal() * blockHeight; int bh = yiCols[0].length; if (yiRow == null) { yiRow = new DenseVector(yiCols.length); } for (int k = 0; k < bh; k++) { for (int j = 0; j < yiCols.length; j++) { yiRow.setQuick(j, yiCols[j][k]); } key.setTaskItemOrdinal(blockBase + k); // pca offset correction if any if (sb != null) { yiRow.assign(sb, Functions.MINUS); } qr.collect(key, yiRow); } } private Path getSplitFilePath(String name, SplitPartitionedWritable spw, Context context) throws InterruptedException, IOException { String uniqueFileName = FileOutputFormat.getUniqueFile(context, name, ""); uniqueFileName = uniqueFileName.replaceFirst("-r-", "-m-"); uniqueFileName = uniqueFileName.replaceFirst("\\d+$", Matcher.quoteReplacement(NUMBER_FORMAT.format(spw.getTaskId()))); return new Path(FileOutputFormat.getWorkOutputPath(context), uniqueFileName); } /** * key doesn't matter here, only value does. key always gets substituted by * SPW. * * @param <K> * bogus */ private <K, V> OutputCollector<K, V> createOutputCollector(String name, final SplitPartitionedWritable spw, Context ctx, Class<V> valueClass) throws IOException, InterruptedException { Path outputPath = getSplitFilePath(name, spw, ctx); final SequenceFile.Writer w = SequenceFile.createWriter( FileSystem.get(outputPath.toUri(), ctx.getConfiguration()), ctx.getConfiguration(), outputPath, SplitPartitionedWritable.class, valueClass); closeables.addFirst(w); return new OutputCollector<K, V>() { @Override public void collect(K key, V val) throws IOException { w.append(spw, val); } }; } @Override protected void cleanup(Context context) throws IOException, InterruptedException { IOUtils.close(closeables); } } public static void run(Configuration conf, Path[] inputAPaths, Path inputBtGlob, Path xiPath, Path sqPath, Path sbPath, Path outputPath, int aBlockRows, int minSplitSize, int k, int p, int outerProdBlockHeight, int numReduceTasks, boolean broadcastBInput) throws ClassNotFoundException, InterruptedException, IOException { JobConf oldApiJob = new JobConf(conf); Job job = new Job(oldApiJob); job.setJobName("ABt-job"); job.setJarByClass(ABtDenseOutJob.class); job.setInputFormatClass(SequenceFileInputFormat.class); FileInputFormat.setInputPaths(job, inputAPaths); if (minSplitSize > 0) { FileInputFormat.setMinInputSplitSize(job, minSplitSize); } FileOutputFormat.setOutputPath(job, outputPath); SequenceFileOutputFormat.setOutputCompressionType(job, CompressionType.BLOCK); job.setMapOutputKeyClass(SplitPartitionedWritable.class); job.setMapOutputValueClass(DenseBlockWritable.class); job.setOutputKeyClass(SplitPartitionedWritable.class); job.setOutputValueClass(VectorWritable.class); job.setMapperClass(ABtMapper.class); job.setReducerClass(QRReducer.class); job.getConfiguration().setInt(QJob.PROP_AROWBLOCK_SIZE, aBlockRows); job.getConfiguration().setInt(BtJob.PROP_OUTER_PROD_BLOCK_HEIGHT, outerProdBlockHeight); job.getConfiguration().setInt(QRFirstStep.PROP_K, k); job.getConfiguration().setInt(QRFirstStep.PROP_P, p); job.getConfiguration().set(PROP_BT_PATH, inputBtGlob.toString()); /* * PCA-related options, MAHOUT-817 */ if (xiPath != null) { job.getConfiguration().set(PROP_XI_PATH, xiPath.toString()); job.getConfiguration().set(PROP_SB_PATH, sbPath.toString()); job.getConfiguration().set(PROP_SQ_PATH, sqPath.toString()); } job.setNumReduceTasks(numReduceTasks); // broadcast Bt files if required. if (broadcastBInput) { job.getConfiguration().set(PROP_BT_BROADCAST, "y"); FileSystem fs = FileSystem.get(inputBtGlob.toUri(), conf); FileStatus[] fstats = fs.globStatus(inputBtGlob); if (fstats != null) { for (FileStatus fstat : fstats) { /* * new api is not enabled yet in our dependencies at this time, still * using deprecated one */ DistributedCache.addCacheFile(fstat.getPath().toUri(), job.getConfiguration()); } } } job.submit(); job.waitForCompletion(false); if (!job.isSuccessful()) { throw new IOException("ABt job unsuccessful."); } } }