Java tutorial
/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package org.apache.sysml.hops; import java.util.ArrayList; import java.util.HashMap; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.sysml.api.DMLScript; import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM; import org.apache.sysml.conf.ConfigurationManager; import org.apache.sysml.lops.CSVReBlock; import org.apache.sysml.lops.Checkpoint; import org.apache.sysml.lops.Compression; import org.apache.sysml.lops.Data; import org.apache.sysml.lops.Lop; import org.apache.sysml.lops.LopProperties.ExecType; import org.apache.sysml.lops.LopsException; import org.apache.sysml.lops.MultipleCP; import org.apache.sysml.lops.ReBlock; import org.apache.sysml.lops.UnaryCP; import org.apache.sysml.parser.Expression.DataType; import org.apache.sysml.parser.Expression.ValueType; import org.apache.sysml.runtime.controlprogram.LocalVariableMap; import org.apache.sysml.runtime.controlprogram.caching.MatrixObject.UpdateType; import org.apache.sysml.runtime.controlprogram.context.SparkExecutionContext; import org.apache.sysml.runtime.controlprogram.parfor.util.IDSequence; import org.apache.sysml.runtime.matrix.MatrixCharacteristics; import org.apache.sysml.runtime.matrix.data.MatrixBlock; import org.apache.sysml.runtime.util.UtilFunctions; public abstract class Hop { protected static final Log LOG = LogFactory.getLog(Hop.class.getName()); public static final long CPThreshold = 2000; public enum VisitStatus { DONE, NOTVISITED } /** * Optional hop interface, to be implemented by multi-threaded hops. */ public interface MultiThreadedHop { public abstract void setMaxNumThreads(int k); public abstract int getMaxNumThreads(); } // static variable to assign an unique ID to every hop that is created private static IDSequence _seqHopID = new IDSequence(); protected long _ID; protected String _name; protected DataType _dataType; protected ValueType _valueType; protected VisitStatus _visited = VisitStatus.NOTVISITED; protected long _dim1 = -1; protected long _dim2 = -1; protected long _rows_in_block = -1; protected long _cols_in_block = -1; protected long _nnz = -1; protected UpdateType _updateType = UpdateType.COPY; protected ArrayList<Hop> _parent = new ArrayList<Hop>(); protected ArrayList<Hop> _input = new ArrayList<Hop>(); protected ExecType _etype = null; //currently used exec type protected ExecType _etypeForced = null; //exec type forced via platform or external optimizer // Estimated size for the output produced from this Hop protected double _outputMemEstimate = OptimizerUtils.INVALID_SIZE; // Estimated size for the entire operation represented by this Hop // It includes the memory required for all inputs as well as the output protected double _memEstimate = OptimizerUtils.INVALID_SIZE; protected double _processingMemEstimate = 0; protected double _spBroadcastMemEstimate = 0; // indicates if there are unknowns during compilation // (in that case re-complication ensures robustness and efficiency) protected boolean _requiresRecompile = false; // indicates if the output of this hop needs to be reblocked // (usually this happens on persistent reads dataops) protected boolean _requiresReblock = false; // indicates if the output of this hop needs to be compressed // (this happens on persistent reads after reblock but before checkpoint) protected boolean _requiresCompression = false; // indicates if the output of this hop needs to be checkpointed (cached) // (the default storage level for caching is not yet exposed here) protected boolean _requiresCheckpoint = false; // indicates if the output of this hops needs to contain materialized empty blocks // if those exists; otherwise only blocks w/ non-zero values are materialized protected boolean _outputEmptyBlocks = true; private Lop _lops = null; protected Hop() { //default constructor for clone } public Hop(String l, DataType dt, ValueType vt) { _ID = getNextHopID(); setName(l); setDataType(dt); setValueType(vt); } private static long getNextHopID() { return _seqHopID.getNextID(); } public long getHopID() { return _ID; } public ExecType getExecType() { return _etype; } public void resetExecType() { _etype = null; } public ExecType getForcedExecType() { return _etypeForced; } public void setForcedExecType(ExecType etype) { _etypeForced = etype; } public abstract boolean allowsAllExecTypes(); /** * Defines if this operation is transpose-safe, which means that * the result of op(input) is equivalent to op(t(input)). * Usually, this applies to aggregate operations with fixed output * dimension. Finally, this information is very useful in order to * safely optimize the plan for sparse vectors, which otherwise * would be (currently) always represented dense. * * * @return always returns false */ public boolean isTransposeSafe() { //by default: its conservatively define as unsafe return false; } public void checkAndSetForcedPlatform() { if (DMLScript.rtplatform == RUNTIME_PLATFORM.SINGLE_NODE) _etypeForced = ExecType.CP; else if (DMLScript.rtplatform == RUNTIME_PLATFORM.HADOOP) _etypeForced = ExecType.MR; else if (DMLScript.rtplatform == RUNTIME_PLATFORM.SPARK) _etypeForced = ExecType.SPARK; } public void checkAndSetInvalidCPDimsAndSize() { if (_etype == ExecType.CP) { boolean invalid = false; //Step 1: check dimensions of output and all inputs (INTEGER) invalid |= !OptimizerUtils.isValidCPDimensions(_dim1, _dim2); for (Hop in : getInput()) invalid |= !OptimizerUtils.isValidCPDimensions(in._dim1, in._dim2); //Step 2: check valid output and input sizes for cp (<16GB for DENSE) //(if the memory estimate is smaller than max_numcells we are guaranteed to have it in sparse representation) invalid |= !(OptimizerUtils.isValidCPMatrixSize(_dim1, _dim2, OptimizerUtils.getSparsity(_dim1, _dim2, _nnz)) || getOutputMemEstimate() < OptimizerUtils.MAX_NUMCELLS_CP_DENSE); for (Hop in : getInput()) invalid |= !(OptimizerUtils.isValidCPMatrixSize(in._dim1, in._dim2, OptimizerUtils.getSparsity(in._dim1, in._dim2, in._nnz)) || in.getOutputMemEstimate() < OptimizerUtils.MAX_NUMCELLS_CP_DENSE); //force exec type mr if necessary if (invalid) { if (DMLScript.rtplatform == RUNTIME_PLATFORM.HYBRID) _etype = ExecType.MR; else if (DMLScript.rtplatform == RUNTIME_PLATFORM.HYBRID_SPARK) _etype = ExecType.SPARK; } } } public void setRequiresReblock(boolean flag) { _requiresReblock = flag; } public void setRequiresCompression(boolean flag) { _requiresCompression = flag; } public boolean hasMatrixInputWithDifferentBlocksizes() { for (Hop c : getInput()) { if (c.getDataType() == DataType.MATRIX && (getRowsInBlock() != c.getRowsInBlock() || getColsInBlock() != c.getColsInBlock())) { return true; } } return false; } public void setOutputBlocksizes(long brlen, long bclen) { setRowsInBlock(brlen); setColsInBlock(bclen); } public boolean requiresReblock() { return _requiresReblock; } public void setRequiresCheckpoint(boolean flag) { _requiresCheckpoint = flag; } public boolean requiresCheckpoint() { return _requiresCheckpoint; } public void constructAndSetLopsDataFlowProperties() throws HopsException { //Step 1: construct reblock lop if required (output of hop) constructAndSetReblockLopIfRequired(); //Step 2: construct compression lop if required constructAndSetCompressionLopIfRequired(); //Step 3: construct checkpoint lop if required (output of hop or reblock) constructAndSetCheckpointLopIfRequired(); } private void constructAndSetReblockLopIfRequired() throws HopsException { //determine execution type ExecType et = ExecType.CP; if (DMLScript.rtplatform != RUNTIME_PLATFORM.SINGLE_NODE && !(getDataType() == DataType.SCALAR)) { et = OptimizerUtils.isSparkExecutionMode() ? ExecType.SPARK : ExecType.MR; } //add reblock lop to output if required if (_requiresReblock && et != ExecType.CP) { Lop input = getLops(); Lop reblock = null; try { if ((this instanceof DataOp // CSV && ((DataOp) this).getDataOpType() == DataOpTypes.PERSISTENTREAD && ((DataOp) this).getInputFormatType() == FileFormatTypes.CSV) || (this instanceof ParameterizedBuiltinOp && ((ParameterizedBuiltinOp) this).getOp() == ParamBuiltinOp.TRANSFORM)) { reblock = new CSVReBlock(input, getRowsInBlock(), getColsInBlock(), getDataType(), getValueType(), et); } else //TEXT / MM / BINARYBLOCK / BINARYCELL { reblock = new ReBlock(input, getRowsInBlock(), getColsInBlock(), getDataType(), getValueType(), _outputEmptyBlocks, et); } } catch (LopsException ex) { throw new HopsException(ex); } setOutputDimensions(reblock); setLineNumbers(reblock); setLops(reblock); } } @SuppressWarnings("unused") //see CHECKPOINT_SPARSE_CSR private void constructAndSetCheckpointLopIfRequired() throws HopsException { //determine execution type ExecType et = ExecType.CP; if (OptimizerUtils.isSparkExecutionMode() && getDataType() != DataType.SCALAR) { //conditional checkpoint based on memory estimate in order to //(1) avoid unnecessary persist and unpersist calls, and //(2) avoid unnecessary creation of spark context (incl executors) if (OptimizerUtils.isHybridExecutionMode() && !OptimizerUtils.exceedsCachingThreshold(getDim2(), _outputMemEstimate) || _etypeForced == ExecType.CP) { et = ExecType.CP; } else //default case { et = ExecType.SPARK; } } //add checkpoint lop to output if required if (_requiresCheckpoint && et != ExecType.CP) { try { //investigate need for serialized storage of large sparse matrices //(compile- instead of runtime-level for better debugging) boolean serializedStorage = false; if (getDataType() == DataType.MATRIX && dimsKnown(true) && !Checkpoint.CHECKPOINT_SPARSE_CSR) { double matrixPSize = OptimizerUtils.estimatePartitionedSizeExactSparsity(_dim1, _dim2, _rows_in_block, _cols_in_block, _nnz); double dataCache = SparkExecutionContext.getDataMemoryBudget(true, true); serializedStorage = (MatrixBlock.evalSparseFormatInMemory(_dim1, _dim2, _nnz) && matrixPSize > dataCache); //sparse in-memory does not fit in agg mem } else if (!dimsKnown(true)) { setRequiresRecompile(); } //construct checkpoint w/ right storage level Lop input = getLops(); Lop chkpoint = new Checkpoint(input, getDataType(), getValueType(), serializedStorage ? Checkpoint.getSerializeStorageLevelString() : Checkpoint.getDefaultStorageLevelString()); setOutputDimensions(chkpoint); setLineNumbers(chkpoint); setLops(chkpoint); } catch (LopsException ex) { throw new HopsException(ex); } } } private void constructAndSetCompressionLopIfRequired() throws HopsException { //determine execution type ExecType et = ExecType.CP; if (OptimizerUtils.isSparkExecutionMode() && getDataType() != DataType.SCALAR) { //conditional checkpoint based on memory estimate in order to avoid unnecessary //persist and unpersist calls (4x the memory budget is conservative) if (OptimizerUtils.isHybridExecutionMode() && 2 * _outputMemEstimate < OptimizerUtils.getLocalMemBudget() || _etypeForced == ExecType.CP) { et = ExecType.CP; } else //default case { et = ExecType.SPARK; } } //add reblock lop to output if required if (_requiresCompression) { try { Lop compress = new Compression(getLops(), getDataType(), getValueType(), et); setOutputDimensions(compress); setLineNumbers(compress); setLops(compress); } catch (LopsException ex) { throw new HopsException(ex); } } } public static Lop createOffsetLop(Hop hop, boolean repCols) throws HopsException, LopsException { Lop offset = null; if (ConfigurationManager.isDynamicRecompilation() && hop.dimsKnown()) { // If dynamic recompilation is enabled and dims are known, we can replace the ncol with // a literal in order to increase the piggybacking potential. This is safe because append // is always marked for recompilation and hence, we have propagated the exact dimensions. offset = Data.createLiteralLop(ValueType.INT, String.valueOf(repCols ? hop.getDim2() : hop.getDim1())); } else { offset = new UnaryCP(hop.constructLops(), repCols ? UnaryCP.OperationTypes.NCOL : UnaryCP.OperationTypes.NROW, DataType.SCALAR, ValueType.INT); } offset.getOutputParameters().setDimensions(0, 0, 0, 0, -1); offset.setAllPositions(hop.getBeginLine(), hop.getBeginColumn(), hop.getEndLine(), hop.getEndColumn()); return offset; } public void setOutputEmptyBlocks(boolean flag) { _outputEmptyBlocks = flag; } public boolean isOutputEmptyBlocks() { return _outputEmptyBlocks; } /** * Returns the memory estimate for the output produced from this Hop. * It must be invoked only within Hops. From outside Hops, one must * only use getMemEstimate(), which gives memory required to store * all inputs and the output. * * @return output size memory estimate */ protected double getOutputSize() { return _outputMemEstimate; } protected double getInputSize() { double sum = 0; int len = _input.size(); for (int i = 0; i < len; i++) //for all inputs { Hop hi = _input.get(i); double hmout = hi.getOutputMemEstimate(); if (hmout > 1024 * 1024) {//for relevant sizes //check if already included in estimate (if an input is used //multiple times it is still only required once in memory) //(not that this check benefits from common subexpression elimination) boolean flag = false; for (int j = 0; j < i; j++) flag |= (hi == _input.get(j)); hmout = flag ? 0 : hmout; } sum += hmout; } //for(Hop h : _input ) { // sum += h._outputMemEstimate; //} return sum; } protected double getInputOutputSize() { double sum = 0; sum += _outputMemEstimate; sum += _processingMemEstimate; sum += getInputSize(); return sum; } protected double getInputSize(int pos) { double ret = 0; if (_input.size() > pos) ret = _input.get(pos)._outputMemEstimate; return ret; } protected double getIntermediateSize() { return _processingMemEstimate; } /** * NOTES: * * Purpose: Whenever the output dimensions / sparsity of a hop are unknown, this hop * should store its worst-case output statistics (if known) in that table. Subsequent * hops can then * * Invocation: Intended to be called for ALL root nodes of one Hops DAG with the same * (initially empty) memo table. * * @return memory estimate */ public double getMemEstimate() { if (OptimizerUtils.isMemoryBasedOptLevel()) { if (!isMemEstimated()) { //LOG.warn("Nonexisting memory estimate - reestimating w/o memo table."); computeMemEstimate(new MemoTable()); } return _memEstimate; } else { return OptimizerUtils.INVALID_SIZE; } } /** * Sets memory estimate in bytes * * @param mem memory estimate */ public void setMemEstimate(double mem) { _memEstimate = mem; } public void clearMemEstimate() { _memEstimate = OptimizerUtils.INVALID_SIZE; } public boolean isMemEstimated() { return (_memEstimate != OptimizerUtils.INVALID_SIZE); } //wrappers for meaningful public names to memory estimates. public double getInputMemEstimate() { return getInputSize(); } public double getOutputMemEstimate() { return getOutputSize(); } public double getIntermediateMemEstimate() { return getIntermediateSize(); } public double getSpBroadcastSize() { return _spBroadcastMemEstimate; } /** * Computes the estimate of memory required to store the input/output of this hop in memory. * This is the default implementation (orchestration of hop-specific implementation) * that should suffice for most hops. If a hop requires more control, this method should * be overwritten with awareness of (1) output estimates, and (2) propagation of worst-case * matrix characteristics (dimensions, sparsity). * * TODO remove memo table and, on constructor refresh, inference in refresh, single compute mem, * maybe general computeMemEstimate, flags to indicate if estimate or not. * * @param memo memory table */ public void computeMemEstimate(MemoTable memo) { long[] wstats = null; //////// //Step 1) Compute hop output memory estimate (incl size inference) switch (getDataType()) { case SCALAR: { //memory estimate always known if (getValueType() == ValueType.DOUBLE) //default case _outputMemEstimate = OptimizerUtils.DOUBLE_SIZE; else //literalops, dataops _outputMemEstimate = computeOutputMemEstimate(_dim1, _dim2, _nnz); break; } case FRAME: case MATRIX: { //1a) mem estimate based on exactly known dimensions and sparsity if (dimsKnown(true)) { //nnz always exactly known (see dimsKnown(true)) _outputMemEstimate = computeOutputMemEstimate(_dim1, _dim2, _nnz); } //1b) infer output statistics and mem estimate based on these statistics else if (memo.hasInputStatistics(this)) { //infer the output stats wstats = inferOutputCharacteristics(memo); if (wstats != null) { //use worst case characteristics to estimate mem long lnnz = ((wstats[2] >= 0) ? wstats[2] : wstats[0] * wstats[1]); _outputMemEstimate = computeOutputMemEstimate(wstats[0], wstats[1], lnnz); //propagate worst-case estimate memo.memoizeStatistics(getHopID(), wstats[0], wstats[1], wstats[2]); } else if (dimsKnown()) { //nnz unknown, estimate mem as dense long lnnz = _dim1 * _dim2; _outputMemEstimate = computeOutputMemEstimate(_dim1, _dim2, lnnz); } else { //unknown output size _outputMemEstimate = OptimizerUtils.DEFAULT_SIZE; } } //1c) mem estimate based on exactly known dimensions and unknown sparsity //(required e.g., for datagenops w/o any input statistics) else if (dimsKnown()) { //nnz unknown, estimate mem as dense long lnnz = _dim1 * _dim2; _outputMemEstimate = computeOutputMemEstimate(_dim1, _dim2, lnnz); } //1d) fallback: unknown output size else { _outputMemEstimate = OptimizerUtils.DEFAULT_SIZE; } break; } case OBJECT: case UNKNOWN: { //memory estimate always unknown _outputMemEstimate = OptimizerUtils.DEFAULT_SIZE; break; } } //////// //Step 2) Compute hop intermediate memory estimate //note: ensure consistency w/ step 1 (for simplified debugging) if (dimsKnown(true)) { //incl scalar output //nnz always exactly known (see dimsKnown(true)) _processingMemEstimate = computeIntermediateMemEstimate(_dim1, _dim2, _nnz); } else if (wstats != null) { //use worst case characteristics to estimate mem long lnnz = ((wstats[2] >= 0) ? wstats[2] : wstats[0] * wstats[1]); _processingMemEstimate = computeIntermediateMemEstimate(wstats[0], wstats[1], lnnz); } else if (dimsKnown()) { //nnz unknown, estimate mem as dense long lnnz = _dim1 * _dim2; _processingMemEstimate = computeIntermediateMemEstimate(_dim1, _dim2, lnnz); } //////// //Step 3) Compute final hop memory estimate //final estimate (sum of inputs/intermediates/output) _memEstimate = getInputOutputSize(); } /** * Computes the hop-specific output memory estimate in bytes. Should be 0 if not * applicable. * * @param dim1 dimension 1 * @param dim2 dimension 2 * @param nnz number of non-zeros * @return memory estimate */ protected abstract double computeOutputMemEstimate(long dim1, long dim2, long nnz); /** * Computes the hop-specific intermediate memory estimate in bytes. Should be 0 if not * applicable. * * @param dim1 dimension 1 * @param dim2 dimension 2 * @param nnz number of non-zeros * @return memory estimate */ protected abstract double computeIntermediateMemEstimate(long dim1, long dim2, long nnz); /** * Computes the output matrix characteristics (rows, cols, nnz) based on worst-case output * and/or input estimates. Should return null if dimensions are unknown. * * @param memo memory table * @return output characteristics as a long array */ protected abstract long[] inferOutputCharacteristics(MemoTable memo); /** * Recursively computes memory estimates for all the Hops in the DAG rooted at the * current hop pointed by <code>this</code>. * * @param memo memory table */ public void refreshMemEstimates(MemoTable memo) { if (getVisited() == VisitStatus.DONE) return; for (Hop h : this.getInput()) h.refreshMemEstimates(memo); this.computeMemEstimate(memo); this.setVisited(VisitStatus.DONE); } /** * This method determines the execution type (CP, MR) based ONLY on the * estimated memory footprint required for this operation, which includes * memory for all inputs and the output represented by this Hop. * * It is used when <code>OptimizationType = MEMORY_BASED</code>. * This optimization schedules an operation to CP whenever inputs+output * fit in memory -- note that this decision MAY NOT be optimal in terms of * execution time. * * @return execution type */ protected ExecType findExecTypeByMemEstimate() { ExecType et = null; char c = ' '; if (getMemEstimate() < OptimizerUtils.getLocalMemBudget()) { et = ExecType.CP; } else { if (DMLScript.rtplatform == DMLScript.RUNTIME_PLATFORM.HYBRID) et = ExecType.MR; else if (DMLScript.rtplatform == DMLScript.RUNTIME_PLATFORM.HYBRID_SPARK) et = ExecType.SPARK; c = '*'; } if (LOG.isDebugEnabled()) { String s = String.format(" %c %-5s %-8s (%s,%s) %s", c, getHopID(), getOpString(), OptimizerUtils.toMB(_outputMemEstimate), OptimizerUtils.toMB(_memEstimate), et); //System.out.println(s); LOG.debug(s); } return et; } protected ExecType findGPUExecTypeByMemEstimate(ExecType et) { if (DMLScript.USE_ACCELERATOR && (DMLScript.FORCE_ACCELERATOR || getMemEstimate() < OptimizerUtils.GPU_MEMORY_BUDGET)) { return ExecType.GPU; } return et; } public ArrayList<Hop> getParent() { return _parent; } public ArrayList<Hop> getInput() { return _input; } public long getRowsInBlock() { return _rows_in_block; } public void setRowsInBlock(long rowsInBlock) { _rows_in_block = rowsInBlock; } public long getColsInBlock() { return _cols_in_block; } public void setColsInBlock(long colsInBlock) { _cols_in_block = colsInBlock; } public void setNnz(long nnz) { _nnz = nnz; } public long getNnz() { return _nnz; } public void setUpdateType(UpdateType update) { _updateType = update; } public UpdateType getUpdateType() { return _updateType; } public abstract Lop constructLops() throws HopsException, LopsException; protected abstract ExecType optFindExecType() throws HopsException; public abstract String getOpString(); protected boolean isVector() { return (dimsKnown() && (_dim1 == 1 || _dim2 == 1)); } protected boolean areDimsBelowThreshold() { return (dimsKnown() && _dim1 <= Hop.CPThreshold && _dim2 <= Hop.CPThreshold); } public boolean dimsKnown() { return (_dataType == DataType.SCALAR || ((_dataType == DataType.MATRIX || _dataType == DataType.FRAME) && _dim1 > 0 && _dim2 > 0)); } public boolean dimsKnown(boolean includeNnz) { return (_dataType == DataType.SCALAR || ((_dataType == DataType.MATRIX || _dataType == DataType.FRAME) && _dim1 > 0 && _dim2 > 0 && ((includeNnz) ? _nnz >= 0 : true))); } public boolean dimsKnownAny() { return (_dataType == DataType.SCALAR || ((_dataType == DataType.MATRIX || _dataType == DataType.FRAME) && (_dim1 > 0 || _dim2 > 0))); } public static void resetVisitStatus(ArrayList<Hop> hops) { if (hops != null) for (Hop hopRoot : hops) hopRoot.resetVisitStatus(); } public void resetVisitStatus() { if (getVisited() == Hop.VisitStatus.NOTVISITED) return; for (Hop h : this.getInput()) h.resetVisitStatus(); setVisited(Hop.VisitStatus.NOTVISITED); } public static void resetRecompilationFlag(ArrayList<Hop> hops, ExecType et) { resetVisitStatus(hops); for (Hop hopRoot : hops) hopRoot.resetRecompilationFlag(et); } public static void resetRecompilationFlag(Hop hops, ExecType et) { hops.resetVisitStatus(); hops.resetRecompilationFlag(et); } private void resetRecompilationFlag(ExecType et) { if (getVisited() == VisitStatus.DONE) return; //process child hops for (Hop h : getInput()) h.resetRecompilationFlag(et); //reset recompile flag if (et == null || getExecType() == et || getExecType() == null) _requiresRecompile = false; this.setVisited(VisitStatus.DONE); } public void printMe() throws HopsException { if (LOG.isDebugEnabled()) { StringBuilder s = new StringBuilder(""); s.append(getClass().getSimpleName() + " " + getHopID() + "\n"); s.append(" Label: " + getName() + "; DataType: " + _dataType + "; ValueType: " + _valueType + "\n"); s.append(" Parent: "); for (Hop h : getParent()) { s.append(h.hashCode() + "; "); } ; s.append("\n Input: "); for (Hop h : getInput()) { s.append(h.getHopID() + "; "); } s.append("\n dims [" + _dim1 + "," + _dim2 + "] blk [" + _rows_in_block + "," + _cols_in_block + "] nnz: " + _nnz + " UpdateInPlace: " + _updateType); s.append(" MemEstimate = Out " + (_outputMemEstimate / 1024 / 1024) + " MB, In&Out " + (_memEstimate / 1024 / 1024) + " MB"); LOG.debug(s.toString()); } } public long getDim1() { return _dim1; } public void setDim1(long dim1) { _dim1 = dim1; } public long getDim2() { return _dim2; } public void setDim2(long dim2) { _dim2 = dim2; } protected void setOutputDimensions(Lop lop) throws HopsException { lop.getOutputParameters().setDimensions(getDim1(), getDim2(), getRowsInBlock(), getColsInBlock(), getNnz(), getUpdateType()); } public Lop getLops() { return _lops; } public void setLops(Lop lops) { _lops = lops; } public VisitStatus getVisited() { return _visited; } public DataType getDataType() { return _dataType; } public void setDataType(DataType dt) { _dataType = dt; } public void setVisited(VisitStatus visited) { _visited = visited; } public void setName(String _name) { this._name = _name; } public String getName() { return _name; } public ValueType getValueType() { return _valueType; } public void setValueType(ValueType vt) { _valueType = vt; } public enum OpOp1 { NOT, ABS, SIN, COS, TAN, ASIN, ACOS, ATAN, SIGN, SQRT, LOG, EXP, CAST_AS_SCALAR, CAST_AS_MATRIX, CAST_AS_FRAME, CAST_AS_DOUBLE, CAST_AS_INT, CAST_AS_BOOLEAN, PRINT, EIGEN, NROW, NCOL, LENGTH, ROUND, IQM, STOP, CEIL, FLOOR, MEDIAN, INVERSE, CHOLESKY, //cumulative sums, products, extreme values CUMSUM, CUMPROD, CUMMIN, CUMMAX, //fused ML-specific operators for performance SPROP, //sample proportion: P * (1 - P) SIGMOID, //sigmoid function: 1 / (1 + exp(-X)) SELP, //select positive: X * (X>0) LOG_NZ, //sparse-safe log; ppred(X,0,"!=")*log(X) } // Operations that require two operands public enum OpOp2 { PLUS, MINUS, MULT, DIV, MODULUS, INTDIV, LESS, LESSEQUAL, GREATER, GREATEREQUAL, EQUAL, NOTEQUAL, MIN, MAX, AND, OR, LOG, POW, PRINT, CONCAT, QUANTILE, INTERQUANTILE, IQM, CENTRALMOMENT, COVARIANCE, CBIND, RBIND, SOLVE, MEDIAN, INVALID, //fused ML-specific operators for performance MINUS_NZ, //sparse-safe minus: X-(mean*ppred(X,0,!=)) LOG_NZ, //sparse-safe log; ppred(X,0,"!=")*log(X,0.5) MINUS1_MULT, //1-X*Y }; // Operations that require 3 operands public enum OpOp3 { QUANTILE, INTERQUANTILE, CTABLE, CENTRALMOMENT, COVARIANCE, PLUS_MULT, MINUS_MULT }; // Operations that require 4 operands public enum OpOp4 { WSLOSS, //weighted sloss mm WSIGMOID, //weighted sigmoid mm WDIVMM, //weighted divide mm WCEMM, //weighted cross entropy mm WUMM //weighted unary mm }; // Operations that require a variable number of operands public enum MultipleOperandOperation { PRINTF } public enum AggOp { SUM, SUM_SQ, MIN, MAX, TRACE, PROD, MEAN, VAR, MAXINDEX, MININDEX }; public enum ReOrgOp { TRANSPOSE, DIAG, RESHAPE, SORT, REV //Note: Diag types are invalid because for unknown sizes this would //create incorrect plans (now we try to infer it for memory estimates //and rewrites but the final choice is made during runtime) //DIAG_V2M, DIAG_M2V, }; public enum ConvOp { MAX_POOLING, MAX_POOLING_BACKWARD, DIRECT_CONV2D, DIRECT_CONV2D_BACKWARD_FILTER, DIRECT_CONV2D_BACKWARD_DATA, BIAS_ADD }; public enum DataGenMethod { RAND, SEQ, SINIT, SAMPLE, INVALID }; public enum ParamBuiltinOp { INVALID, CDF, INVCDF, GROUPEDAGG, RMEMPTY, REPLACE, REXPAND, TRANSFORM, TRANSFORMAPPLY, TRANSFORMDECODE, TRANSFORMMETA, TOSTRING }; public enum FileFormatTypes { TEXT, BINARY, MM, CSV }; public enum DataOpTypes { PERSISTENTREAD, PERSISTENTWRITE, TRANSIENTREAD, TRANSIENTWRITE, FUNCTIONOUTPUT }; public enum Direction { RowCol, Row, Col }; protected static final HashMap<DataOpTypes, org.apache.sysml.lops.Data.OperationTypes> HopsData2Lops; static { HopsData2Lops = new HashMap<Hop.DataOpTypes, org.apache.sysml.lops.Data.OperationTypes>(); HopsData2Lops.put(DataOpTypes.PERSISTENTREAD, org.apache.sysml.lops.Data.OperationTypes.READ); HopsData2Lops.put(DataOpTypes.PERSISTENTWRITE, org.apache.sysml.lops.Data.OperationTypes.WRITE); HopsData2Lops.put(DataOpTypes.TRANSIENTWRITE, org.apache.sysml.lops.Data.OperationTypes.WRITE); HopsData2Lops.put(DataOpTypes.TRANSIENTREAD, org.apache.sysml.lops.Data.OperationTypes.READ); } protected static final HashMap<Hop.AggOp, org.apache.sysml.lops.Aggregate.OperationTypes> HopsAgg2Lops; static { HopsAgg2Lops = new HashMap<Hop.AggOp, org.apache.sysml.lops.Aggregate.OperationTypes>(); HopsAgg2Lops.put(AggOp.SUM, org.apache.sysml.lops.Aggregate.OperationTypes.KahanSum); HopsAgg2Lops.put(AggOp.SUM_SQ, org.apache.sysml.lops.Aggregate.OperationTypes.KahanSumSq); HopsAgg2Lops.put(AggOp.TRACE, org.apache.sysml.lops.Aggregate.OperationTypes.KahanTrace); HopsAgg2Lops.put(AggOp.MIN, org.apache.sysml.lops.Aggregate.OperationTypes.Min); HopsAgg2Lops.put(AggOp.MAX, org.apache.sysml.lops.Aggregate.OperationTypes.Max); HopsAgg2Lops.put(AggOp.MAXINDEX, org.apache.sysml.lops.Aggregate.OperationTypes.MaxIndex); HopsAgg2Lops.put(AggOp.MININDEX, org.apache.sysml.lops.Aggregate.OperationTypes.MinIndex); HopsAgg2Lops.put(AggOp.PROD, org.apache.sysml.lops.Aggregate.OperationTypes.Product); HopsAgg2Lops.put(AggOp.MEAN, org.apache.sysml.lops.Aggregate.OperationTypes.Mean); HopsAgg2Lops.put(AggOp.VAR, org.apache.sysml.lops.Aggregate.OperationTypes.Var); } protected static final HashMap<ReOrgOp, org.apache.sysml.lops.Transform.OperationTypes> HopsTransf2Lops; static { HopsTransf2Lops = new HashMap<ReOrgOp, org.apache.sysml.lops.Transform.OperationTypes>(); HopsTransf2Lops.put(ReOrgOp.TRANSPOSE, org.apache.sysml.lops.Transform.OperationTypes.Transpose); HopsTransf2Lops.put(ReOrgOp.REV, org.apache.sysml.lops.Transform.OperationTypes.Rev); HopsTransf2Lops.put(ReOrgOp.DIAG, org.apache.sysml.lops.Transform.OperationTypes.Diag); HopsTransf2Lops.put(ReOrgOp.RESHAPE, org.apache.sysml.lops.Transform.OperationTypes.Reshape); HopsTransf2Lops.put(ReOrgOp.SORT, org.apache.sysml.lops.Transform.OperationTypes.Sort); } protected static final HashMap<ConvOp, org.apache.sysml.lops.ConvolutionTransform.OperationTypes> HopsConv2Lops; static { HopsConv2Lops = new HashMap<ConvOp, org.apache.sysml.lops.ConvolutionTransform.OperationTypes>(); HopsConv2Lops.put(ConvOp.MAX_POOLING, org.apache.sysml.lops.ConvolutionTransform.OperationTypes.MAX_POOLING); HopsConv2Lops.put(ConvOp.MAX_POOLING_BACKWARD, org.apache.sysml.lops.ConvolutionTransform.OperationTypes.MAX_POOLING_BACKWARD); HopsConv2Lops.put(ConvOp.DIRECT_CONV2D, org.apache.sysml.lops.ConvolutionTransform.OperationTypes.DIRECT_CONV2D); HopsConv2Lops.put(ConvOp.BIAS_ADD, org.apache.sysml.lops.ConvolutionTransform.OperationTypes.BIAS_ADD); HopsConv2Lops.put(ConvOp.DIRECT_CONV2D_BACKWARD_FILTER, org.apache.sysml.lops.ConvolutionTransform.OperationTypes.DIRECT_CONV2D_BACKWARD_FILTER); HopsConv2Lops.put(ConvOp.DIRECT_CONV2D_BACKWARD_DATA, org.apache.sysml.lops.ConvolutionTransform.OperationTypes.DIRECT_CONV2D_BACKWARD_DATA); } protected static final HashMap<Hop.Direction, org.apache.sysml.lops.PartialAggregate.DirectionTypes> HopsDirection2Lops; static { HopsDirection2Lops = new HashMap<Hop.Direction, org.apache.sysml.lops.PartialAggregate.DirectionTypes>(); HopsDirection2Lops.put(Direction.RowCol, org.apache.sysml.lops.PartialAggregate.DirectionTypes.RowCol); HopsDirection2Lops.put(Direction.Col, org.apache.sysml.lops.PartialAggregate.DirectionTypes.Col); HopsDirection2Lops.put(Direction.Row, org.apache.sysml.lops.PartialAggregate.DirectionTypes.Row); } protected static final HashMap<Hop.OpOp2, org.apache.sysml.lops.Binary.OperationTypes> HopsOpOp2LopsB; static { HopsOpOp2LopsB = new HashMap<Hop.OpOp2, org.apache.sysml.lops.Binary.OperationTypes>(); HopsOpOp2LopsB.put(OpOp2.PLUS, org.apache.sysml.lops.Binary.OperationTypes.ADD); HopsOpOp2LopsB.put(OpOp2.MINUS, org.apache.sysml.lops.Binary.OperationTypes.SUBTRACT); HopsOpOp2LopsB.put(OpOp2.MULT, org.apache.sysml.lops.Binary.OperationTypes.MULTIPLY); HopsOpOp2LopsB.put(OpOp2.DIV, org.apache.sysml.lops.Binary.OperationTypes.DIVIDE); HopsOpOp2LopsB.put(OpOp2.MODULUS, org.apache.sysml.lops.Binary.OperationTypes.MODULUS); HopsOpOp2LopsB.put(OpOp2.INTDIV, org.apache.sysml.lops.Binary.OperationTypes.INTDIV); HopsOpOp2LopsB.put(OpOp2.MINUS1_MULT, org.apache.sysml.lops.Binary.OperationTypes.MINUS1_MULTIPLY); HopsOpOp2LopsB.put(OpOp2.LESS, org.apache.sysml.lops.Binary.OperationTypes.LESS_THAN); HopsOpOp2LopsB.put(OpOp2.LESSEQUAL, org.apache.sysml.lops.Binary.OperationTypes.LESS_THAN_OR_EQUALS); HopsOpOp2LopsB.put(OpOp2.GREATER, org.apache.sysml.lops.Binary.OperationTypes.GREATER_THAN); HopsOpOp2LopsB.put(OpOp2.GREATEREQUAL, org.apache.sysml.lops.Binary.OperationTypes.GREATER_THAN_OR_EQUALS); HopsOpOp2LopsB.put(OpOp2.EQUAL, org.apache.sysml.lops.Binary.OperationTypes.EQUALS); HopsOpOp2LopsB.put(OpOp2.NOTEQUAL, org.apache.sysml.lops.Binary.OperationTypes.NOT_EQUALS); HopsOpOp2LopsB.put(OpOp2.MIN, org.apache.sysml.lops.Binary.OperationTypes.MIN); HopsOpOp2LopsB.put(OpOp2.MAX, org.apache.sysml.lops.Binary.OperationTypes.MAX); HopsOpOp2LopsB.put(OpOp2.AND, org.apache.sysml.lops.Binary.OperationTypes.OR); HopsOpOp2LopsB.put(OpOp2.OR, org.apache.sysml.lops.Binary.OperationTypes.AND); HopsOpOp2LopsB.put(OpOp2.SOLVE, org.apache.sysml.lops.Binary.OperationTypes.SOLVE); HopsOpOp2LopsB.put(OpOp2.POW, org.apache.sysml.lops.Binary.OperationTypes.POW); HopsOpOp2LopsB.put(OpOp2.LOG, org.apache.sysml.lops.Binary.OperationTypes.NOTSUPPORTED); } protected static final HashMap<Hop.OpOp2, org.apache.sysml.lops.BinaryScalar.OperationTypes> HopsOpOp2LopsBS; static { HopsOpOp2LopsBS = new HashMap<Hop.OpOp2, org.apache.sysml.lops.BinaryScalar.OperationTypes>(); HopsOpOp2LopsBS.put(OpOp2.PLUS, org.apache.sysml.lops.BinaryScalar.OperationTypes.ADD); HopsOpOp2LopsBS.put(OpOp2.MINUS, org.apache.sysml.lops.BinaryScalar.OperationTypes.SUBTRACT); HopsOpOp2LopsBS.put(OpOp2.MULT, org.apache.sysml.lops.BinaryScalar.OperationTypes.MULTIPLY); HopsOpOp2LopsBS.put(OpOp2.DIV, org.apache.sysml.lops.BinaryScalar.OperationTypes.DIVIDE); HopsOpOp2LopsBS.put(OpOp2.MODULUS, org.apache.sysml.lops.BinaryScalar.OperationTypes.MODULUS); HopsOpOp2LopsBS.put(OpOp2.INTDIV, org.apache.sysml.lops.BinaryScalar.OperationTypes.INTDIV); HopsOpOp2LopsBS.put(OpOp2.LESS, org.apache.sysml.lops.BinaryScalar.OperationTypes.LESS_THAN); HopsOpOp2LopsBS.put(OpOp2.LESSEQUAL, org.apache.sysml.lops.BinaryScalar.OperationTypes.LESS_THAN_OR_EQUALS); HopsOpOp2LopsBS.put(OpOp2.GREATER, org.apache.sysml.lops.BinaryScalar.OperationTypes.GREATER_THAN); HopsOpOp2LopsBS.put(OpOp2.GREATEREQUAL, org.apache.sysml.lops.BinaryScalar.OperationTypes.GREATER_THAN_OR_EQUALS); HopsOpOp2LopsBS.put(OpOp2.EQUAL, org.apache.sysml.lops.BinaryScalar.OperationTypes.EQUALS); HopsOpOp2LopsBS.put(OpOp2.NOTEQUAL, org.apache.sysml.lops.BinaryScalar.OperationTypes.NOT_EQUALS); HopsOpOp2LopsBS.put(OpOp2.MIN, org.apache.sysml.lops.BinaryScalar.OperationTypes.MIN); HopsOpOp2LopsBS.put(OpOp2.MAX, org.apache.sysml.lops.BinaryScalar.OperationTypes.MAX); HopsOpOp2LopsBS.put(OpOp2.AND, org.apache.sysml.lops.BinaryScalar.OperationTypes.AND); HopsOpOp2LopsBS.put(OpOp2.OR, org.apache.sysml.lops.BinaryScalar.OperationTypes.OR); HopsOpOp2LopsBS.put(OpOp2.LOG, org.apache.sysml.lops.BinaryScalar.OperationTypes.LOG); HopsOpOp2LopsBS.put(OpOp2.POW, org.apache.sysml.lops.BinaryScalar.OperationTypes.POW); HopsOpOp2LopsBS.put(OpOp2.PRINT, org.apache.sysml.lops.BinaryScalar.OperationTypes.PRINT); } protected static final HashMap<Hop.OpOp2, org.apache.sysml.lops.Unary.OperationTypes> HopsOpOp2LopsU; static { HopsOpOp2LopsU = new HashMap<Hop.OpOp2, org.apache.sysml.lops.Unary.OperationTypes>(); HopsOpOp2LopsU.put(OpOp2.PLUS, org.apache.sysml.lops.Unary.OperationTypes.ADD); HopsOpOp2LopsU.put(OpOp2.MINUS, org.apache.sysml.lops.Unary.OperationTypes.SUBTRACT); HopsOpOp2LopsU.put(OpOp2.MULT, org.apache.sysml.lops.Unary.OperationTypes.MULTIPLY); HopsOpOp2LopsU.put(OpOp2.DIV, org.apache.sysml.lops.Unary.OperationTypes.DIVIDE); HopsOpOp2LopsU.put(OpOp2.MODULUS, org.apache.sysml.lops.Unary.OperationTypes.MODULUS); HopsOpOp2LopsU.put(OpOp2.INTDIV, org.apache.sysml.lops.Unary.OperationTypes.INTDIV); HopsOpOp2LopsU.put(OpOp2.MINUS1_MULT, org.apache.sysml.lops.Unary.OperationTypes.MINUS1_MULTIPLY); HopsOpOp2LopsU.put(OpOp2.LESSEQUAL, org.apache.sysml.lops.Unary.OperationTypes.LESS_THAN_OR_EQUALS); HopsOpOp2LopsU.put(OpOp2.LESS, org.apache.sysml.lops.Unary.OperationTypes.LESS_THAN); HopsOpOp2LopsU.put(OpOp2.GREATEREQUAL, org.apache.sysml.lops.Unary.OperationTypes.GREATER_THAN_OR_EQUALS); HopsOpOp2LopsU.put(OpOp2.GREATER, org.apache.sysml.lops.Unary.OperationTypes.GREATER_THAN); HopsOpOp2LopsU.put(OpOp2.EQUAL, org.apache.sysml.lops.Unary.OperationTypes.EQUALS); HopsOpOp2LopsU.put(OpOp2.NOTEQUAL, org.apache.sysml.lops.Unary.OperationTypes.NOT_EQUALS); HopsOpOp2LopsU.put(OpOp2.AND, org.apache.sysml.lops.Unary.OperationTypes.NOTSUPPORTED); HopsOpOp2LopsU.put(OpOp2.OR, org.apache.sysml.lops.Unary.OperationTypes.NOTSUPPORTED); HopsOpOp2LopsU.put(OpOp2.MAX, org.apache.sysml.lops.Unary.OperationTypes.MAX); HopsOpOp2LopsU.put(OpOp2.MIN, org.apache.sysml.lops.Unary.OperationTypes.MIN); HopsOpOp2LopsU.put(OpOp2.LOG, org.apache.sysml.lops.Unary.OperationTypes.LOG); HopsOpOp2LopsU.put(OpOp2.POW, org.apache.sysml.lops.Unary.OperationTypes.POW); HopsOpOp2LopsU.put(OpOp2.MINUS_NZ, org.apache.sysml.lops.Unary.OperationTypes.SUBTRACT_NZ); HopsOpOp2LopsU.put(OpOp2.LOG_NZ, org.apache.sysml.lops.Unary.OperationTypes.LOG_NZ); } protected static final HashMap<Hop.OpOp1, org.apache.sysml.lops.Unary.OperationTypes> HopsOpOp1LopsU; static { HopsOpOp1LopsU = new HashMap<Hop.OpOp1, org.apache.sysml.lops.Unary.OperationTypes>(); HopsOpOp1LopsU.put(OpOp1.NOT, org.apache.sysml.lops.Unary.OperationTypes.NOT); HopsOpOp1LopsU.put(OpOp1.ABS, org.apache.sysml.lops.Unary.OperationTypes.ABS); HopsOpOp1LopsU.put(OpOp1.SIN, org.apache.sysml.lops.Unary.OperationTypes.SIN); HopsOpOp1LopsU.put(OpOp1.COS, org.apache.sysml.lops.Unary.OperationTypes.COS); HopsOpOp1LopsU.put(OpOp1.TAN, org.apache.sysml.lops.Unary.OperationTypes.TAN); HopsOpOp1LopsU.put(OpOp1.ASIN, org.apache.sysml.lops.Unary.OperationTypes.ASIN); HopsOpOp1LopsU.put(OpOp1.ACOS, org.apache.sysml.lops.Unary.OperationTypes.ACOS); HopsOpOp1LopsU.put(OpOp1.ATAN, org.apache.sysml.lops.Unary.OperationTypes.ATAN); HopsOpOp1LopsU.put(OpOp1.SIGN, org.apache.sysml.lops.Unary.OperationTypes.SIGN); HopsOpOp1LopsU.put(OpOp1.SQRT, org.apache.sysml.lops.Unary.OperationTypes.SQRT); HopsOpOp1LopsU.put(OpOp1.EXP, org.apache.sysml.lops.Unary.OperationTypes.EXP); HopsOpOp1LopsU.put(OpOp1.LOG, org.apache.sysml.lops.Unary.OperationTypes.LOG); HopsOpOp1LopsU.put(OpOp1.ROUND, org.apache.sysml.lops.Unary.OperationTypes.ROUND); HopsOpOp1LopsU.put(OpOp1.CEIL, org.apache.sysml.lops.Unary.OperationTypes.CEIL); HopsOpOp1LopsU.put(OpOp1.FLOOR, org.apache.sysml.lops.Unary.OperationTypes.FLOOR); HopsOpOp1LopsU.put(OpOp1.CUMSUM, org.apache.sysml.lops.Unary.OperationTypes.CUMSUM); HopsOpOp1LopsU.put(OpOp1.CUMPROD, org.apache.sysml.lops.Unary.OperationTypes.CUMPROD); HopsOpOp1LopsU.put(OpOp1.CUMMIN, org.apache.sysml.lops.Unary.OperationTypes.CUMMIN); HopsOpOp1LopsU.put(OpOp1.CUMMAX, org.apache.sysml.lops.Unary.OperationTypes.CUMMAX); HopsOpOp1LopsU.put(OpOp1.INVERSE, org.apache.sysml.lops.Unary.OperationTypes.INVERSE); HopsOpOp1LopsU.put(OpOp1.CHOLESKY, org.apache.sysml.lops.Unary.OperationTypes.CHOLESKY); HopsOpOp1LopsU.put(OpOp1.CAST_AS_SCALAR, org.apache.sysml.lops.Unary.OperationTypes.NOTSUPPORTED); HopsOpOp1LopsU.put(OpOp1.CAST_AS_MATRIX, org.apache.sysml.lops.Unary.OperationTypes.NOTSUPPORTED); HopsOpOp1LopsU.put(OpOp1.SPROP, org.apache.sysml.lops.Unary.OperationTypes.SPROP); HopsOpOp1LopsU.put(OpOp1.SIGMOID, org.apache.sysml.lops.Unary.OperationTypes.SIGMOID); HopsOpOp1LopsU.put(OpOp1.SELP, org.apache.sysml.lops.Unary.OperationTypes.SELP); HopsOpOp1LopsU.put(OpOp1.LOG_NZ, org.apache.sysml.lops.Unary.OperationTypes.LOG_NZ); HopsOpOp1LopsU.put(OpOp1.CAST_AS_MATRIX, org.apache.sysml.lops.Unary.OperationTypes.CAST_AS_MATRIX); HopsOpOp1LopsU.put(OpOp1.CAST_AS_FRAME, org.apache.sysml.lops.Unary.OperationTypes.CAST_AS_FRAME); } protected static final HashMap<Hop.OpOp1, org.apache.sysml.lops.UnaryCP.OperationTypes> HopsOpOp1LopsUS; static { HopsOpOp1LopsUS = new HashMap<Hop.OpOp1, org.apache.sysml.lops.UnaryCP.OperationTypes>(); HopsOpOp1LopsUS.put(OpOp1.NOT, org.apache.sysml.lops.UnaryCP.OperationTypes.NOT); HopsOpOp1LopsUS.put(OpOp1.ABS, org.apache.sysml.lops.UnaryCP.OperationTypes.ABS); HopsOpOp1LopsUS.put(OpOp1.SIN, org.apache.sysml.lops.UnaryCP.OperationTypes.SIN); HopsOpOp1LopsUS.put(OpOp1.COS, org.apache.sysml.lops.UnaryCP.OperationTypes.COS); HopsOpOp1LopsUS.put(OpOp1.TAN, org.apache.sysml.lops.UnaryCP.OperationTypes.TAN); HopsOpOp1LopsUS.put(OpOp1.ASIN, org.apache.sysml.lops.UnaryCP.OperationTypes.ASIN); HopsOpOp1LopsUS.put(OpOp1.ACOS, org.apache.sysml.lops.UnaryCP.OperationTypes.ACOS); HopsOpOp1LopsUS.put(OpOp1.ATAN, org.apache.sysml.lops.UnaryCP.OperationTypes.ATAN); HopsOpOp1LopsUS.put(OpOp1.SQRT, org.apache.sysml.lops.UnaryCP.OperationTypes.SQRT); HopsOpOp1LopsUS.put(OpOp1.EXP, org.apache.sysml.lops.UnaryCP.OperationTypes.EXP); HopsOpOp1LopsUS.put(OpOp1.LOG, org.apache.sysml.lops.UnaryCP.OperationTypes.LOG); HopsOpOp1LopsUS.put(OpOp1.CAST_AS_SCALAR, org.apache.sysml.lops.UnaryCP.OperationTypes.CAST_AS_SCALAR); HopsOpOp1LopsUS.put(OpOp1.CAST_AS_MATRIX, org.apache.sysml.lops.UnaryCP.OperationTypes.CAST_AS_MATRIX); HopsOpOp1LopsUS.put(OpOp1.CAST_AS_FRAME, org.apache.sysml.lops.UnaryCP.OperationTypes.CAST_AS_FRAME); HopsOpOp1LopsUS.put(OpOp1.CAST_AS_DOUBLE, org.apache.sysml.lops.UnaryCP.OperationTypes.CAST_AS_DOUBLE); HopsOpOp1LopsUS.put(OpOp1.CAST_AS_INT, org.apache.sysml.lops.UnaryCP.OperationTypes.CAST_AS_INT); HopsOpOp1LopsUS.put(OpOp1.CAST_AS_BOOLEAN, org.apache.sysml.lops.UnaryCP.OperationTypes.CAST_AS_BOOLEAN); HopsOpOp1LopsUS.put(OpOp1.NROW, org.apache.sysml.lops.UnaryCP.OperationTypes.NROW); HopsOpOp1LopsUS.put(OpOp1.NCOL, org.apache.sysml.lops.UnaryCP.OperationTypes.NCOL); HopsOpOp1LopsUS.put(OpOp1.LENGTH, org.apache.sysml.lops.UnaryCP.OperationTypes.LENGTH); HopsOpOp1LopsUS.put(OpOp1.PRINT, org.apache.sysml.lops.UnaryCP.OperationTypes.PRINT); HopsOpOp1LopsUS.put(OpOp1.ROUND, org.apache.sysml.lops.UnaryCP.OperationTypes.ROUND); HopsOpOp1LopsUS.put(OpOp1.CEIL, org.apache.sysml.lops.UnaryCP.OperationTypes.CEIL); HopsOpOp1LopsUS.put(OpOp1.FLOOR, org.apache.sysml.lops.UnaryCP.OperationTypes.FLOOR); HopsOpOp1LopsUS.put(OpOp1.STOP, org.apache.sysml.lops.UnaryCP.OperationTypes.STOP); } /** * Maps from a multiple (variable number of operands) Hop operation type to * the corresponding Lop operation type. This is called in the MultipleOp * constructLops() method that is used to construct the Lops that correspond * to a Hop. */ protected static final HashMap<MultipleOperandOperation, MultipleCP.OperationType> MultipleOperandOperationHopTypeToLopType; static { MultipleOperandOperationHopTypeToLopType = new HashMap<MultipleOperandOperation, MultipleCP.OperationType>(); MultipleOperandOperationHopTypeToLopType.put(MultipleOperandOperation.PRINTF, MultipleCP.OperationType.PRINTF); } protected static final HashMap<Hop.OpOp1, String> HopsOpOp12String; static { HopsOpOp12String = new HashMap<OpOp1, String>(); HopsOpOp12String.put(OpOp1.ABS, "abs"); HopsOpOp12String.put(OpOp1.CAST_AS_SCALAR, "castAsScalar"); HopsOpOp12String.put(OpOp1.COS, "cos"); HopsOpOp12String.put(OpOp1.EIGEN, "eigen"); HopsOpOp12String.put(OpOp1.EXP, "exp"); HopsOpOp12String.put(OpOp1.IQM, "iqm"); HopsOpOp12String.put(OpOp1.MEDIAN, "median"); HopsOpOp12String.put(OpOp1.LENGTH, "length"); HopsOpOp12String.put(OpOp1.LOG, "log"); HopsOpOp12String.put(OpOp1.NCOL, "ncol"); HopsOpOp12String.put(OpOp1.NOT, "!"); HopsOpOp12String.put(OpOp1.NROW, "nrow"); HopsOpOp12String.put(OpOp1.PRINT, "print"); HopsOpOp12String.put(OpOp1.ROUND, "round"); HopsOpOp12String.put(OpOp1.SIN, "sin"); HopsOpOp12String.put(OpOp1.SQRT, "sqrt"); HopsOpOp12String.put(OpOp1.TAN, "tan"); HopsOpOp12String.put(OpOp1.ASIN, "asin"); HopsOpOp12String.put(OpOp1.ACOS, "acos"); HopsOpOp12String.put(OpOp1.ATAN, "atan"); HopsOpOp12String.put(OpOp1.STOP, "stop"); HopsOpOp12String.put(OpOp1.INVERSE, "inv"); HopsOpOp12String.put(OpOp1.SPROP, "sprop"); HopsOpOp12String.put(OpOp1.SIGMOID, "sigmoid"); } protected static final HashMap<Hop.ParamBuiltinOp, org.apache.sysml.lops.ParameterizedBuiltin.OperationTypes> HopsParameterizedBuiltinLops; static { HopsParameterizedBuiltinLops = new HashMap<Hop.ParamBuiltinOp, org.apache.sysml.lops.ParameterizedBuiltin.OperationTypes>(); HopsParameterizedBuiltinLops.put(ParamBuiltinOp.CDF, org.apache.sysml.lops.ParameterizedBuiltin.OperationTypes.CDF); HopsParameterizedBuiltinLops.put(ParamBuiltinOp.INVCDF, org.apache.sysml.lops.ParameterizedBuiltin.OperationTypes.INVCDF); HopsParameterizedBuiltinLops.put(ParamBuiltinOp.RMEMPTY, org.apache.sysml.lops.ParameterizedBuiltin.OperationTypes.RMEMPTY); HopsParameterizedBuiltinLops.put(ParamBuiltinOp.REPLACE, org.apache.sysml.lops.ParameterizedBuiltin.OperationTypes.REPLACE); HopsParameterizedBuiltinLops.put(ParamBuiltinOp.REXPAND, org.apache.sysml.lops.ParameterizedBuiltin.OperationTypes.REXPAND); HopsParameterizedBuiltinLops.put(ParamBuiltinOp.TRANSFORM, org.apache.sysml.lops.ParameterizedBuiltin.OperationTypes.TRANSFORM); HopsParameterizedBuiltinLops.put(ParamBuiltinOp.TRANSFORMAPPLY, org.apache.sysml.lops.ParameterizedBuiltin.OperationTypes.TRANSFORMAPPLY); HopsParameterizedBuiltinLops.put(ParamBuiltinOp.TRANSFORMDECODE, org.apache.sysml.lops.ParameterizedBuiltin.OperationTypes.TRANSFORMDECODE); HopsParameterizedBuiltinLops.put(ParamBuiltinOp.TRANSFORMMETA, org.apache.sysml.lops.ParameterizedBuiltin.OperationTypes.TRANSFORMMETA); HopsParameterizedBuiltinLops.put(ParamBuiltinOp.TOSTRING, org.apache.sysml.lops.ParameterizedBuiltin.OperationTypes.TOSTRING); } protected static final HashMap<Hop.OpOp2, String> HopsOpOp2String; static { HopsOpOp2String = new HashMap<Hop.OpOp2, String>(); HopsOpOp2String.put(OpOp2.PLUS, "+"); HopsOpOp2String.put(OpOp2.MINUS, "-"); HopsOpOp2String.put(OpOp2.MINUS_NZ, "-nz"); HopsOpOp2String.put(OpOp2.MINUS1_MULT, "-1*"); HopsOpOp2String.put(OpOp2.MULT, "*"); HopsOpOp2String.put(OpOp2.DIV, "/"); HopsOpOp2String.put(OpOp2.MODULUS, "%%"); HopsOpOp2String.put(OpOp2.INTDIV, "%/%"); HopsOpOp2String.put(OpOp2.MIN, "min"); HopsOpOp2String.put(OpOp2.MAX, "max"); HopsOpOp2String.put(OpOp2.LESSEQUAL, "<="); HopsOpOp2String.put(OpOp2.LESS, "<"); HopsOpOp2String.put(OpOp2.GREATEREQUAL, ">="); HopsOpOp2String.put(OpOp2.GREATER, ">"); HopsOpOp2String.put(OpOp2.EQUAL, "="); HopsOpOp2String.put(OpOp2.NOTEQUAL, "!="); HopsOpOp2String.put(OpOp2.OR, "|"); HopsOpOp2String.put(OpOp2.AND, "&"); HopsOpOp2String.put(OpOp2.LOG, "log"); HopsOpOp2String.put(OpOp2.LOG_NZ, "log_nz"); HopsOpOp2String.put(OpOp2.POW, "^"); HopsOpOp2String.put(OpOp2.CONCAT, "concat"); HopsOpOp2String.put(OpOp2.INVALID, "?"); HopsOpOp2String.put(OpOp2.QUANTILE, "quantile"); HopsOpOp2String.put(OpOp2.INTERQUANTILE, "interquantile"); HopsOpOp2String.put(OpOp2.IQM, "IQM"); HopsOpOp2String.put(OpOp2.MEDIAN, "median"); HopsOpOp2String.put(OpOp2.CENTRALMOMENT, "cm"); HopsOpOp2String.put(OpOp2.COVARIANCE, "cov"); HopsOpOp2String.put(OpOp2.CBIND, "cbind"); HopsOpOp2String.put(OpOp2.RBIND, "rbind"); HopsOpOp2String.put(OpOp2.SOLVE, "solve"); } protected static final HashMap<Hop.OpOp3, String> HopsOpOp3String; static { HopsOpOp3String = new HashMap<Hop.OpOp3, String>(); HopsOpOp3String.put(OpOp3.QUANTILE, "quantile"); HopsOpOp3String.put(OpOp3.INTERQUANTILE, "interquantile"); HopsOpOp3String.put(OpOp3.CTABLE, "ctable"); HopsOpOp3String.put(OpOp3.CENTRALMOMENT, "cm"); HopsOpOp3String.put(OpOp3.COVARIANCE, "cov"); HopsOpOp3String.put(OpOp3.PLUS_MULT, "+*"); HopsOpOp3String.put(OpOp3.MINUS_MULT, "-*"); } protected static final HashMap<Hop.OpOp4, String> HopsOpOp4String; static { HopsOpOp4String = new HashMap<Hop.OpOp4, String>(); HopsOpOp4String.put(OpOp4.WSLOSS, "wsloss"); HopsOpOp4String.put(OpOp4.WSIGMOID, "wsigmoid"); HopsOpOp4String.put(OpOp4.WCEMM, "wcemm"); HopsOpOp4String.put(OpOp4.WDIVMM, "wdivmm"); HopsOpOp4String.put(OpOp4.WUMM, "wumm"); } protected static final HashMap<Hop.Direction, String> HopsDirection2String; static { HopsDirection2String = new HashMap<Hop.Direction, String>(); HopsDirection2String.put(Direction.RowCol, "RC"); HopsDirection2String.put(Direction.Col, "C"); HopsDirection2String.put(Direction.Row, "R"); } protected static final HashMap<Hop.AggOp, String> HopsAgg2String; static { HopsAgg2String = new HashMap<Hop.AggOp, String>(); HopsAgg2String.put(AggOp.SUM, "+"); HopsAgg2String.put(AggOp.SUM_SQ, "sq+"); HopsAgg2String.put(AggOp.PROD, "*"); HopsAgg2String.put(AggOp.MIN, "min"); HopsAgg2String.put(AggOp.MAX, "max"); HopsAgg2String.put(AggOp.MAXINDEX, "maxindex"); HopsAgg2String.put(AggOp.MININDEX, "minindex"); HopsAgg2String.put(AggOp.TRACE, "trace"); HopsAgg2String.put(AggOp.MEAN, "mean"); HopsAgg2String.put(AggOp.VAR, "var"); } protected static final HashMap<Hop.ReOrgOp, String> HopsTransf2String; static { HopsTransf2String = new HashMap<ReOrgOp, String>(); HopsTransf2String.put(ReOrgOp.TRANSPOSE, "t"); HopsTransf2String.put(ReOrgOp.DIAG, "diag"); HopsTransf2String.put(ReOrgOp.RESHAPE, "rshape"); HopsTransf2String.put(ReOrgOp.SORT, "sort"); } protected static final HashMap<DataOpTypes, String> HopsData2String; static { HopsData2String = new HashMap<Hop.DataOpTypes, String>(); HopsData2String.put(DataOpTypes.PERSISTENTREAD, "PRead"); HopsData2String.put(DataOpTypes.PERSISTENTWRITE, "PWrite"); HopsData2String.put(DataOpTypes.TRANSIENTWRITE, "TWrite"); HopsData2String.put(DataOpTypes.TRANSIENTREAD, "TRead"); } public static OpOp2 getOpOp2ForOuterVectorOperation(String op) { if ("+".equals(op)) return OpOp2.PLUS; else if ("-".equals(op)) return OpOp2.MINUS; else if ("*".equals(op)) return OpOp2.MULT; else if ("/".equals(op)) return OpOp2.DIV; else if ("%%".equals(op)) return OpOp2.MODULUS; else if ("%/%".equals(op)) return OpOp2.INTDIV; else if ("min".equals(op)) return OpOp2.MIN; else if ("max".equals(op)) return OpOp2.MAX; else if ("<=".equals(op)) return OpOp2.LESSEQUAL; else if ("<".equals(op)) return OpOp2.LESS; else if (">=".equals(op)) return OpOp2.GREATEREQUAL; else if (">".equals(op)) return OpOp2.GREATER; else if ("==".equals(op)) return OpOp2.EQUAL; else if ("!=".equals(op)) return OpOp2.NOTEQUAL; else if ("|".equals(op)) return OpOp2.OR; else if ("&".equals(op)) return OpOp2.AND; else if ("log".equals(op)) return OpOp2.LOG; else if ("^".equals(op)) return OpOp2.POW; return null; } ///////////////////////////////////// // methods for dynamic re-compilation ///////////////////////////////////// /** * Indicates if dynamic recompilation is required for this hop. * * @return true if dynamic recompilcation required */ public boolean requiresRecompile() { return _requiresRecompile; } public void setRequiresRecompile() { _requiresRecompile = true; } /** * Update the output size information for this hop. */ public abstract void refreshSizeInformation(); /** * Util function for refreshing scalar rows input parameter. * * @param input high-level operator */ protected void refreshRowsParameterInformation(Hop input) { long size = computeSizeInformation(input); //always set the computed size not just if known (positive) in order to allow //recompile with unknowns to reset sizes (otherwise potential for incorrect results) setDim1(size); } /** * Util function for refreshing scalar cols input parameter. * * @param input high-level operator */ protected void refreshColsParameterInformation(Hop input) { long size = computeSizeInformation(input); //always set the computed size not just if known (positive) in order to allow //recompile with unknowns to reset sizes (otherwise potential for incorrect results) setDim2(size); } public long computeSizeInformation(Hop input) { long ret = -1; try { long tmp = OptimizerUtils.rEvalSimpleLongExpression(input, new HashMap<Long, Long>()); if (tmp != Long.MAX_VALUE) ret = tmp; } catch (Exception ex) { LOG.error("Failed to compute size information.", ex); ret = -1; } return ret; } public void refreshRowsParameterInformation(Hop input, LocalVariableMap vars) { long size = computeSizeInformation(input, vars); //always set the computed size not just if known (positive) in order to allow //recompile with unknowns to reset sizes (otherwise potential for incorrect results) setDim1(size); } public void refreshColsParameterInformation(Hop input, LocalVariableMap vars) { long size = computeSizeInformation(input, vars); //always set the computed size not just if known (positive) in order to allow //recompile with unknowns to reset sizes (otherwise potential for incorrect results) setDim2(size); } public long computeSizeInformation(Hop input, LocalVariableMap vars) { long ret = -1; try { long tmp = OptimizerUtils.rEvalSimpleLongExpression(input, new HashMap<Long, Long>(), vars); if (tmp != Long.MAX_VALUE) ret = tmp; } catch (Exception ex) { LOG.error("Failed to compute size information.", ex); ret = -1; } return ret; } public double computeBoundsInformation(Hop input) { double ret = Double.MAX_VALUE; try { ret = OptimizerUtils.rEvalSimpleDoubleExpression(input, new HashMap<Long, Double>()); } catch (Exception ex) { LOG.error("Failed to compute bounds information.", ex); ret = Double.MAX_VALUE; } return ret; } /** * Computes bound information for sequence if possible, otherwise returns * Double.MAX_VALUE * * @param input high-level operator * @param vars local variable map * @return bounds information */ public double computeBoundsInformation(Hop input, LocalVariableMap vars) { double ret = Double.MAX_VALUE; try { ret = OptimizerUtils.rEvalSimpleDoubleExpression(input, new HashMap<Long, Double>(), vars); } catch (Exception ex) { LOG.error("Failed to compute bounds information.", ex); ret = Double.MAX_VALUE; } return ret; } /** * Compute worst case estimate for size expression based on worst-case * statistics of inputs. Limited set of supported operations in comparison * to refresh rows/cols. * * @param input high-level operator * @param memo memory table * @return worst case estimate for size expression */ protected long computeDimParameterInformation(Hop input, MemoTable memo) { long ret = -1; if (input instanceof UnaryOp) { if (((UnaryOp) input).getOp() == Hop.OpOp1.NROW) { MatrixCharacteristics mc = memo.getAllInputStats(input.getInput().get(0)); if (mc.getRows() > 0) ret = mc.getRows(); } else if (((UnaryOp) input).getOp() == Hop.OpOp1.NCOL) { MatrixCharacteristics mc = memo.getAllInputStats(input.getInput().get(0)); if (mc.getCols() > 0) ret = mc.getCols(); } } else if (input instanceof LiteralOp) { ret = UtilFunctions.parseToLong(input.getName()); } else if (input instanceof BinaryOp) { long dim = rEvalSimpleBinaryLongExpression(input, new HashMap<Long, Long>(), memo); if (dim != Long.MAX_VALUE) //if known ret = dim; } return ret; } protected long rEvalSimpleBinaryLongExpression(Hop root, HashMap<Long, Long> valMemo, MemoTable memo) { //memoization (prevent redundant computation of common subexpr) if (valMemo.containsKey(root.getHopID())) return valMemo.get(root.getHopID()); long ret = Long.MAX_VALUE; if (root instanceof LiteralOp) { long dim = UtilFunctions.parseToLong(root.getName()); if (dim != -1) //if known ret = dim; } else if (root instanceof UnaryOp) { UnaryOp uroot = (UnaryOp) root; long dim = -1; if (uroot.getOp() == Hop.OpOp1.NROW) { MatrixCharacteristics mc = memo.getAllInputStats(uroot.getInput().get(0)); dim = mc.getRows(); } else if (uroot.getOp() == Hop.OpOp1.NCOL) { MatrixCharacteristics mc = memo.getAllInputStats(uroot.getInput().get(0)); dim = mc.getCols(); } if (dim != -1) //if known ret = dim; } else if (root instanceof BinaryOp) { if (OptimizerUtils.ALLOW_WORSTCASE_SIZE_EXPRESSION_EVALUATION) { BinaryOp broot = (BinaryOp) root; long lret = rEvalSimpleBinaryLongExpression(broot.getInput().get(0), valMemo, memo); long rret = rEvalSimpleBinaryLongExpression(broot.getInput().get(1), valMemo, memo); //note: positive and negative values might be valid subexpressions if (lret != Long.MAX_VALUE && rret != Long.MAX_VALUE) //if known { switch (broot.getOp()) { case PLUS: ret = lret + rret; break; case MULT: ret = lret * rret; break; case MIN: ret = Math.min(lret, rret); break; case MAX: ret = Math.max(lret, rret); break; default: ret = Long.MAX_VALUE; } } //exploit min constraint to propagate else if (broot.getOp() == OpOp2.MIN && (lret != Double.MAX_VALUE || rret != Double.MAX_VALUE)) { ret = Math.min(lret, rret); } } } valMemo.put(root.getHopID(), ret); return ret; } /** * Clones the attributes of that and copies it over to this. * * @param that high-level operator * @param withRefs true if with references * @throws CloneNotSupportedException if CloneNotSupportedException occurs */ protected void clone(Hop that, boolean withRefs) throws CloneNotSupportedException { if (withRefs) throw new CloneNotSupportedException("Hops deep copy w/ lops/inputs/parents not supported."); _ID = that._ID; _name = that._name; _dataType = that._dataType; _valueType = that._valueType; _visited = that._visited; _dim1 = that._dim1; _dim2 = that._dim2; _rows_in_block = that._rows_in_block; _cols_in_block = that._cols_in_block; _nnz = that._nnz; _updateType = that._updateType; //no copy of lops (regenerated) _parent = new ArrayList<Hop>(); _input = new ArrayList<Hop>(); _lops = null; _etype = that._etype; _etypeForced = that._etypeForced; _outputMemEstimate = that._outputMemEstimate; _memEstimate = that._memEstimate; _processingMemEstimate = that._processingMemEstimate; _requiresRecompile = that._requiresRecompile; _requiresReblock = that._requiresReblock; _requiresCheckpoint = that._requiresCheckpoint; _outputEmptyBlocks = that._outputEmptyBlocks; _beginLine = that._beginLine; _beginColumn = that._beginColumn; _endLine = that._endLine; _endColumn = that._endColumn; } public abstract Object clone() throws CloneNotSupportedException; public abstract boolean compare(Hop that); /////////////////////////////////////////////////////////////////////////// // store position information for Hops /////////////////////////////////////////////////////////////////////////// public int _beginLine, _beginColumn; public int _endLine, _endColumn; public void setBeginLine(int passed) { _beginLine = passed; } public void setBeginColumn(int passed) { _beginColumn = passed; } public void setEndLine(int passed) { _endLine = passed; } public void setEndColumn(int passed) { _endColumn = passed; } public void setAllPositions(int blp, int bcp, int elp, int ecp) { _beginLine = blp; _beginColumn = bcp; _endLine = elp; _endColumn = ecp; } public int getBeginLine() { return _beginLine; } public int getBeginColumn() { return _beginColumn; } public int getEndLine() { return _endLine; } public int getEndColumn() { return _endColumn; } public String printErrorLocation() { return "ERROR: line " + _beginLine + ", column " + _beginColumn + " -- "; } /** * Sets the linenumbers of this hop to a given lop. * * @param lop low-level operator */ protected void setLineNumbers(Lop lop) { lop.setAllPositions(this.getBeginLine(), this.getBeginColumn(), this.getEndLine(), this.getEndColumn()); } } // end class