Example usage for org.apache.hadoop.conf Configuration getBoolean

Introduction

In this page you can find the example usage for org.apache.hadoop.conf Configuration getBoolean.

Prototype

public boolean getBoolean(String name, boolean defaultValue) 

Source Link

Document

Get the value of the name property as a boolean.

Usage

From source file:ml.shifu.guagua.yarn.util.YarnUtils.java

License:Apache License

/**
 * Populate the environment string map to be added to the environment vars in a remote execution container.
 * //from  w  w w.  j a v a2s  . c  o m
 * @param env
 *            the map of env var values.
 * @param conf
 *            the Configuration to pull values from.
 */
public static void addLocalClasspathToEnv(final Map<String, String> env, final Configuration conf) {
    StringBuilder classPathEnv = new StringBuilder(Environment.CLASSPATH.$());

    // add current folder
    classPathEnv.append(File.pathSeparatorChar).append("./*");

    // add yarn app classpath
    for (String cpEntry : conf.getStrings(YarnConfiguration.YARN_APPLICATION_CLASSPATH,
            YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)) {
        classPathEnv.append(File.pathSeparatorChar).append(cpEntry.trim());
    }

    for (String jar : conf.getStrings(MRJobConfig.MAPREDUCE_APPLICATION_CLASSPATH,
            org.apache.hadoop.util.StringUtils
                    .getStrings(MRJobConfig.DEFAULT_MAPREDUCE_APPLICATION_CLASSPATH))) {
        classPathEnv.append(File.pathSeparatorChar).append(jar.trim());
    }

    // add the runtime classpath needed for tests to work
    if (conf.getBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, false)) {
        classPathEnv.append(File.pathSeparatorChar).append(Environment.CLASSPATH.$());
    }

    // add guagua app jar file
    String path = getFileName(conf.get(GuaguaYarnConstants.GUAGUA_YARN_APP_JAR));
    classPathEnv.append(GuaguaYarnConstants.CURRENT_DIR).append(path).append(File.pathSeparatorChar);

    // Any libraries we may have copied over?
    String libs = conf.get(GuaguaYarnConstants.GUAGUA_YARN_APP_LIB_JAR);
    if (StringUtils.isNotEmpty(libs)) {
        for (String jar : Splitter.on(GuaguaYarnConstants.GUAGUA_APP_LIBS_SEPERATOR).split(libs)) {
            classPathEnv.append(GuaguaYarnConstants.CURRENT_DIR).append(getFileName(jar.trim()))
                    .append(File.pathSeparatorChar);
        }
    }

    // add log4j
    classPathEnv.append(GuaguaYarnConstants.CURRENT_DIR).append(GuaguaYarnConstants.GUAGUA_LOG4J_PROPERTIES)
            .append(File.pathSeparatorChar);

    // add guagua-conf.xml
    classPathEnv.append(GuaguaYarnConstants.CURRENT_DIR).append(GuaguaYarnConstants.GUAGUA_CONF_FILE);

    env.put(Environment.CLASSPATH.toString(), classPathEnv.toString());
}

---

From source file:msc.fall2015.stock.kmeans.hbase.mapreduce.pwd.SWGMap.java

License:Open Source License

public void map(LongWritable blockIndex, Text value, Context context) throws IOException, InterruptedException {
    long startTime = System.nanoTime();
    Configuration conf = context.getConfiguration();
    Counter alignmentCounter = context.getCounter(Constants.RecordCounters.ALIGNMENTS);
    String valString = value.toString();
    String valArgs[] = valString.split(Constants.BREAK);

    long rowBlock = Long.parseLong(valArgs[0]);
    long columnBlock = Long.parseLong(valArgs[1]);
    boolean isDiagonal = Boolean.parseBoolean(valArgs[2]);
    System.out.println("row column" + rowBlock + "  " + columnBlock + "  " + isDiagonal + "  " + valArgs[2]);

    long blockSize = conf.getLong(Constants.BLOCK_SIZE, 1000);
    long noOfSequences = conf.getLong(Constants.NO_OF_SEQUENCES, blockSize * 10);
    long noOfDivisions = conf.getLong(Constants.NO_OF_DIVISIONS, noOfSequences / blockSize);
    boolean weightEnabled = conf.getBoolean(Constants.WEIGHT_ENABLED, false);

    long row = rowBlock * blockSize;
    long column = columnBlock * blockSize;

    long parseStartTime = System.nanoTime();
    FileSystem fs = FileSystem.getLocal(conf);
    // parse the inputFilePart for row
    Path rowPath = new Path(Constants.HDFS_SEQ_FILENAME + "_" + rowBlock);
    FSDataInputStream rowInStream = fs.open(rowPath);
    List<VectorPoint> rowSequences;
    rowSequences = SequenceParser.ParseFile(rowInStream);
    // parse the inputFilePart for column if this is not a diagonal block
    List<VectorPoint> colSequences;
    if (isDiagonal) {
        colSequences = rowSequences;//www. j  a  v a 2 s.c  o m
    } else {
        // parse the inputFilePart for column
        Path colPath = new Path(Constants.HDFS_SEQ_FILENAME + "_" + columnBlock);
        FSDataInputStream colInStream = fs.open(colPath);
        colSequences = SequenceParser.ParseFile(colInStream);
    }
    System.out.println("Parsing time : " + ((System.nanoTime() - parseStartTime) / 1000000) + "ms");

    short[][] alignments = new short[(int) blockSize][(int) blockSize];
    for (int rowIndex = 0; ((rowIndex < blockSize) & ((row + rowIndex) < noOfSequences)); rowIndex++) {
        int columnIndex = 0;
        for (; ((columnIndex < blockSize) & ((column + columnIndex) < noOfSequences)); columnIndex++) {
            double alignment = 0;
            if (weightEnabled) {
                alignment = rowSequences.get(rowIndex).weight(colSequences.get(columnIndex));
            } else {
                alignment = rowSequences.get(rowIndex).corr(colSequences.get(columnIndex));
            }

            // Get the identity and make it percent identity
            short scaledScore = (short) (alignment * Short.MAX_VALUE);
            alignments[rowIndex][columnIndex] = scaledScore;
        }
        alignmentCounter.increment(columnIndex);
    }

    SWGWritable dataWritable = new SWGWritable(rowBlock, columnBlock, blockSize, false);
    dataWritable.setAlignments(alignments);
    context.write(new LongWritable(rowBlock), dataWritable);

    if (!isDiagonal) {
        // Create the transpose matrix of (rowBlock, colBlock) block to fill the
        // (colBlock, rowBlock) block.
        SWGWritable inverseDataWritable = new SWGWritable(columnBlock, rowBlock, blockSize, true);
        inverseDataWritable.setAlignments(alignments);
        context.write(new LongWritable(columnBlock), inverseDataWritable);
    }
    System.out.println("Map time : " + ((System.nanoTime() - startTime) / 1000000) + "ms");
}

---

From source file:msc.fall2015.stock.kmeans.hbase.mapreduce.pwd.SWGReduce.java

License:Open Source License

public void reduce(LongWritable key, Iterable<SWGWritable> values, Context context) throws IOException {
    long startTime = System.nanoTime();
    Configuration conf = context.getConfiguration();

    long blockSize = conf.getLong(Constants.BLOCK_SIZE, 1000);
    long noOfSequences = conf.getLong(Constants.NO_OF_SEQUENCES, blockSize * 10);
    long noOfDivisions = conf.getLong(Constants.NO_OF_DIVISIONS, noOfSequences / blockSize);
    boolean weightEnabled = conf.getBoolean(Constants.WEIGHT_ENABLED, false);

    // to handle the edge blocks with lesser number of sequences
    int row = (int) (key.get() * blockSize);
    int currentRowBlockSize = (int) blockSize;
    if ((row + blockSize) > (noOfSequences)) {
        currentRowBlockSize = (int) (noOfSequences - row);
    }//from ww  w . j a v  a2 s. c  o m

    // TODO do this in the byte level
    short[][] alignments = new short[(int) currentRowBlockSize][(int) noOfSequences];

    for (SWGWritable alignmentWritable : values) {
        System.out.println("key " + key.get() + " col " + alignmentWritable.getColumnBlock() + " row "
                + alignmentWritable.getRowBlock() + " blocksize " + blockSize);
        DataInput in = alignmentWritable.getDataInput();
        int column = (int) (alignmentWritable.getColumnBlock() * blockSize);

        // to handle the edge blocks with lesser number of sequences
        int currentColumnBlockSize = (int) blockSize;
        if ((column + blockSize) > (noOfSequences)) {
            currentColumnBlockSize = (int) (noOfSequences - column);
        }

        for (int i = 0; i < currentRowBlockSize; i++) {
            // byte[] b = new byte[currentBlockSize /* * 2*/];
            //            System.out.println("row block "+i+"  currentBlockSize"+currentRowBlockSize);
            for (int j = 0; j < currentColumnBlockSize; j++) {
                short readShort = in.readShort();
                //               System.out.print(readShort+" ");
                alignments[i][column + j] = readShort;
            }
            //            System.out.println();
            //TODO try to do the above using byte[] copy 
            // in.readFully(b);
            // System.out.println(new String(b));
            // System.arraycopy(b, 0, alignments[i], (column /* * 2*/),
            // currentBlockSize);
        }
    }

    // retrieve the output dir
    String outDir = context.getConfiguration().get("mapred.output.dir");

    FileSystem fs = FileSystem.get(conf);
    // out dir is created in the main driver.
    String childName = "rowblock_cor_" + key.get() + "_blockSize_" + blockSize;
    if (weightEnabled) {
        childName = "rowblock_weight_" + key.get() + "_blockSize_" + blockSize;
    }
    Path outFilePart = new Path(outDir, childName);
    writeOutFile(alignments, fs, outFilePart);
    System.out.println("Reduce Processing Time: " + ((System.nanoTime() - startTime) / 1000000));
}

---

From source file:mvm.rya.accumulo.mr.RyaOutputFormat.java

License:Apache License

private static GeoIndexer getGeoIndexer(Configuration conf) throws IOException {
    if (!conf.getBoolean(ENABLE_GEO, true)) {
        return new NullGeoIndexer();
    }//from ww w.  j a  v a 2 s  .  c o  m

    GeoMesaGeoIndexer geo = new GeoMesaGeoIndexer();
    geo.setConf(conf);
    return geo;

}

---

From source file:mvm.rya.accumulo.mr.RyaOutputFormat.java

License:Apache License

private static FreeTextIndexer getFreeTextIndexer(Configuration conf) throws IOException {
    if (!conf.getBoolean(ENABLE_FREETEXT, true)) {
        return new NullFreeTextIndexer();
    }// w  w  w  .j  a  v  a  2 s  .co m

    AccumuloFreeTextIndexer freeText = new AccumuloFreeTextIndexer();
    freeText.setConf(conf);
    return freeText;

}

---

From source file:mvm.rya.accumulo.mr.RyaOutputFormat.java

License:Apache License

private static TemporalIndexer getTemporalIndexer(Configuration conf) throws IOException {
    if (!conf.getBoolean(ENABLE_TEMPORAL, true)) {
        return new NullTemporalIndexer();
    }//from   w  w w.  j a  v  a2s  . co  m
    AccumuloTemporalIndexer temporal = new AccumuloTemporalIndexer();
    temporal.setConf(conf);
    return temporal;
}

---

From source file:mvm.rya.indexing.accumulo.ConfigUtils.java

License:Apache License

public static boolean isDisplayQueryPlan(Configuration conf) {
    return conf.getBoolean(DISPLAY_QUERY_PLAN, false);
}

---

From source file:mvm.rya.indexing.accumulo.ConfigUtils.java

License:Apache License

public static boolean useMockInstance(Configuration conf) {
    return conf.getBoolean(USE_MOCK_INSTANCE, false);
}

---

From source file:mvm.rya.indexing.accumulo.ConfigUtils.java

License:Apache License

public static boolean getUseGeo(Configuration conf) {
    return conf.getBoolean(USE_GEO, false);
}

---

From source file:mvm.rya.indexing.accumulo.ConfigUtils.java

License:Apache License

public static boolean getUseFreeText(Configuration conf) {
    return conf.getBoolean(USE_FREETEXT, false);
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• 41
• 42
• 43
• 44
• 45
• 46
• 47
• 48
• 49
• 50
• »