Example usage for org.apache.hadoop.conf Configuration setLong

Introduction

In this page you can find the example usage for org.apache.hadoop.conf Configuration setLong.

Prototype

public void setLong(String name, long value) 

Source Link

Document

Set the value of the name property to a long.

Usage

From source file:com.asakusafw.runtime.stage.inprocess.InProcessStageConfiguratorTest.java

License:Apache License

/**
 * simple case./*  ww w .  j  a  va 2 s. co m*/
 * @throws Exception if failed
 */
@Test
public void simple() throws Exception {
    Job job = newJob();
    Configuration conf = job.getConfiguration();

    conf.setLong(KEY_LIMIT, 100);
    StageInputFormat.setSplitCombinerClass(job, IdentitySplitCombiner.class);

    new Mock(100).configure(job);
    assertThat(conf.get(StageConstants.PROP_JOB_RUNNER), is(notNullValue()));
    assertThat(StageResourceDriver.getAccessMode(job), is(StageResourceDriver.AccessMode.DIRECT));
    assertThat(StageInputFormat.getSplitCombinerClass(job), is((Object) ExtremeSplitCombiner.class));
}

---

From source file:com.asakusafw.runtime.stage.inprocess.InProcessStageConfiguratorTest.java

License:Apache License

/**
 * input size is exceeded.//from   ww  w. j  a v a  2  s. co m
 * @throws Exception if failed
 */
@Test
public void large() throws Exception {
    Job job = newJob();
    Configuration conf = job.getConfiguration();

    conf.setLong(KEY_LIMIT, 100);
    new Mock(101).configure(job);
    assertThat(conf.get(StageConstants.PROP_JOB_RUNNER), is(nullValue()));
}

---

From source file:com.asakusafw.runtime.stage.inprocess.InProcessStageConfiguratorTest.java

License:Apache License

/**
 * activate alternative properties./*from w  w  w. j a  v a 2 s.  c  om*/
 * @throws Exception if failed
 */
@Test
public void activate_properties() throws Exception {
    Job job = newJob();
    Configuration conf = job.getConfiguration();

    conf.setLong(KEY_LIMIT, 100);
    conf.set(KEY_PREFIX_REPLACE + "com.example.testing", "YES!");
    new Mock(100).configure(job);
    assertThat(conf.get("com.example.testing"), is("YES!"));
}

---

From source file:com.asakusafw.runtime.stage.inprocess.InProcessStageConfiguratorTest.java

License:Apache License

/**
 * activate alternative properties.//  w ww  .  j  a v  a2 s .  co m
 * @throws Exception if failed
 */
@Test
public void activate_properties_skip() throws Exception {
    Job job = newJob();
    Configuration conf = job.getConfiguration();

    conf.setLong(KEY_LIMIT, 100);
    conf.set(KEY_PREFIX_REPLACE + "com.example.testing", "YES!");
    new Mock(1000).configure(job);
    assertThat(conf.get("com.example.testing"), is(not("YES!")));
}

---

From source file:com.asakusafw.runtime.stage.resource.StageResourceDriver.java

License:Apache License

/**
 * Adds a resource path into the target job object.
 * @param job the target job//from  w  ww. j a v a2s.  c  om
 * @param resourcePath the resource path expression (this must be accessible from task execution nodes)
 * @param resourceName the resource name
 * @throws IOException if failed to detect resources on the path
 * @throws IllegalArgumentException if some parameters are {@code null}
 */
public static void add(Job job, String resourcePath, String resourceName) throws IOException {
    if (job == null) {
        throw new IllegalArgumentException("job must not be null"); //$NON-NLS-1$
    }
    if (resourcePath == null) {
        throw new IllegalArgumentException("resourcePath must not be null"); //$NON-NLS-1$
    }
    if (resourceName == null) {
        throw new IllegalArgumentException("resourceName must not be null"); //$NON-NLS-1$
    }
    Configuration conf = job.getConfiguration();
    List<FileStatus> list = TemporaryStorage.listStatus(conf, new Path(resourcePath));
    if (list.isEmpty()) {
        throw new IOException(MessageFormat.format("Resource not found: {0}", resourcePath));
    }
    List<String> localNames = restoreStrings(conf, getLocalCacheNameKey(resourceName));
    List<String> remotePaths = restoreStrings(conf, getRemotePathKey(resourceName));
    long size = conf.getLong(KEY_SIZE, 0L);
    int index = localNames.size();
    for (FileStatus status : list) {
        String name = String.format("%s-%04d", resourceName, index++); //$NON-NLS-1$
        StringBuilder buf = new StringBuilder();
        buf.append(status.getPath().toString());
        buf.append('#');
        buf.append(name);
        String cachePath = buf.toString();

        remotePaths.add(status.getPath().toString());
        localNames.add(name);
        try {
            URI uri = new URI(cachePath);
            DistributedCache.addCacheFile(uri, conf);
        } catch (URISyntaxException e) {
            throw new IllegalStateException(e);
        }
        size += status.getLen();
    }
    conf.setStrings(getLocalCacheNameKey(resourceName), localNames.toArray(new String[localNames.size()]));
    conf.setStrings(getRemotePathKey(resourceName), remotePaths.toArray(new String[remotePaths.size()]));
    conf.setLong(KEY_SIZE, size);
    if (JobCompatibility.isLocalMode(job)) {
        if (LOG.isDebugEnabled()) {
            LOG.debug("symlinks for distributed cache will not be created in standalone mode"); //$NON-NLS-1$
        }
    } else {
        DistributedCache.createSymlink(conf);
    }
}

---

From source file:com.asakusafw.thundergate.runtime.cache.mapreduce.Invalidation.java

License:Apache License

/**
 * Initializes invalidation timestamp./*  w  w w.  j  a v a  2 s . c  o m*/
 * @param configuration the target configuration
 * @param tableName the target table name
 */
public static void setupInvalidationTimestamp(Configuration configuration, String tableName) {
    long timestamp = getTimestamp(configuration);
    if (timestamp > 0L && isTarget(configuration, tableName)) {
        LOG.info(MessageFormat.format("enabling ThunderGate cache invalidation: {0} until {1}", tableName,
                configuration.get(KEY_INVALIDATION_TIMESTAMP)));
        configuration.setLong(KEY_INTERNAL_TIMESTAMP, timestamp);
    }
}

---

From source file:com.blackberry.logdriver.mapreduce.avro.AvroBlockInputFormat.java

License:Apache License

/**
 * Creates a new AvroBlockRecordReader./*w  w w.  ja  v a2s .c  o m*/
 * 
 * Increases there default value mapreduce.job.max.split.locations to 100000,
 * if it's not already set.
 * 
 * Also sets mapred.max.split.size to the default block size for the root
 * directory ("/"), if it's not already set.
 * 
 * @param split
 *          The InputSplit.
 * @param context
 *          The TaskAttemptContext.
 * @return A new AvroBlockRecordReader.
 * @throws IOException
 *           If there is an I/O error.
 */
@SuppressWarnings("deprecation")
@Override
public RecordReader<AvroFileHeader, BytesWritable> createRecordReader(InputSplit split,
        TaskAttemptContext context) throws IOException {
    Configuration conf = context.getConfiguration();

    // Ensure we have sensible defaults for how we build blocks.
    if (conf.get("mapreduce.job.max.split.locations") == null) {
        conf.setLong("mapreduce.job.max.split.locations", MAX_SPLIT_LOCATIONS);
    }
    if (conf.get("mapred.max.split.size") == null) {
        // Try to set the split size to the default block size. In case of
        // failure, we'll use this 128MB default.
        long blockSize = 128 * 1024 * 1024; // 128MB
        try {
            blockSize = FileSystem.get(conf).getDefaultBlockSize();
        } catch (IOException e) {
            LOG.error("Error getting filesystem to get get default block size (this does not bode well).");
        }
        conf.setLong("mapred.max.split.size", blockSize);
    }

    return new AvroBlockRecordReader();
}

---

From source file:com.blackberry.logdriver.mapreduce.boom.BoomInputFormat.java

License:Apache License

@SuppressWarnings("deprecation")
public List<InputSplit> getSplits(JobContext context) throws IOException {
    Configuration conf = context.getConfiguration();
    // Ensure we have sensible defaults for how we build blocks.
    if (conf.get("mapreduce.job.max.split.locations") == null) {
        conf.setLong("mapreduce.job.max.split.locations", MAX_SPLIT_LOCATIONS);
    }// w w w. j ava 2  s .c o m
    if (conf.get("mapred.max.split.size") == null) {
        // Try to set the split size to the default block size. In case of
        // failure, we'll use this 128MB default.
        long blockSize = 128 * 1024 * 1024; // 128MB
        try {
            blockSize = FileSystem.get(conf).getDefaultBlockSize();
        } catch (IOException e) {
            LOG.error("Error getting filesystem to get get default block size (this does not bode well).");
        }
        conf.setLong("mapred.max.split.size", blockSize);
    }
    for (String key : new String[] { "mapreduce.job.max.split.locations", "mapred.max.split.size" }) {
        LOG.info("{} = {}", key, context.getConfiguration().get(key));
    }

    return super.getSplits(context);
}

---

From source file:com.blackberry.logdriver.util.CatByTime.java

License:Apache License

@Override
public int run(String[] args) throws Exception {
    Configuration conf = getConf(); // Configuration processed by ToolRunner
    List<String> searchArgs = new ArrayList<String>();

    if (args.length < 6) {
        System.out.println("Usage: " + this.getClass().getSimpleName() + " DC SVC COMP START END OUT");
    }/*from w  w  w.  j  a v a 2 s  .  c  o m*/
    String dcNumber = args[0];
    String service = args[1];
    String component = args[2];
    long startTime = Long.parseLong(args[3]);
    long endTime = Long.parseLong(args[4]);
    String output = args[5];

    // Add the start and end time to the configuration
    conf.setLong("logdriver.search.start.time", startTime);
    conf.setLong("logdriver.search.end.time", endTime);

    // Get paths
    FileManager fm = new FileManager(conf);
    List<PathInfo> paths = fm.getPathInfo(dcNumber, service, component, startTime, endTime);

    if (paths.isEmpty()) {
        System.err.println("No logs found for the given component(s) and time range.");
        return 1;
    }

    int retval = 99;
    try {
        // Lock, then get the real paths
        fm.acquireReadLocks(paths);
        for (PathInfo pi : paths) {
            LOG.info("Adding path: {}", pi.getFullPath());
            searchArgs.addAll(fm.getInputPaths(pi));
        }

        // The last arg is output directory
        searchArgs.add(output);

        // Now run Cat
        LOG.info("Sending args to Cat: {}", searchArgs);
        retval = ToolRunner.run(conf, new Cat(), searchArgs.toArray(new String[0]));
    } finally {
        fm.releaseReadLocks(paths);
    }

    return retval;
}

---

From source file:com.blackberry.logdriver.util.FastSearchByTime.java

License:Apache License

@Override
public int run(String[] args) throws Exception {
    Configuration conf = getConf(); // Configuration processed by ToolRunner
    List<String> searchArgs = new ArrayList<String>();

    String searchString = args[0];
    String dcNumber = args[1];/*  w w  w  .ja  v a2s  . co m*/
    String service = args[2];
    String component = args[3];
    long startTime = Long.parseLong(args[4]);
    long endTime = Long.parseLong(args[5]);
    String output = args[6];

    // Add the start and end time to the configuration
    conf.setLong("logdriver.search.start.time", startTime);
    conf.setLong("logdriver.search.end.time", endTime);

    // the first arg is the search string
    searchArgs.add(searchString);

    // Get paths
    FileManager fm = new FileManager(conf);
    List<PathInfo> paths = fm.getPathInfo(dcNumber, service, component, startTime, endTime);

    if (paths.isEmpty()) {
        System.err.println("No logs found for the given component(s) and time range.");
        return 1;
    }

    int retval = 99;
    try {
        // Lock, then get the real paths
        fm.acquireReadLocks(paths);
        for (PathInfo pi : paths) {
            LOG.info("Adding path: {}", pi.getFullPath());
            searchArgs.addAll(fm.getInputPaths(pi));
        }

        // The last arg is output directory
        searchArgs.add(output);

        // Now run Search
        LOG.info("Sending args to FastSearch: {}", searchArgs);
        retval = ToolRunner.run(conf, new FastSearch(), searchArgs.toArray(new String[0]));
    } finally {
        fm.releaseReadLocks(paths);
    }

    return retval;
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• »