Example usage for org.apache.hadoop.conf Configuration get

List of usage examples for org.apache.hadoop.conf Configuration get

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.conf.*
  5. Configuration
  6. get

Introduction

In this page you can find the example usage for org.apache.hadoop.conf Configuration get.

Prototype

public String get(String name, String defaultValue)

Source Link

Document

Get the value of the name.

Usage

From source file:cascading.tap.hadoop.Dfs.java

License:Open Source License

@Override
protected FileSystem getDefaultFileSystem(Configuration configuration) {
    String name = configuration.get("fs.default.name", "hdfs://localhost:5001/");

    if (name.equals("local") || name.matches(".*://.*") && !name.startsWith("hdfs://"))
        name = "hdfs://localhost:5001/";
    else if (name.indexOf('/') == -1)
        name = "hdfs://" + name;

    try {//www  .  j  a  va2  s.  c  om
        return FileSystem.get(URI.create(name), configuration);
    } catch (IOException exception) {
        throw new TapException("unable to get handle to get filesystem for: " + name, exception);
    }
}

From source file:cascading.tap.hadoop.Hfs.java

License:Open Source License

protected static String getLocalModeScheme(Configuration conf, String defaultValue) {
    return conf.get(HfsProps.LOCAL_MODE_SCHEME, defaultValue);
}

From source file:cascading.tap.hadoop.Hfs.java

License:Open Source License

protected static boolean getUseCombinedInput(Configuration conf) {
    String platform = conf.get("cascading.flow.platform", "");
    boolean combineEnabled = conf.getBoolean("cascading.hadoop.hfs.combine.files", false);

    // only supported by these platforms
    if (platform.equals("hadoop") || platform.equals("hadoop2-mr1"))
        return combineEnabled;

    if (combineEnabled && !Boolean.getBoolean("cascading.hadoop.hfs.combine.files.warned")) {
        LOG.warn(// w  w  w  . j  a va2s.  c o m
                "'cascading.hadoop.hfs.combine.files' has been set to true, but is unsupported by this platform: {}, will be ignored to prevent failures",
                platform);
        System.setProperty("cascading.hadoop.hfs.combine.files.warned", "true");
    }

    return false;
}

From source file:cascading.tap.hadoop.S3HttpFileSystem.java

License:Open Source License

@Override
public void initialize(URI uri, Configuration conf) throws IOException {
    LOG.warn(/*from  w  w w. j ava 2s  .  co m*/
            "the S3HttpFileSystem (s3tp://) is deprecated, please use the Hadoop NativeS3NativeFileSystem (s3n://)");

    setConf(conf);

    String key = conf.get("fs.s3tp.awsAccessKeyId", System.getProperty("fs.s3tp.awsAccessKeyId"));
    String secret = conf.get("fs.s3tp.awsSecretAccessKey", System.getProperty("fs.s3tp.awsSecretAccessKey"));

    this.s3Service = S3Util.getS3Service(uri, key, secret);
    this.s3Bucket = S3Util.getS3Bucket(uri);
    this.uri = URI.create(uri.getScheme() + "://" + uri.getAuthority());
}

From source file:cascading.tap.hadoop.util.Hadoop18TapUtil.java

License:Open Source License

/**
 * should only be called if not in a Flow
 *
 * @param conf/*from   ww w  .j  a  v  a 2s . c  o m*/
 * @throws IOException
 */
public static void setupJob(Configuration conf) throws IOException {
    Path outputPath = FileOutputFormat.getOutputPath(asJobConfInstance(conf));

    if (outputPath == null)
        return;

    if (getFSSafe(conf, outputPath) == null)
        return;

    String taskID = conf.get("mapred.task.id", conf.get("mapreduce.task.id"));

    if (taskID == null) // need to stuff a fake id
    {
        String mapper = conf.getBoolean("mapred.task.is.map", conf.getBoolean("mapreduce.task.is.map", true))
                ? "m"
                : "r";
        String value = String.format("attempt_%012d_0000_%s_000000_0",
                (int) Math.rint(System.currentTimeMillis()), mapper);
        conf.set("mapred.task.id", value);
        conf.set("mapreduce.task.id", value);
    }

    makeTempPath(conf);

    if (writeDirectlyToWorkingPath(conf, outputPath)) {
        LOG.info("writing directly to output path: {}", outputPath);
        setWorkOutputPath(conf, outputPath);
        return;
    }

    // "mapred.work.output.dir"
    Path taskOutputPath = getTaskOutputPath(conf);
    setWorkOutputPath(conf, taskOutputPath);
}

From source file:cascading.tap.hadoop.util.Hadoop18TapUtil.java

License:Open Source License

private static Path getTaskOutputPath(Configuration conf) {
    String taskId = conf.get("mapred.task.id", conf.get("mapreduce.task.id"));

    Path p = new Path(FileOutputFormat.getOutputPath(asJobConfInstance(conf)),
            TEMPORARY_PATH + Path.SEPARATOR + "_" + taskId);

    try {/*from w  w  w .  java2 s  . com*/
        FileSystem fs = p.getFileSystem(conf);
        return p.makeQualified(fs);
    } catch (IOException ie) {
        return p;
    }
}

From source file:cascading.tap.hadoop.util.Hadoop18TapUtil.java

License:Open Source License

private static void moveTaskOutputs(Configuration conf, FileSystem fs, Path jobOutputDir, Path taskOutput)
        throws IOException {
    String taskId = conf.get("mapred.task.id", conf.get("mapreduce.task.id"));

    if (fs.isFile(taskOutput)) {
        Path finalOutputPath = getFinalPath(jobOutputDir, taskOutput, getTaskOutputPath(conf));
        if (!fs.rename(taskOutput, finalOutputPath)) {
            if (!fs.delete(finalOutputPath, true))
                throw new IOException("Failed to delete earlier output of task: " + taskId);

            if (!fs.rename(taskOutput, finalOutputPath))
                throw new IOException("Failed to save output of task: " + taskId);
        }//w w w .ja  va 2s .  co m

        LOG.debug("Moved {} to {}", taskOutput, finalOutputPath);
    } else if (fs.getFileStatus(taskOutput).isDir()) {
        FileStatus[] paths = fs.listStatus(taskOutput);
        Path finalOutputPath = getFinalPath(jobOutputDir, taskOutput, getTaskOutputPath(conf));
        fs.mkdirs(finalOutputPath);
        if (paths != null) {
            for (FileStatus path : paths)
                moveTaskOutputs(conf, fs, jobOutputDir, path.getPath());
        }
    }
}

From source file:cn.easyhbase.client.hbase.HBaseAsyncOperationFactory.java

License:Apache License

public static HBaseAsyncOperation create(Configuration configuration) throws IOException {
    boolean enableAsyncMethod = configuration.getBoolean(ENABLE_ASYNC_METHOD, DEFAULT_ENABLE_ASYNC_METHOD);
    LOGGER.info("hbase.client.async.enable: " + enableAsyncMethod);
    if (!enableAsyncMethod) {
        return DisabledHBaseAsyncOperation.INSTANCE;
    }//from   w  w  w. j  a  va  2 s. c o  m

    int queueSize = configuration.getInt(ASYNC_IN_QUEUE_SIZE, DEFAULT_ASYNC_IN_QUEUE_SIZE);

    if (configuration.get(ASYNC_PERIODIC_FLUSH_TIME, null) == null) {
        configuration.setInt(ASYNC_PERIODIC_FLUSH_TIME, DEFAULT_ASYNC_PERIODIC_FLUSH_TIME);
    }

    if (configuration.get(ASYNC_RETRY_COUNT, null) == null) {
        configuration.setInt(ASYNC_RETRY_COUNT, DEFAULT_ASYNC_RETRY_COUNT);
    }

    return new HBaseAsyncTemplate(configuration, queueSize);
}

From source file:cn.easyhbase.client.hbase.HBaseAsyncOperationFactory.java

License:Apache License

public static HBaseAsyncOperation create(Connection connection, Configuration configuration)
        throws IOException {
    boolean enableAsyncMethod = configuration.getBoolean(ENABLE_ASYNC_METHOD, DEFAULT_ENABLE_ASYNC_METHOD);
    if (!enableAsyncMethod) {
        return DisabledHBaseAsyncOperation.INSTANCE;
    }//from   w w  w . j a  v a  2  s  .  c  om

    int queueSize = configuration.getInt(ASYNC_IN_QUEUE_SIZE, DEFAULT_ASYNC_IN_QUEUE_SIZE);

    if (configuration.get(ASYNC_PERIODIC_FLUSH_TIME, null) == null) {
        configuration.setInt(ASYNC_PERIODIC_FLUSH_TIME, DEFAULT_ASYNC_PERIODIC_FLUSH_TIME);
    }

    if (configuration.get(ASYNC_RETRY_COUNT, null) == null) {
        configuration.setInt(ASYNC_RETRY_COUNT, DEFAULT_ASYNC_RETRY_COUNT);
    }

    return new HBaseAsyncTemplate(connection, configuration, queueSize);
}

From source file:cn.edu.hfut.dmic.webcollector.fetcher.FetcherReducer.java

@Override
public void run(Context context) throws IOException, InterruptedException {

    Configuration conf = context.getConfiguration();
    String requesterClass = conf.get("requester.class", CommonRequester.class.getName());
    try {/*from   w ww  .  j a v  a  2s.c om*/
        requester = Plugin.<Requester>createPlugin(requesterClass);
    } catch (Exception ex) {
        LOG.info("Exception when initializing requester " + requesterClass, ex);
        return;
    }
    String visitorClass = conf.get("visitor.class");
    if (visitorClass == null) {
        LOG.info("Must specify a visitor!");
        return;
    }
    try {
        visitor = Plugin.<Visitor>createPlugin(visitorClass);
    } catch (Exception ex) {
        LOG.info("Exception when initializing visitor " + visitorClass, ex);
        return;
    }

    try {
        running = true;

        lastRequestStart = new AtomicLong(System.currentTimeMillis());

        activeThreads = new AtomicInteger(0);
        spinWaiting = new AtomicInteger(0);
        fetchQueue = new FetchQueue();
        feeder = new QueueFeeder(fetchQueue, context, 1000);
        feeder.start();

        FetcherThread[] fetcherThreads = new FetcherThread[threads];
        for (int i = 0; i < threads; i++) {
            fetcherThreads[i] = new FetcherThread(context);
            fetcherThreads[i].start();
        }

        do {
            try {
                Thread.sleep(1000);
            } catch (InterruptedException ex) {
            }
            LOG.info("-activeThreads=" + activeThreads.get() + ", spinWaiting=" + spinWaiting.get()
                    + ", fetchQueue.size=" + fetchQueue.getSize());

            if (!feeder.isAlive() && fetchQueue.getSize() < 5) {
                fetchQueue.dump();
            }

            if ((System.currentTimeMillis() - lastRequestStart.get()) > Config.THREAD_KILLER) {
                LOG.info("Aborting with " + activeThreads + " hung threads.");
                break;
            }

        } while (activeThreads.get() > 0 && running);
        running = false;
        long waitThreadEndStartTime = System.currentTimeMillis();
        if (activeThreads.get() > 0) {
            LOG.info("wait for activeThreads to end");
        }
        /*?*/
        while (activeThreads.get() > 0) {
            LOG.info("-activeThreads=" + activeThreads.get());
            try {
                Thread.sleep(500);
            } catch (Exception ex) {
            }
            if (System.currentTimeMillis() - waitThreadEndStartTime > Config.WAIT_THREAD_END_TIME) {
                LOG.info("kill threads");
                for (int i = 0; i < fetcherThreads.length; i++) {
                    if (fetcherThreads[i].isAlive()) {
                        try {
                            fetcherThreads[i].stop();
                            LOG.info("kill thread " + i);
                        } catch (Exception ex) {
                            LOG.info("Exception", ex);
                        }
                    }
                }
                break;
            }
        }
        LOG.info("clear all activeThread");
        feeder.stopFeeder();
        fetchQueue.clear();
    } finally {

    }
}