Example usage for org.apache.hadoop.conf Configuration set

List of usage examples for org.apache.hadoop.conf Configuration set

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.conf.*
  5. Configuration
  6. set

Introduction

In this page you can find the example usage for org.apache.hadoop.conf Configuration set.

Prototype

public void set(String name, String value)

Source Link

Document

Set the value of the name property.

Usage

From source file:alluxio.checker.MapReduceIntegrationChecker.java

License:Apache License

/**
 * Implements MapReduce with Alluxio integration checker.
 *
 * @return 0 for success, 2 for unable to find Alluxio classes, 1 otherwise
 *//*from  w w  w .  j av a 2  s.co m*/
private int run(String[] args) throws Exception {
    Configuration conf = new Configuration();
    String numMaps = new GenericOptionsParser(conf, args).getRemainingArgs()[0];
    conf.set(MRJobConfig.NUM_MAPS, numMaps);
    createHdfsFilesystem(conf);

    Job job = Job.getInstance(conf, "MapReduceIntegrationChecker");
    job.setJarByClass(MapReduceIntegrationChecker.class);
    job.setMapperClass(CheckerMapper.class);
    job.setCombinerClass(CheckerReducer.class);
    job.setReducerClass(CheckerReducer.class);
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(Text.class);
    job.setInputFormatClass(EmptyInputFormat.class);
    FileOutputFormat.setOutputPath(job, mOutputFilePath);

    try {
        if (!job.waitForCompletion(true)) {
            return 1;
        }
        Status resultStatus = generateReport();
        return resultStatus.equals(Status.SUCCESS) ? 0
                : (resultStatus.equals(Status.FAIL_TO_FIND_CLASS) ? 2 : 1);
    } finally {
        if (mFileSystem.exists(mOutputFilePath)) {
            mFileSystem.delete(mOutputFilePath, true);
        }
        mFileSystem.close();
    }
}

From source file:alluxio.client.hadoop.FileSystemAclIntegrationTest.java

License:Apache License

@BeforeClass
public static void beforeClass() throws Exception {
    Configuration conf = new Configuration();
    conf.set("fs.alluxio.impl", FileSystem.class.getName());
    conf = HadoopConfigurationUtils.mergeAlluxioConfiguration(conf, ServerConfiguration.global());

    URI uri = URI.create(sLocalAlluxioClusterResource.get().getMasterURI());

    sTFS = org.apache.hadoop.fs.FileSystem.get(uri,
            HadoopConfigurationUtils.mergeAlluxioConfiguration(conf, ServerConfiguration.global()));
    sUfsRoot = ServerConfiguration.get(PropertyKey.MASTER_MOUNT_TABLE_ROOT_UFS);
    sUfs = UnderFileSystem.Factory.createForRoot(ServerConfiguration.global());
}

From source file:alluxio.client.hadoop.FileSystemBlockLocationIntegrationTest.java

License:Apache License

@BeforeClass
public static void beforeClass() throws Exception {
    Configuration conf = new Configuration();
    conf.set("fs.alluxio.impl", FileSystem.class.getName());

    alluxio.client.file.FileSystem alluxioFS = sLocalAlluxioClusterResource.get().getClient();
    FileSystemTestUtils.createByteFile(alluxioFS, "/testFile1", WritePType.CACHE_THROUGH, FILE_LEN);

    URI uri = URI.create(sLocalAlluxioClusterResource.get().getMasterURI());
    sTFS = org.apache.hadoop.fs.FileSystem.get(uri,
            HadoopConfigurationUtils.mergeAlluxioConfiguration(conf, ServerConfiguration.global()));
}

From source file:alluxio.client.hadoop.FileSystemIntegrationTest.java

License:Apache License

@BeforeClass
public static void beforeClass() throws Exception {
    Configuration conf = new Configuration();
    conf.set("fs.alluxio.impl", FileSystem.class.getName());

    URI uri = URI.create(sLocalAlluxioClusterResource.get().getMasterURI());

    sTFS = org.apache.hadoop.fs.FileSystem.get(uri,
            HadoopConfigurationUtils.mergeAlluxioConfiguration(conf, ServerConfiguration.global()));
}

From source file:alluxio.client.hadoop.FileSystemRenameIntegrationTest.java

License:Apache License

@BeforeClass
public static void beforeClass() throws Exception {
    Configuration conf = new Configuration();
    conf.set("fs.alluxio.impl", FileSystem.class.getName());

    URI uri = URI.create(sLocalAlluxioClusterResource.get().getMasterURI());

    sTFS = org.apache.hadoop.fs.FileSystem.get(uri,
            HadoopConfigurationUtils.mergeAlluxioConfiguration(conf, ServerConfiguration.global()));
    sUfsRoot = ServerConfiguration.get(PropertyKey.MASTER_MOUNT_TABLE_ROOT_UFS);
    sUfs = UnderFileSystem.Factory.createForRoot(ServerConfiguration.global());
}

From source file:alluxio.client.hadoop.FileSystemStatisticsTest.java

License:Apache License

@BeforeClass
public static void beforeClass() throws Exception {
    Configuration conf = new Configuration();
    conf.set("fs.alluxio.impl", FileSystem.class.getName());

    alluxio.client.file.FileSystem alluxioFS = sLocalAlluxioClusterResource.get().getClient();
    FileSystemTestUtils.createByteFile(alluxioFS, "/testFile-read", WritePType.CACHE_THROUGH, FILE_LEN);

    URI uri = URI.create(sLocalAlluxioClusterResource.get().getMasterURI());
    sTFS = org.apache.hadoop.fs.FileSystem.get(uri,
            HadoopConfigurationUtils.mergeAlluxioConfiguration(conf, ServerConfiguration.global()));
    sStatistics = org.apache.hadoop.fs.FileSystem.getStatistics(uri.getScheme(), sTFS.getClass());
}

From source file:alluxio.client.hadoop.FileSystemUriIntegrationTest.java

License:Apache License

/**
 * Tests connections to Alluxio cluster using URIs with connect details in authorities.
 *
 * @param authority the authority to test
 *///w w w .j a v  a  2s. co m
private void testConnection(String authority) throws Exception {
    Configuration conf = new Configuration();
    conf.set("fs.alluxio.impl", FileSystem.class.getName());
    URI uri = URI.create("alluxio://" + authority + "/tmp/path.txt");
    org.apache.hadoop.fs.FileSystem fs = org.apache.hadoop.fs.FileSystem.get(uri, conf);

    mCluster.waitForAllNodesRegistered(WAIT_TIMEOUT_MS);

    Path file = new Path("/testFile");
    FsPermission permission = FsPermission.createImmutable((short) 0666);
    FSDataOutputStream o = fs.create(file, permission, false /* ignored */, 10 /* ignored */,
            (short) 1 /* ignored */, 512 /* ignored */, null /* ignored */);
    o.writeBytes("Test Bytes");
    o.close();
    // with mark of delete-on-exit, the close method will try to delete it
    fs.deleteOnExit(file);
    fs.close();
    mCluster.notifySuccess();
}

From source file:alluxio.hadoop.contract.FileSystemContract.java

License:Apache License

@Override
public void init() throws IOException {
    super.init();
    Configuration conf = new Configuration();
    conf.set("fs.alluxio.impl", FileSystem.class.getName());

    URI uri = URI.create(mLocalAlluxioCluster.getMasterURI());

    mFS = org.apache.hadoop.fs.FileSystem.get(uri, conf);
}

From source file:alluxio.hadoop.FileSystemAclIntegrationTest.java

License:Apache License

@BeforeClass
public static void beforeClass() throws Exception {
    Configuration conf = new Configuration();
    conf.set("fs.alluxio.impl", FileSystem.class.getName());

    URI uri = URI.create(sLocalAlluxioClusterResource.get().getMasterURI());

    sTFS = org.apache.hadoop.fs.FileSystem.get(uri, conf);
    sUfsRoot = PathUtils.concatPath(alluxio.Configuration.get(PropertyKey.UNDERFS_ADDRESS));
    sUfs = UnderFileSystem.get(sUfsRoot);
}

From source file:alluxio.hadoop.FileSystemBlockLocationIntegrationTest.java

License:Apache License

@BeforeClass
public static void beforeClass() throws Exception {
    Configuration conf = new Configuration();
    conf.set("fs.alluxio.impl", FileSystem.class.getName());

    alluxio.client.file.FileSystem alluxioFS = sLocalAlluxioClusterResource.get().getClient();
    FileSystemTestUtils.createByteFile(alluxioFS, "/testFile1", WriteType.CACHE_THROUGH, FILE_LEN);

    URI uri = URI.create(sLocalAlluxioClusterResource.get().getMasterURI());
    sTFS = org.apache.hadoop.fs.FileSystem.get(uri, conf);
}