Example usage for org.apache.hadoop.conf Configuration set

Introduction

In this page you can find the example usage for org.apache.hadoop.conf Configuration set.

Prototype

public void set(String name, String value)

Source Link

Document

Set the value of the name property.

Usage

From source file:com.cloudera.dataflow.spark.TransformTranslator.java

License:Open Source License

private static <K, V> TransformEvaluator<HadoopIO.Write.Bound<K, V>> writeHadoop() {
    return new TransformEvaluator<HadoopIO.Write.Bound<K, V>>() {
        @Override/*w  w w .j  ava  2 s . c o  m*/
        public void evaluate(HadoopIO.Write.Bound<K, V> transform, EvaluationContext context) {
            @SuppressWarnings("unchecked")
            JavaPairRDD<K, V> last = ((JavaRDDLike<WindowedValue<KV<K, V>>, ?>) context.getInputRDD(transform))
                    .map(WindowingHelpers.<KV<K, V>>unwindowFunction())
                    .mapToPair(new PairFunction<KV<K, V>, K, V>() {
                        @Override
                        public Tuple2<K, V> call(KV<K, V> t) throws Exception {
                            return new Tuple2<>(t.getKey(), t.getValue());
                        }
                    });
            ShardTemplateInformation shardTemplateInfo = new ShardTemplateInformation(transform.getNumShards(),
                    transform.getShardTemplate(), transform.getFilenamePrefix(), transform.getFilenameSuffix());
            Configuration conf = new Configuration();
            for (Map.Entry<String, String> e : transform.getConfigurationProperties().entrySet()) {
                conf.set(e.getKey(), e.getValue());
            }
            writeHadoopFile(last, conf, shardTemplateInfo, transform.getKeyClass(), transform.getValueClass(),
                    transform.getFormatClass());
        }
    };
}

From source file:com.cloudera.dataflow.spark.TransformTranslator.java

License:Open Source License

private static <K, V> void writeHadoopFile(JavaPairRDD<K, V> rdd, Configuration conf,
        ShardTemplateInformation shardTemplateInfo, Class<?> keyClass, Class<?> valueClass,
        Class<? extends FileOutputFormat> formatClass) {
    int numShards = shardTemplateInfo.getNumShards();
    String shardTemplate = shardTemplateInfo.getShardTemplate();
    String filenamePrefix = shardTemplateInfo.getFilenamePrefix();
    String filenameSuffix = shardTemplateInfo.getFilenameSuffix();
    if (numShards != 0) {
        // number of shards was set explicitly, so repartition
        rdd = rdd.repartition(numShards);
    }//from  w  ww . j  a  va  2 s .c om
    int actualNumShards = rdd.partitions().size();
    String template = replaceShardCount(shardTemplate, actualNumShards);
    String outputDir = getOutputDirectory(filenamePrefix, template);
    String filePrefix = getOutputFilePrefix(filenamePrefix, template);
    String fileTemplate = getOutputFileTemplate(filenamePrefix, template);

    conf.set(ShardNameTemplateHelper.OUTPUT_FILE_PREFIX, filePrefix);
    conf.set(ShardNameTemplateHelper.OUTPUT_FILE_TEMPLATE, fileTemplate);
    conf.set(ShardNameTemplateHelper.OUTPUT_FILE_SUFFIX, filenameSuffix);
    rdd.saveAsNewAPIHadoopFile(outputDir, keyClass, valueClass, formatClass, conf);
}

From source file:com.cloudera.hadoop.hdfs.nfs.nfs4.TestWritableFileFileHandleStore.java

License:Apache License

@Test
public void testWrite() throws Exception {
    Configuration conf = new Configuration();
    conf.setClass(NFS_FILEHANDLE_STORE_CLASS, WritableFileFileHandleStore.class, FileHandleStore.class);
    File file = new File("/tmp/" + UUID.randomUUID().toString());
    file.delete();/*from w  w  w  .  j  av a2s . c om*/
    conf.set(NFS_FILEHANDLE_STORE_FILE, file.getAbsolutePath());
    WritableFileFileHandleStore store = (WritableFileFileHandleStore) FileHandleStore.get(conf);
    assertTrue(store.getAll().isEmpty());
    long inode = System.currentTimeMillis();
    byte[] fileHandle = String.valueOf(inode).getBytes();
    String name = String.valueOf(inode);
    FileHandleStoreEntry entry = new FileHandleStoreEntry(fileHandle, name, inode);
    store.storeFileHandle(entry);
    assertTrue(store.getAll().size() == 1);
    entry = store.getAll().get(0);
    assertTrue(entry.fileID == inode);
    assertTrue(Arrays.equals(fileHandle, entry.fileHandle));
    assertEquals(name, entry.path);
    store.close();

    store = (WritableFileFileHandleStore) FileHandleStore.get(conf);
    assertTrue(store.getAll().size() == 1);
    entry = store.getAll().get(0);
    assertTrue(entry.fileID == inode);
    assertTrue(Arrays.equals(fileHandle, entry.fileHandle));
    assertEquals(name, entry.path);
    store.close();
    file.delete();
}

From source file:com.cloudera.hadoop.hdfs.nfs.TestUtils.java

License:Apache License

public static Configuration setupConf(Configuration conf) throws IOException {
    conf.set(NFS_FILEHANDLE_STORE_FILE, "/tmp/" + UUID.randomUUID().toString());
    return conf;//from   w ww.  j  ava2 s.  c o  m
}

From source file:com.cloudera.hoop.client.fs.TestHoopFileSystem.java

License:Open Source License

private void createHoopServer() throws Exception {
    File homeDir = getTestDir();// w w w . ja v a  2  s. c o m
    Assert.assertTrue(new File(homeDir, "conf").mkdir());
    Assert.assertTrue(new File(homeDir, "log").mkdir());
    Assert.assertTrue(new File(homeDir, "temp").mkdir());
    HoopServer.setHomeDirForCurrentThread(homeDir.getAbsolutePath());

    String fsDefaultName = getHadoopConf().get("fs.default.name");
    Configuration conf = new Configuration(false);
    conf.set("hoop.hadoop.conf:fs.default.name", fsDefaultName);
    conf.set("hoop.base.url", getJettyURL().toExternalForm());
    conf.set("hoop.proxyuser." + getHadoopProxyUser() + ".groups", getHadoopProxyUserGroups());
    conf.set("hoop.proxyuser." + getHadoopProxyUser() + ".hosts", getHadoopProxyUserHosts());
    File hoopSite = new File(new File(homeDir, "conf"), "hoop-site.xml");
    OutputStream os = new FileOutputStream(hoopSite);
    conf.writeXml(os);
    os.close();

    File currentDir = new File("foo").getAbsoluteFile().getParentFile();
    if (currentDir.getName().equals("target")) {
        currentDir = currentDir.getParentFile();
    }
    if (currentDir.getName().equals("hoop-client")) {
        currentDir = currentDir.getParentFile();
    }
    File hoopDir = new File(currentDir, "hoop-webapp");
    Assert.assertTrue(hoopDir.exists(),
            "Could not locate hoop-webapp source dir: " + hoopDir.getAbsolutePath());
    String hoopWebAppDir = new File(new File(new File(hoopDir, "src"), "main"), "webapp").getAbsolutePath();
    WebAppContext context = new WebAppContext(hoopWebAppDir, "/");

    Server server = getJettyServer();
    server.addHandler(context);
    server.start();
}

From source file:com.cloudera.hoop.client.fs.TestHoopFileSystem.java

License:Open Source License

private void testGet() throws Exception {
    Configuration conf = new Configuration();
    conf.set("fs.http.impl", HoopFileSystem.class.getName());
    FileSystem fs = FileSystem.get(getJettyURL().toURI(), conf);
    Assert.assertNotNull(fs);// w  w w.  j av  a 2 s.  co  m
    Assert.assertEquals(fs.getUri(), getJettyURL().toURI());
    fs.close();
}

From source file:com.cloudera.hoop.client.fs.TestHoopFileSystem.java

License:Open Source License

private void testOpen() throws Exception {
    FileSystem fs = FileSystem.get(getHadoopConf());
    Path path = new Path(getHadoopTestDir(), "foo.txt");
    OutputStream os = fs.create(path);
    os.write(1);/*from  ww w.j  a v a2s .c  o  m*/
    os.close();
    fs.close();
    Configuration conf = new Configuration();
    conf.set("fs.http.impl", HoopFileSystem.class.getName());
    fs = FileSystem.get(getJettyURL().toURI(), conf);
    InputStream is = fs.open(new Path(path.toUri().getPath()));
    Assert.assertEquals(is.read(), 1);
    is.close();
    fs.close();
}

From source file:com.cloudera.hoop.client.fs.TestHoopFileSystem.java

License:Open Source License

private void testCreate(Path path, boolean override) throws Exception {
    Configuration conf = new Configuration();
    conf.set("fs.http.impl", HoopFileSystem.class.getName());
    FileSystem fs = FileSystem.get(getJettyURL().toURI(), conf);
    FsPermission permission = new FsPermission(FsAction.READ_WRITE, FsAction.NONE, FsAction.NONE);
    OutputStream os = fs.create(new Path(path.toUri().getPath()), permission, override, 1024, (short) 2,
            100 * 1024 * 1024, null);/*from w w w . ja  v  a 2 s  .  c o m*/
    os.write(1);
    os.close();
    fs.close();

    fs = FileSystem.get(getHadoopConf());
    FileStatus status = fs.getFileStatus(path);
    Assert.assertEquals(status.getReplication(), 2);
    Assert.assertEquals(status.getBlockSize(), 100 * 1024 * 1024);
    Assert.assertEquals(status.getPermission(), permission);
    InputStream is = fs.open(path);
    Assert.assertEquals(is.read(), 1);
    is.close();
    fs.close();
}

From source file:com.cloudera.hoop.client.fs.TestHoopFileSystem.java

License:Open Source License

private void testAppend() throws Exception {
    FileSystem fs = FileSystem.get(getHadoopConf());
    Path path = new Path(getHadoopTestDir(), "foo.txt");
    OutputStream os = fs.create(path);
    os.write(1);/*from   ww  w .j  a  va2  s  .  c  o  m*/
    os.close();
    fs.close();
    Configuration conf = new Configuration();
    conf.set("fs.http.impl", HoopFileSystem.class.getName());
    fs = FileSystem.get(getJettyURL().toURI(), conf);
    os = fs.append(new Path(path.toUri().getPath()));
    os.write(2);
    os.close();
    fs.close();
    fs = FileSystem.get(getHadoopConf());
    InputStream is = fs.open(path);
    Assert.assertEquals(is.read(), 1);
    Assert.assertEquals(is.read(), 2);
    Assert.assertEquals(is.read(), -1);
    is.close();
    fs.close();
}

From source file:com.cloudera.hoop.client.fs.TestHoopFileSystem.java

License:Open Source License

private void testRename() throws Exception {
    FileSystem fs = FileSystem.get(getHadoopConf());
    Path path = new Path(getHadoopTestDir(), "foo");
    fs.mkdirs(path);/*from   w w w  . jav a2 s.  co m*/
    fs.close();
    Configuration conf = new Configuration();
    conf.set("fs.http.impl", HoopFileSystem.class.getName());
    fs = FileSystem.get(getJettyURL().toURI(), conf);
    Path oldPath = new Path(path.toUri().getPath());
    Path newPath = new Path(path.getParent(), "bar");
    fs.rename(oldPath, newPath);
    fs.close();
    fs = FileSystem.get(getHadoopConf());
    Assert.assertFalse(fs.exists(oldPath));
    Assert.assertTrue(fs.exists(newPath));
    fs.close();
}