Example usage for org.apache.hadoop.conf Configuration getTrimmedStringCollection

Introduction

In this page you can find the example usage for org.apache.hadoop.conf Configuration getTrimmedStringCollection.

Prototype

public Collection<String> getTrimmedStringCollection(String name) 

Source Link

Document

Get the comma delimited values of the name property as a collection of Strings, trimmed of the leading and trailing whitespace.

Usage

From source file:com.mellanox.r4h.MiniDFSCluster.java

License:Apache License

private void createNameNodesAndSetConf(MiniDFSNNTopology nnTopology, boolean manageNameDfsDirs,
        boolean manageNameDfsSharedDirs, boolean enableManagedDfsDirsRedundancy, boolean format,
        StartupOption operation, String clusterId, Configuration conf) throws IOException {
    Preconditions.checkArgument(nnTopology.countNameNodes() > 0, "empty NN topology: no namenodes specified!");

    if (!federation && nnTopology.countNameNodes() == 1) {
        NNConf onlyNN = nnTopology.getOnlyNameNode();
        // we only had one NN, set DEFAULT_NAME for it. If not explicitly
        // specified initially, the port will be 0 to make NN bind to any
        // available port. It will be set to the right address after
        // NN is started.
        conf.set(FS_DEFAULT_NAME_KEY, "hdfs://127.0.0.1:" + MiniDFSClusterBridge.getNNConf_ipcPort(onlyNN));
    }// w  w  w .jav  a  2 s .co m

    List<String> allNsIds = Lists.newArrayList();
    for (MiniDFSNNTopology.NSConf nameservice : nnTopology.getNameservices()) {
        if (nameservice.getId() != null) {
            allNsIds.add(nameservice.getId());
        }
    }
    if (!allNsIds.isEmpty()) {
        conf.set(DFS_NAMESERVICES, Joiner.on(",").join(allNsIds));
    }

    int nnCounter = 0;
    for (MiniDFSNNTopology.NSConf nameservice : nnTopology.getNameservices()) {
        String nsId = nameservice.getId();
        String lastDefaultFileSystem = null;

        Preconditions.checkArgument(!federation || nsId != null,
                "if there is more than one NS, they must have names");

        // First set up the configuration which all of the NNs
        // need to have - have to do this a priori before starting
        // *any* of the NNs, so they know to come up in standby.
        List<String> nnIds = Lists.newArrayList();
        // Iterate over the NNs in this nameservice
        for (NNConf nn : nameservice.getNNs()) {
            nnIds.add(MiniDFSClusterBridge.getNNConf_nnId(nn));

            initNameNodeAddress(conf, nameservice.getId(), nn);
        }

        // If HA is enabled on this nameservice, enumerate all the namenodes
        // in the configuration. Also need to set a shared edits dir
        if (nnIds.size() > 1) {
            conf.set(DFSUtil.addKeySuffixes(DFS_HA_NAMENODES_KEY_PREFIX, nameservice.getId()),
                    Joiner.on(",").join(nnIds));
            if (manageNameDfsSharedDirs) {
                URI sharedEditsUri = getSharedEditsDir(nnCounter, nnCounter + nnIds.size() - 1);
                conf.set(DFS_NAMENODE_SHARED_EDITS_DIR_KEY, sharedEditsUri.toString());
                // Clean out the shared edits dir completely, including all subdirectories.
                FileUtil.fullyDelete(new File(sharedEditsUri));
            }
        }

        // Now format first NN and copy the storage directory from that node to the others.
        int i = 0;
        Collection<URI> prevNNDirs = null;
        int nnCounterForFormat = nnCounter;
        for (NNConf nn : nameservice.getNNs()) {
            initNameNodeConf(conf, nsId, MiniDFSClusterBridge.getNNConf_nnId(nn), manageNameDfsDirs,
                    enableManagedDfsDirsRedundancy, nnCounterForFormat);
            Collection<URI> namespaceDirs = FSNamesystem.getNamespaceDirs(conf);
            if (format) {
                for (URI nameDirUri : namespaceDirs) {
                    File nameDir = new File(nameDirUri);
                    if (nameDir.exists() && !FileUtil.fullyDelete(nameDir)) {
                        throw new IOException("Could not fully delete " + nameDir);
                    }
                }
                Collection<URI> checkpointDirs = Util.stringCollectionAsURIs(
                        conf.getTrimmedStringCollection(DFS_NAMENODE_CHECKPOINT_DIR_KEY));
                for (URI checkpointDirUri : checkpointDirs) {
                    File checkpointDir = new File(checkpointDirUri);
                    if (checkpointDir.exists() && !FileUtil.fullyDelete(checkpointDir)) {
                        throw new IOException("Could not fully delete " + checkpointDir);
                    }
                }
            }

            boolean formatThisOne = format;
            if (format && i++ > 0) {
                // Don't format the second NN in an HA setup - that
                // would result in it having a different clusterID,
                // block pool ID, etc. Instead, copy the name dirs
                // from the first one.
                formatThisOne = false;
                assert (null != prevNNDirs);
                copyNameDirs(prevNNDirs, namespaceDirs, conf);
            }

            nnCounterForFormat++;
            if (formatThisOne) {
                // Allow overriding clusterID for specific NNs to test
                // misconfiguration.
                if (MiniDFSClusterBridge.getNNConf_cliusterId(nn) == null) {
                    StartupOption.FORMAT.setClusterId(clusterId);
                } else {
                    StartupOption.FORMAT.setClusterId(MiniDFSClusterBridge.getNNConf_cliusterId(nn));
                }
                DFSTestUtil.formatNameNode(conf);
            }
            prevNNDirs = namespaceDirs;
        }

        // Start all Namenodes
        for (NNConf nn : nameservice.getNNs()) {
            initNameNodeConf(conf, nsId, MiniDFSClusterBridge.getNNConf_nnId(nn), manageNameDfsDirs,
                    enableManagedDfsDirsRedundancy, nnCounter);
            createNameNode(nnCounter, conf, numDataNodes, false, operation, clusterId, nsId,
                    MiniDFSClusterBridge.getNNConf_nnId(nn));
            // Record the last namenode uri
            if (nameNodes[nnCounter] != null && nameNodes[nnCounter].conf != null) {
                lastDefaultFileSystem = nameNodes[nnCounter].conf.get(FS_DEFAULT_NAME_KEY);
            }
            nnCounter++;
        }
        if (!federation && lastDefaultFileSystem != null) {
            // Set the default file system to the actual bind address of NN.
            conf.set(FS_DEFAULT_NAME_KEY, lastDefaultFileSystem);
        }
    }

}

---

From source file:org.apache.kylin.storage.hbase.HBaseConnection.java

License:Apache License

public static void addHBaseClusterNNHAConfiguration(Configuration conf) {
    String hdfsConfigFile = KylinConfig.getInstanceFromEnv().getHBaseClusterHDFSConfigFile();
    if (hdfsConfigFile == null || hdfsConfigFile.isEmpty()) {
        return;/*from   w  w  w  .j  av  a2s . co m*/
    }
    Configuration hdfsConf = new Configuration(false);
    hdfsConf.addResource(hdfsConfigFile);
    Collection<String> nameServices = hdfsConf.getTrimmedStringCollection(DFSConfigKeys.DFS_NAMESERVICES);
    Collection<String> mainNameServices = conf.getTrimmedStringCollection(DFSConfigKeys.DFS_NAMESERVICES);
    for (String serviceId : nameServices) {
        mainNameServices.add(serviceId);

        String serviceConfKey = DFSConfigKeys.DFS_HA_NAMENODES_KEY_PREFIX + "." + serviceId;
        String proxyConfKey = DFSConfigKeys.DFS_CLIENT_FAILOVER_PROXY_PROVIDER_KEY_PREFIX + "." + serviceId;
        conf.set(serviceConfKey, hdfsConf.get(serviceConfKey, ""));
        conf.set(proxyConfKey, hdfsConf.get(proxyConfKey, ""));

        Collection<String> nameNodes = hdfsConf.getTrimmedStringCollection(serviceConfKey);
        for (String nameNode : nameNodes) {
            String rpcConfKey = DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY + "." + serviceId + "." + nameNode;
            conf.set(rpcConfKey, hdfsConf.get(rpcConfKey, ""));
        }
    }
    conf.setStrings(DFSConfigKeys.DFS_NAMESERVICES, mainNameServices.toArray(new String[0]));
    // See YARN-3021, instruct RM skip renew token of hbase cluster name services
    conf.setStrings(JOB_NAMENODES_TOKEN_RENEWAL_EXCLUDE, nameServices.toArray(new String[0]));
}

---

From source file:org.apache.tajo.storage.DiskUtil.java

License:Apache License

public static List<URI> getStorageDirs() {
    Configuration conf = new HdfsConfiguration();
    Collection<String> dirNames = conf.getTrimmedStringCollection(DFS_DATANODE_DATA_DIR_KEY);
    return Util.stringCollectionAsURIs(dirNames);
}

---