List of usage examples for org.apache.hadoop.conf Configuration getTrimmedStrings
public String[] getTrimmedStrings(String name)
name property as an array of Strings, trimmed of the leading and trailing whitespace. From source file:co.cask.tephra.snapshot.SnapshotCodecProvider.java
License:Apache License
/** * Register all codec specified in the configuration with this provider. * There can only be one codec for a given version. */// ww w. ja v a 2 s .c o m private void initialize(Configuration configuration) { String[] codecClassNames = configuration .getTrimmedStrings(TxConstants.Persist.CFG_TX_SNAPHOT_CODEC_CLASSES); List<Class> codecClasses = Lists.newArrayList(); if (codecClassNames != null) { for (String clsName : codecClassNames) { try { codecClasses.add(Class.forName(clsName)); } catch (ClassNotFoundException cnfe) { LOG.warn("Unable to load class configured for " + TxConstants.Persist.CFG_TX_SNAPHOT_CODEC_CLASSES + ": " + clsName, cnfe); } } } if (codecClasses.size() == 0) { codecClasses.addAll(Arrays.asList(TxConstants.Persist.DEFAULT_TX_SNAPHOT_CODEC_CLASSES)); } for (Class<?> codecClass : codecClasses) { try { SnapshotCodec codec = (SnapshotCodec) (codecClass.newInstance()); codecs.put(codec.getVersion(), codec); LOG.debug("Using snapshot codec {} for snapshots of version {}", codecClass.getName(), codec.getVersion()); } catch (Exception e) { LOG.warn("Error instantiating snapshot codec {}. Skipping.", codecClass.getName(), e); } } }
From source file:com.mellanox.r4h.DFSClient.java
License:Apache License
/** * Create a new DFSClient connected to the given nameNodeUri or rpcNamenode. * If HA is enabled and a positive value is set for {@link DFSConfigKeys#DFS_CLIENT_TEST_DROP_NAMENODE_RESPONSE_NUM_KEY} in the * configuration, the DFSClient will use {@link LossyRetryInvocationHandler} as its RetryInvocationHandler. Otherwise one of nameNodeUri or * rpcNamenode//from www . ja va2 s . c om * must be null. */ @VisibleForTesting public DFSClient(URI nameNodeUri, ClientProtocol rpcNamenode, Configuration conf, FileSystem.Statistics stats) throws IOException { SpanReceiverHost.get(conf, DFSConfigKeys.DFS_CLIENT_HTRACE_PREFIX); traceSampler = new SamplerBuilder(TraceUtils.wrapHadoopConf(DFSConfigKeys.DFS_CLIENT_HTRACE_PREFIX, conf)) .build(); // Copy only the required DFSClient configuration this.dfsClientConf = new DFSClientConfBridge2_7(conf); if (this.dfsClientConf.isUseLegacyBlockReaderLocal()) { LOG.debug("Using legacy short-circuit local reads."); } this.conf = conf; this.stats = stats; this.socketFactory = NetUtils.getSocketFactory(conf, ClientProtocol.class); this.dtpReplaceDatanodeOnFailure = ReplaceDatanodeOnFailure.get(conf); this.ugi = UserGroupInformation.getCurrentUser(); this.authority = nameNodeUri == null ? "null" : nameNodeUri.getAuthority(); this.clientName = "DFSClient_" + dfsClientConf.getTaskId() + "_" + DFSUtil.getRandom().nextInt() + "_" + Thread.currentThread().getId(); provider = DFSUtil.createKeyProvider(conf); if (LOG.isDebugEnabled()) { if (provider == null) { LOG.debug("No KeyProvider found."); } else { LOG.debug("Found KeyProvider: " + provider.toString()); } } int numResponseToDrop = conf.getInt(DFSConfigKeys.DFS_CLIENT_TEST_DROP_NAMENODE_RESPONSE_NUM_KEY, DFSConfigKeys.DFS_CLIENT_TEST_DROP_NAMENODE_RESPONSE_NUM_DEFAULT); NameNodeProxies.ProxyAndInfo<ClientProtocol> proxyInfo = null; AtomicBoolean nnFallbackToSimpleAuth = new AtomicBoolean(false); if (numResponseToDrop > 0) { // This case is used for testing. LOG.warn(DFSConfigKeys.DFS_CLIENT_TEST_DROP_NAMENODE_RESPONSE_NUM_KEY + " is set to " + numResponseToDrop + ", this hacked client will proactively drop responses"); proxyInfo = NameNodeProxies.createProxyWithLossyRetryHandler(conf, nameNodeUri, ClientProtocol.class, numResponseToDrop, nnFallbackToSimpleAuth); } if (proxyInfo != null) { this.dtService = proxyInfo.getDelegationTokenService(); this.namenode = proxyInfo.getProxy(); } else if (rpcNamenode != null) { // This case is used for testing. Preconditions.checkArgument(nameNodeUri == null); this.namenode = rpcNamenode; dtService = null; } else { Preconditions.checkArgument(nameNodeUri != null, "null URI"); proxyInfo = NameNodeProxies.createProxy(conf, nameNodeUri, ClientProtocol.class, nnFallbackToSimpleAuth); this.dtService = proxyInfo.getDelegationTokenService(); this.namenode = proxyInfo.getProxy(); } String localInterfaces[] = conf.getTrimmedStrings(DFSConfigKeys.DFS_CLIENT_LOCAL_INTERFACES); localInterfaceAddrs = getLocalInterfaceAddrs(localInterfaces); if (LOG.isDebugEnabled() && 0 != localInterfaces.length) { LOG.debug("Using local interfaces [" + Joiner.on(',').join(localInterfaces) + "] with addresses [" + Joiner.on(',').join(localInterfaceAddrs) + "]"); } Boolean readDropBehind = (conf.get(DFS_CLIENT_CACHE_DROP_BEHIND_READS) == null) ? null : conf.getBoolean(DFS_CLIENT_CACHE_DROP_BEHIND_READS, false); Long readahead = (conf.get(DFS_CLIENT_CACHE_READAHEAD) == null) ? null : conf.getLong(DFS_CLIENT_CACHE_READAHEAD, 0); Boolean writeDropBehind = (conf.get(DFS_CLIENT_CACHE_DROP_BEHIND_WRITES) == null) ? null : conf.getBoolean(DFS_CLIENT_CACHE_DROP_BEHIND_WRITES, false); this.defaultReadCachingStrategy = new CachingStrategy(readDropBehind, readahead); this.defaultWriteCachingStrategy = new CachingStrategy(writeDropBehind, readahead); this.clientContext = ClientContext.get(conf.get(DFS_CLIENT_CONTEXT, DFS_CLIENT_CONTEXT_DEFAULT), dfsClientConf); this.hedgedReadThresholdMillis = conf.getLong(DFSConfigKeys.DFS_DFSCLIENT_HEDGED_READ_THRESHOLD_MILLIS, DFSConfigKeys.DEFAULT_DFSCLIENT_HEDGED_READ_THRESHOLD_MILLIS); int numThreads = conf.getInt(DFSConfigKeys.DFS_DFSCLIENT_HEDGED_READ_THREADPOOL_SIZE, DFSConfigKeys.DEFAULT_DFSCLIENT_HEDGED_READ_THREADPOOL_SIZE); if (numThreads > 0) { this.initThreadsNumForHedgedReads(numThreads); } this.saslClient = new SaslDataTransferClient(conf, DataTransferSaslUtil.getSaslPropertiesResolver(conf), TrustedChannelResolver.getInstance(conf), nnFallbackToSimpleAuth); }
From source file:org.apache.accumulo.core.trace.DistributedTrace.java
License:Apache License
private static synchronized void loadSpanReceivers(Configuration conf) { if (!receivers.isEmpty()) { log.info("Already loaded span receivers, enable tracing does not need to be called again"); return;/*from ww w . j a v a 2s .c o m*/ } String[] receiverNames = conf.getTrimmedStrings(Property.TRACE_SPAN_RECEIVERS.toString()); if (receiverNames == null || receiverNames.length == 0) { return; } for (String className : receiverNames) { SpanReceiverBuilder builder = new SpanReceiverBuilder(wrapHadoopConf(conf)); SpanReceiver rcvr = builder.spanReceiverClass(className.trim()).build(); if (rcvr == null) { log.warn("Failed to load SpanReceiver " + className); } else { receivers.add(rcvr); log.info("SpanReceiver " + className + " was loaded successfully."); } } for (SpanReceiver rcvr : receivers) { org.apache.htrace.Trace.addReceiver(rcvr); } }
From source file:org.apache.tez.dag.app.rm.TezTestServiceTaskSchedulerService.java
License:Apache License
public TezTestServiceTaskSchedulerService(TaskSchedulerContext taskSchedulerContext) { // Accepting configuration here to allow setting up fields as final super(taskSchedulerContext); this.serviceHosts = new LinkedList<String>(); this.containerFactory = new ContainerFactory(taskSchedulerContext.getApplicationAttemptId(), taskSchedulerContext.getCustomClusterIdentifier()); Configuration conf = null; try {/*from www .ja v a 2s. c o m*/ conf = TezUtils.createConfFromUserPayload(taskSchedulerContext.getInitialUserPayload()); } catch (IOException e) { throw new TezUncheckedException(e); } this.memoryPerInstance = conf.getInt(TezTestServiceConfConstants.TEZ_TEST_SERVICE_MEMORY_PER_INSTANCE_MB, -1); Preconditions.checkArgument(memoryPerInstance > 0, TezTestServiceConfConstants.TEZ_TEST_SERVICE_MEMORY_PER_INSTANCE_MB + " must be configured"); this.executorsPerInstance = conf .getInt(TezTestServiceConfConstants.TEZ_TEST_SERVICE_NUM_EXECUTORS_PER_INSTANCE, -1); Preconditions.checkArgument(executorsPerInstance > 0, TezTestServiceConfConstants.TEZ_TEST_SERVICE_NUM_EXECUTORS_PER_INSTANCE + " must be configured"); this.coresPerInstance = conf.getInt(TezTestServiceConfConstants.TEZ_TEST_SERVICE_VCPUS_PER_INSTANCE, executorsPerInstance); this.containerPort = conf.getInt(TezTestServiceConfConstants.TEZ_TEST_SERVICE_RPC_PORT, -1); Preconditions.checkArgument(executorsPerInstance > 0, TezTestServiceConfConstants.TEZ_TEST_SERVICE_RPC_PORT + " must be configured"); int memoryPerContainer = (int) (memoryPerInstance / (float) executorsPerInstance); int coresPerContainer = (int) (coresPerInstance / (float) executorsPerInstance); this.resourcePerContainer = Resource.newInstance(memoryPerContainer, coresPerContainer); String[] hosts = conf.getTrimmedStrings(TezTestServiceConfConstants.TEZ_TEST_SERVICE_HOSTS); if (hosts == null || hosts.length == 0) { hosts = new String[] { "localhost" }; } for (String host : hosts) { serviceHosts.add(host); } LOG.info("Running with configuration: " + "memoryPerInstance=" + memoryPerInstance + ", vcoresPerInstance=" + coresPerInstance + ", executorsPerInstance=" + executorsPerInstance + ", resourcePerContainerInferred=" + resourcePerContainer + ", hosts=" + serviceHosts.toString()); }