List of usage examples for org.apache.hadoop.fs FileSystem getDefaultUri
public static URI getDefaultUri(Configuration conf)
From source file:com.moz.fiji.mapreduce.IntegrationTestFijiBulkLoad.java
License:Apache License
/** * Generates a random HDFS path.//from ww w. j ava 2 s. c o m * * @param prefix Prefix for the random file name. * @return a random HDFS path. * @throws Exception on error. */ private Path makeRandomPath(String prefix) throws Exception { Preconditions.checkNotNull(mFS); final Path base = new Path(FileSystem.getDefaultUri(mConf)); final Random random = new Random(System.nanoTime()); return new Path(base, String.format("/%s-%s", prefix, random.nextLong())); }
From source file:com.splicemachine.derby.impl.io.HdfsDirFile.java
License:Apache License
/** * Construct a HdfsDirFile from a directory name and a file name. * * @param directoryName The directory part of the path name. * @param fileName The name of the file within the directory. *///www . j av a 2s .c om HdfsDirFile(String directoryName, String fileName) { if (fileName == null) { throw new IllegalArgumentException("The argument 'path' cannot be null."); } if (directoryName == null || directoryName.equals("")) { path = FileSystem.getDefaultUri(HConfiguration.unwrapDelegate()).getPath() + File.separatorChar + fileName; } else { path = directoryName + File.separatorChar + fileName; } }
From source file:com.streamsets.pipeline.spark.SparkStreamingBinding.java
License:Apache License
@Override public void init() throws Exception { for (Object key : properties.keySet()) { logMessage("Property => " + key + " => " + properties.getProperty(key.toString()), isRunningInMesos); }// w ww . j a v a 2 s . c o m final SparkConf conf = new SparkConf().setAppName("StreamSets Data Collector - Streaming Mode"); conf.set("spark.serializer", "org.apache.spark.serializer.KryoSerializer"); final String topic = getProperty(TOPIC); final long duration; String durationAsString = getProperty(MAX_WAIT_TIME); try { duration = Long.parseLong(durationAsString); } catch (NumberFormatException ex) { String msg = "Invalid " + MAX_WAIT_TIME + " '" + durationAsString + "' : " + ex; throw new IllegalArgumentException(msg, ex); } Configuration hadoopConf = new SparkHadoopUtil().newConfiguration(conf); if (isRunningInMesos) { hadoopConf = getHadoopConf(hadoopConf); } else { hadoopConf = new Configuration(); } URI hdfsURI = FileSystem.getDefaultUri(hadoopConf); logMessage("Default FS URI: " + hdfsURI, isRunningInMesos); FileSystem hdfs = (new Path(hdfsURI)).getFileSystem(hadoopConf); Path sdcCheckpointPath = new Path(hdfs.getHomeDirectory(), ".streamsets-spark-streaming/" + getProperty("sdc.id") + "/" + encode(topic)); // encode as remote pipeline name might have colon within it String pipelineName = encode(getProperty("cluster.pipeline.name")); final Path checkPointPath = new Path(sdcCheckpointPath, pipelineName); hdfs.mkdirs(checkPointPath); if (!hdfs.isDirectory(checkPointPath)) { throw new IllegalStateException("Could not create checkpoint path: " + sdcCheckpointPath); } if (isRunningInMesos) { String scheme = hdfsURI.getScheme(); if (scheme.equals("hdfs")) { File mesosBootstrapFile = BootstrapCluster.getMesosBootstrapFile(); Path mesosBootstrapPath = new Path(checkPointPath, mesosBootstrapFile.getName()); // in case of hdfs, copy the jar file from local path to hdfs hdfs.copyFromLocalFile(false, true, new Path(mesosBootstrapFile.toURI()), mesosBootstrapPath); conf.setJars(new String[] { mesosBootstrapPath.toString() }); } else if (scheme.equals("s3") || scheme.equals("s3n") || scheme.equals("s3a")) { // we cant upload the jar to s3 as executors wont understand s3 scheme without the aws jar. // So have the jar available on http conf.setJars(new String[] { getProperty("mesos.jar.url") }); } else { throw new IllegalStateException("Unsupported scheme: " + scheme); } } JavaStreamingContextFactory javaStreamingContextFactory = new JavaStreamingContextFactoryImpl(conf, duration, checkPointPath.toString(), getProperty(METADATA_BROKER_LIST), topic, properties.getProperty(AUTO_OFFSET_RESET, "").trim(), isRunningInMesos); ssc = JavaStreamingContext.getOrCreate(checkPointPath.toString(), hadoopConf, javaStreamingContextFactory, true); // mesos tries to stop the context internally, so don't do it here - deadlock bug in spark if (!isRunningInMesos) { final Thread shutdownHookThread = new Thread("Spark.shutdownHook") { @Override public void run() { LOG.debug("Gracefully stopping Spark Streaming Application"); ssc.stop(true, true); LOG.info("Application stopped"); } }; Runtime.getRuntime().addShutdownHook(shutdownHookThread); } logMessage("Making calls through spark context ", isRunningInMesos); ssc.start(); }
From source file:com.thinkbiganalytics.kylo.hadoop.FileSystemUtil.java
License:Apache License
/** * Parses and modifies the specified string to produce a {@link URL} that can be used with {@link URLClassLoader}. * * @param s string to parse as a URL//from ww w . j a va 2 s . co m * @param conf Hadoop configuration * @return parsed URL */ @Nonnull public static URL parseUrl(@Nonnull final String s, @Nullable final Configuration conf) { // Parse as a URI URI uri = URI.create(s); if (uri.getScheme() == null) { uri = ((conf != null) ? FileSystem.getDefaultUri(conf) : URI.create("file:///").resolve(new File("").getAbsolutePath() + "/")).resolve(uri); } if (!IGNORED_PROTOCOLS.contains(uri.getScheme())) { uri = URI.create("hadoop:" + uri); } // Create the URL try { return uri.toURL(); } catch (final MalformedURLException e) { throw new IllegalArgumentException(e); } }
From source file:com.thinkbiganalytics.nifi.security.ApplySecurityPolicy.java
License:Apache License
public boolean validateUserWithKerberos(ComponentLog loggerInstance, String HadoopConfigurationResources, String Principal, String KeyTab) throws Exception { ClassLoader savedClassLoader = Thread.currentThread().getContextClassLoader(); Thread.currentThread().setContextClassLoader(this.getClass().getClassLoader()); try {/* w ww. java 2s .c o m*/ loggerInstance.info("Start of hadoop configuration read"); Configuration config = getConfigurationFromResources(HadoopConfigurationResources); config.set("hadoop.security.authentication", "Kerberos"); loggerInstance.info("End of hadoop configuration read"); // first check for timeout on HDFS connection, because FileSystem has a hard coded 15 minute timeout loggerInstance.info("Start of HDFS timeout check"); checkHdfsUriForTimeout(config); loggerInstance.info("End of HDFS timeout check"); // disable caching of Configuration and FileSystem objects, else we cannot reconfigure the processor without a complete // restart String disableCacheName = String.format("fs.%s.impl.disable.cache", FileSystem.getDefaultUri(config).getScheme()); // If kerberos is enabled, create the file system as the kerberos principal // -- use RESOURCE_LOCK to guarantee UserGroupInformation is accessed by only a single thread at at time FileSystem fs; UserGroupInformation ugi; synchronized (RESOURCES_LOCK) { if (SecurityUtil.isSecurityEnabled(config)) { loggerInstance.info("Start of Kerberos Security Check"); UserGroupInformation.setConfiguration(config); UserGroupInformation.loginUserFromKeytab(Principal, KeyTab); loggerInstance.info("End of Kerberos Security Check"); } else { config.set("ipc.client.fallback-to-simple-auth-allowed", "true"); config.set("hadoop.security.authentication", "simple"); ugi = SecurityUtil.loginSimple(config); fs = getFileSystemAsUser(config, ugi); } } config.set(disableCacheName, "true"); return true; } catch (Exception e) { loggerInstance.error("Unable to validate user : " + e.getMessage()); return false; } finally { Thread.currentThread().setContextClassLoader(savedClassLoader); } }
From source file:com.thinkbiganalytics.nifi.security.ApplySecurityPolicy.java
License:Apache License
protected void checkHdfsUriForTimeout(Configuration config) throws IOException { URI hdfsUri = FileSystem.getDefaultUri(config); String address = hdfsUri.getAuthority(); int port = hdfsUri.getPort(); if (address == null || address.isEmpty() || port < 0) { return;/*from ww w .jav a 2 s .co m*/ } InetSocketAddress namenode = NetUtils.createSocketAddr(address, port); SocketFactory socketFactory = NetUtils.getDefaultSocketFactory(config); Socket socket = null; try { socket = socketFactory.createSocket(); NetUtils.connect(socket, namenode, 1000); // 1 second timeout } finally { IOUtils.closeQuietly(socket); } }
From source file:com.thinkbiganalytics.nifi.v2.hdfs.AbstractHadoopProcessor.java
License:Apache License
/** * Reset Hadoop Configuration and FileSystem based on the supplied configuration resources. * * @param configResources for configuration * @param dir the target directory * @param context for context, which gives access to the principal * @return An HdfsResources object// www. j a va2 s . c o m * @throws IOException if unable to access HDFS */ HdfsResources resetHDFSResources(String configResources, String dir, ProcessContext context) throws IOException { // org.apache.hadoop.conf.Configuration saves its current thread context class loader to use for threads that it creates // later to do I/O. We need this class loader to be the NarClassLoader instead of the magical // NarThreadContextClassLoader. ClassLoader savedClassLoader = Thread.currentThread().getContextClassLoader(); Thread.currentThread().setContextClassLoader(this.getClass().getClassLoader()); try { Configuration config = getConfigurationFromResources(configResources); // first check for timeout on HDFS connection, because FileSystem has a hard coded 15 minute timeout checkHdfsUriForTimeout(config); // disable caching of Configuration and FileSystem objects, else we cannot reconfigure the processor without a complete // restart String disableCacheName = String.format("fs.%s.impl.disable.cache", FileSystem.getDefaultUri(config).getScheme()); config.set(disableCacheName, "true"); // If kerberos is enabled, create the file system as the kerberos principal // -- use RESOURCE_LOCK to guarantee UserGroupInformation is accessed by only a single thread at at time FileSystem fs = null; UserGroupInformation ugi = null; synchronized (RESOURCES_LOCK) { if (config.get("hadoop.security.authentication").equalsIgnoreCase("kerberos")) { String principal = context.getProperty(kerberosPrincipal).getValue(); String keyTab = context.getProperty(kerberosKeytab).getValue(); UserGroupInformation.setConfiguration(config); ugi = UserGroupInformation.loginUserFromKeytabAndReturnUGI(principal, keyTab); modifyConfig(context, config); fs = getFileSystemAsUser(config, ugi); lastKerberosReloginTime = System.currentTimeMillis() / 1000; } else { config.set("ipc.client.fallback-to-simple-auth-allowed", "true"); config.set("hadoop.security.authentication", "simple"); modifyConfig(context, config); fs = getFileSystem(config); } } getLog().info( "Initialized a new HDFS File System with working dir: {} default block size: {} default replication: {} config: {}", new Object[] { fs.getWorkingDirectory(), fs.getDefaultBlockSize(new Path(dir)), fs.getDefaultReplication(new Path(dir)), config.toString() }); return new HdfsResources(config, fs, ugi); } finally { Thread.currentThread().setContextClassLoader(savedClassLoader); } }
From source file:com.thinkbiganalytics.nifi.v2.hdfs.AbstractHadoopProcessor.java
License:Apache License
/** * This exists in order to allow unit tests to override it so that they don't take several minutes waiting for UDP packets to be received * * @param config the configuration to use * @return the FileSystem that is created for the given Configuration * @throws IOException if unable to create the FileSystem *//*from ww w. j av a 2s .com*/ protected FileSystem getFileSystem(final Configuration config) throws IOException { if (getLog().isDebugEnabled()) { String disableCacheName = String.format("fs.%s.impl.disable.cache", FileSystem.getDefaultUri(config).getScheme()); getLog().debug(String.format("'%s'='%s'", disableCacheName, config.get(disableCacheName))); } return FileSystem.get(config); }
From source file:com.uber.hoodie.io.storage.HoodieWrapperFileSystem.java
License:Apache License
public static Path convertToHoodiePath(Path file, Configuration conf) { String scheme = FileSystem.getDefaultUri(conf).getScheme(); return convertPathWithScheme(file, getHoodieScheme(scheme)); }
From source file:common.NameNode.java
License:Apache License
public static InetSocketAddress getAddress(Configuration conf) { URI filesystemURI = FileSystem.getDefaultUri(conf); String authority = filesystemURI.getAuthority(); if (authority == null) { throw new IllegalArgumentException( String.format("Invalid URI for NameNode address (check %s): %s has no authority.", FileSystem.FS_DEFAULT_NAME_KEY, filesystemURI.toString())); }// w w w . j a v a 2 s. c om if (!FSConstants.HDFS_URI_SCHEME.equalsIgnoreCase(filesystemURI.getScheme())) { throw new IllegalArgumentException( String.format("Invalid URI for NameNode address (check %s): %s is not of scheme '%s'.", FileSystem.FS_DEFAULT_NAME_KEY, filesystemURI.toString(), FSConstants.HDFS_URI_SCHEME)); } return getAddress(authority); }