List of usage examples for org.apache.hadoop.fs FileSystem newInstance
public static FileSystem newInstance(URI uri, Configuration config) throws IOException
From source file:com.datatorrent.stram.StramClient.java
License:Apache License
public void copyInitialState(Path origAppDir) throws IOException { // locate previous snapshot String newAppDir = this.dag.assertAppPath(); FSRecoveryHandler recoveryHandler = new FSRecoveryHandler(origAppDir.toString(), conf); // read snapshot against new dependencies Object snapshot = recoveryHandler.restore(); if (snapshot == null) { throw new IllegalArgumentException("No previous application state found in " + origAppDir); }//www . j a v a 2 s . c om InputStream logIs = recoveryHandler.getLog(); // modify snapshot state to switch app id ((StreamingContainerManager.CheckpointState) snapshot).setApplicationId(this.dag, conf); Path checkpointPath = new Path(newAppDir, LogicalPlan.SUBDIR_CHECKPOINTS); FileSystem fs = FileSystem.newInstance(origAppDir.toUri(), conf); // remove the path that was created by the storage agent during deserialization and replacement fs.delete(checkpointPath, true); // write snapshot to new location recoveryHandler = new FSRecoveryHandler(newAppDir, conf); recoveryHandler.save(snapshot); OutputStream logOs = recoveryHandler.rotateLog(); IOUtils.copy(logIs, logOs); logOs.flush(); logOs.close(); logIs.close(); // copy sub directories that are not present in target FileStatus[] lFiles = fs.listStatus(origAppDir); for (FileStatus f : lFiles) { if (f.isDirectory()) { String targetPath = f.getPath().toString().replace(origAppDir.toString(), newAppDir); if (!fs.exists(new Path(targetPath))) { LOG.debug("Copying {} to {}", f.getPath(), targetPath); FileUtil.copy(fs, f.getPath(), fs, new Path(targetPath), false, conf); //FSUtil.copy(fs, f, fs, new Path(targetPath), false, false, conf); } else { LOG.debug("Ignoring {} as it already exists under {}", f.getPath(), targetPath); //FSUtil.setPermission(fs, new Path(targetPath), new FsPermission((short)0777)); } } } }
From source file:com.datatorrent.stram.StreamingContainerManager.java
License:Apache License
/** * This method is for saving meta information about this application in HDFS -- the meta information that generally * does not change across multiple attempts *//* w ww . ja v a2 s . com*/ private void saveMetaInfo() throws IOException { Path path = new Path(this.vars.appPath, APP_META_FILENAME + "." + System.nanoTime()); FileSystem fs = FileSystem.newInstance(path.toUri(), new Configuration()); try { FSDataOutputStream os = fs.create(path); try { JSONObject top = new JSONObject(); JSONObject attributes = new JSONObject(); for (Map.Entry<Attribute<?>, Object> entry : this.plan.getLogicalPlan().getAttributes() .entrySet()) { attributes.put(entry.getKey().getSimpleName(), entry.getValue()); } JSONObject customMetrics = new JSONObject(); for (Map.Entry<String, Map<String, Object>> entry : latestLogicalMetrics.entrySet()) { customMetrics.put(entry.getKey(), new JSONArray(entry.getValue().keySet())); } top.put(APP_META_KEY_ATTRIBUTES, attributes); top.put(APP_META_KEY_CUSTOM_METRICS, customMetrics); os.write(top.toString().getBytes()); } catch (JSONException ex) { throw new RuntimeException(ex); } finally { os.close(); } Path origPath = new Path(this.vars.appPath, APP_META_FILENAME); fs.rename(path, origPath); } finally { fs.close(); } }
From source file:com.datatorrent.stram.util.FSJsonLineFile.java
License:Apache License
public FSJsonLineFile(Path path, FsPermission permission) throws IOException { fs = FileSystem.newInstance(path.toUri(), new Configuration()); FSDataOutputStream myos;/*from w w w. j a v a2s. c o m*/ if (fs.exists(path)) { try { // happens if not the first application attempt myos = fs.append(path); } catch (IOException ex) { LOG.warn("Caught exception (OK during unit test): {}", ex.getMessage()); myos = FileSystem.create(fs, path, permission); } } else { myos = FileSystem.create(fs, path, permission); } os = myos; this.objectMapper = (new JSONSerializationProvider()).getContext(null); }
From source file:com.datatorrent.stram.util.FSPartFileCollection.java
License:Apache License
public void setup() throws IOException { if (basePath.startsWith("file:")) { isLocalMode = true;//from ww w. j av a 2s .c om localBasePath = basePath.substring(5); (new File(localBasePath)).mkdirs(); } fs = FileSystem.newInstance(new Path(basePath).toUri(), new Configuration()); Path pa = new Path(basePath, META_FILE); if (isLocalMode) { metaOs = new FSDataOutputStream(new FileOutputStream(localBasePath + "/" + META_FILE), null); } else { metaOs = fs.create(pa); } pa = new Path(basePath, INDEX_FILE); if (isLocalMode) { indexOutStr = new FSDataOutputStream(new FileOutputStream(localBasePath + "/" + INDEX_FILE), null); } else { indexOutStr = fs.create(pa); } }
From source file:com.example.PassthroughFailOperator.java
License:Apache License
/** * Loads file from HDFS and sets {@link #killed} flag if it already exists * * @param context/* w w w . ja va 2s . co m*/ */ @Override public void setup(Context.OperatorContext context) { super.setup(context); String appId = context.getValue(Context.DAGContext.APPLICATION_ID); filePath = directoryPath + "/" + appId; LOG.info("FilePath: " + filePath); filePathObj = new Path(filePath); try { hdfs = FileSystem.newInstance(filePathObj.toUri(), new Configuration()); } catch (IOException e) { e.printStackTrace(); } try { if (hdfs.exists(filePathObj)) { killed = true; LOG.info("file already exists -> Operator has been killed before"); } } catch (IOException e) { e.printStackTrace(); } }
From source file:com.flipkart.fdp.migration.distcp.codec.DCMCodecFactory.java
License:Apache License
public static FileSystem getFilesystem(Configuration conf, ConnectionConfig config, String fsURI) throws Exception { if (config.getSecurityType() == DCMConstants.SecurityType.KERBEROS) return FileSystem.newInstance(new URI(fsURI), conf); else/* ww w. j ava 2s.c om*/ return FileSystem.newInstance(new URI(fsURI), conf, config.getUserName()); }
From source file:com.flipkart.fdp.migration.distcp.utils.FileCountDriver.java
License:Apache License
private FileSystem getFileSystem(String nn) throws IOException, URISyntaxException { FileSystem fileSystem = FileSystem.newInstance(new URI(nn), configuration); return fileSystem; }
From source file:com.gemstone.gemfire.cache.hdfs.internal.HDFSStoreImpl.java
License:Apache License
private FileSystem createFileSystem(Configuration hconf, String configFile, boolean forceNew) throws IOException { FileSystem filesystem = null; // load hdfs client config file if specified. The path is on local file // system/*from www .j a va 2s .c o m*/ if (configFile != null) { if (logger.isDebugEnabled()) { logger.debug("{}Adding resource config file to hdfs configuration:" + configFile, logPrefix); } hconf.addResource(new Path(configFile)); if (!new File(configFile).exists()) { logger.warn(LocalizedMessage.create(LocalizedStrings.HOPLOG_HDFS_CLIENT_CONFIG_FILE_ABSENT, configFile)); } } // This setting disables shutdown hook for file system object. Shutdown // hook may cause FS object to close before the cache or store and // unpredictable behavior. This setting is provided for GFXD like server // use cases where FS close is managed by a server. This setting is not // supported by old versions of hadoop, HADOOP-4829 hconf.setBoolean("fs.automatic.close", false); // Hadoop has a configuration parameter io.serializations that is a list of serialization // classes which can be used for obtaining serializers and deserializers. This parameter // by default contains avro classes. When a sequence file is created, it calls // SerializationFactory.getSerializer(keyclass). This internally creates objects using // reflection of all the classes that were part of io.serializations. But since, there is // no avro class available it throws an exception. // Before creating a sequenceFile, override the io.serializations parameter and pass only the classes // that are important to us. hconf.setStrings("io.serializations", new String[] { "org.apache.hadoop.io.serializer.WritableSerialization" }); // create writer SchemaMetrics.configureGlobally(hconf); String nameNodeURL = null; if ((nameNodeURL = getNameNodeURL()) == null) { nameNodeURL = hconf.get("fs.default.name"); } URI namenodeURI = URI.create(nameNodeURL); //if (! GemFireCacheImpl.getExisting().isHadoopGfxdLonerMode()) { String authType = hconf.get("hadoop.security.authentication"); //The following code handles Gemfire XD with secure HDFS //A static set is used to cache all known secure HDFS NameNode urls. UserGroupInformation.setConfiguration(hconf); //Compare authentication method ignoring case to make GFXD future version complaint //At least version 2.0.2 starts complaining if the string "kerberos" is not in all small case. //However it seems current version of hadoop accept the authType in any case if (authType.equalsIgnoreCase("kerberos")) { String principal = hconf.get(HoplogConfig.KERBEROS_PRINCIPAL); String keyTab = hconf.get(HoplogConfig.KERBEROS_KEYTAB_FILE); if (!PERFORM_SECURE_HDFS_CHECK) { if (logger.isDebugEnabled()) logger.debug("{}Ignore secure hdfs check", logPrefix); } else { if (!secureNameNodes.contains(nameNodeURL)) { if (logger.isDebugEnabled()) logger.debug("{}Executing secure hdfs check", logPrefix); try { filesystem = FileSystem.newInstance(namenodeURI, hconf); //Make sure no IOExceptions are generated when accessing insecure HDFS. filesystem.listFiles(new Path("/"), false); throw new HDFSIOException( "Gemfire XD HDFS client and HDFS cluster security levels do not match. The configured HDFS Namenode is not secured."); } catch (IOException ex) { secureNameNodes.add(nameNodeURL); } finally { //Close filesystem to avoid resource leak if (filesystem != null) { closeFileSystemIgnoreError(filesystem); } } } } // check to ensure the namenode principal is defined String nameNodePrincipal = hconf.get("dfs.namenode.kerberos.principal"); if (nameNodePrincipal == null) { throw new IOException(LocalizedStrings.GF_KERBEROS_NAMENODE_PRINCIPAL_UNDEF.toLocalizedString()); } // ok, the user specified a gfxd principal so we will try to login if (principal != null) { //If NameNode principal is the same as Gemfire XD principal, there is a //potential security hole String regex = "[/@]"; if (nameNodePrincipal != null) { String HDFSUser = nameNodePrincipal.split(regex)[0]; String GFXDUser = principal.split(regex)[0]; if (HDFSUser.equals(GFXDUser)) { logger.warn( LocalizedMessage.create(LocalizedStrings.HDFS_USER_IS_SAME_AS_GF_USER, GFXDUser)); } } // a keytab must exist if the user specifies a principal if (keyTab == null) { throw new IOException(LocalizedStrings.GF_KERBEROS_KEYTAB_UNDEF.toLocalizedString()); } // the keytab must exist as well File f = new File(keyTab); if (!f.exists()) { throw new FileNotFoundException( LocalizedStrings.GF_KERBEROS_KEYTAB_FILE_ABSENT.toLocalizedString(f.getAbsolutePath())); } //Authenticate Gemfire XD principal to Kerberos KDC using Gemfire XD keytab file String principalWithValidHost = SecurityUtil.getServerPrincipal(principal, ""); UserGroupInformation.loginUserFromKeytab(principalWithValidHost, keyTab); } else { logger.warn(LocalizedMessage.create(LocalizedStrings.GF_KERBEROS_PRINCIPAL_UNDEF)); } } //} filesystem = getFileSystemFactory().create(namenodeURI, hconf, forceNew); if (logger.isDebugEnabled()) { logger.debug("{}Initialized FileSystem linked to " + filesystem.getUri() + " " + filesystem.hashCode(), logPrefix); } return filesystem; }
From source file:com.mellanox.r4h.MiniDFSCluster.java
License:Apache License
/** * Get another FileSystem instance that is different from FileSystem.get(conf). * This simulating different threads working on different FileSystem instances. *//* w w w .j av a 2 s. com*/ public FileSystem getNewFileSystemInstance(int nnIndex) throws IOException { return FileSystem.newInstance(getURI(nnIndex), nameNodes[nnIndex].conf); }
From source file:com.pinterest.terrapin.hadoop.BaseUploader.java
License:Apache License
/** * Validates the first non-empty partition hfile has right partitioning function. * It reads several keys, then calculates the partition according to the partitioning function * client offering. If the calculated partition number is different with actual partition number * an exception is thrown. If all partition hfiles are empty, an exception is thrown. * * @param parts full absolute path for all partitions * @param partitionerType type of paritioning function * @param numShards total number of partitions * @throws IOException if something goes wrong when reading the hfiles * @throws IllegalArgumentException if the partitioner type is wrong or all partitions are empty */// w w w. j ava 2 s. c o m public void validate(List<Path> parts, PartitionerType partitionerType, int numShards) throws IOException { boolean hasNonEmptyPartition = false; HColumnDescriptor columnDescriptor = new HColumnDescriptor(); // Disable block cache to ensure it reads the actual file content. columnDescriptor.setBlockCacheEnabled(false); for (int shardIndex = 0; shardIndex < parts.size(); shardIndex++) { Path fileToBeValidated = parts.get(shardIndex); HFile.Reader reader = null; try { FileSystem fs = FileSystem.newInstance(fileToBeValidated.toUri(), conf); CacheConfig cc = new CacheConfig(conf, columnDescriptor); reader = HFile.createReader(fs, fileToBeValidated, cc); Partitioner partitioner = PartitionerFactory.getPartitioner(partitionerType); byte[] rowKey = reader.getFirstRowKey(); if (rowKey == null) { LOG.warn(String.format("empty partition %s", fileToBeValidated.toString())); reader.close(); continue; } hasNonEmptyPartition = true; BytesWritable key = new BytesWritable(rowKey); int partition = partitioner.getPartition(key, null, numShards); if (partition != shardIndex) { throw new IllegalArgumentException( String.format("wrong partition type %s for key %s in partition %d, expected %d", partitionerType.toString(), new String(key.getBytes()), shardIndex, partition)); } } finally { if (reader != null) { reader.close(); } } } if (!hasNonEmptyPartition) { throw new IllegalArgumentException("all partitions are empty"); } }