List of usage examples for org.apache.hadoop.fs FileSystem getDefaultUri
public static URI getDefaultUri(Configuration conf)
From source file:TestParascaleFileSystem.java
License:Apache License
public void testGetDefaultURI() { FileSystem.getDefaultUri(getConf()); }
From source file:co.cask.tephra.persist.CommitMarkerCodecTest.java
License:Apache License
@BeforeClass public static void setupBeforeClass() throws Exception { Configuration hConf = new Configuration(); hConf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, TMP_FOLDER.newFolder().getAbsolutePath()); dfsCluster = new MiniDFSCluster.Builder(hConf).numDataNodes(1).build(); conf = new Configuration(dfsCluster.getFileSystem().getConf()); fs = FileSystem.newInstance(FileSystem.getDefaultUri(conf), conf); }
From source file:co.cask.tephra.persist.HDFSTransactionLogTest.java
License:Apache License
private void testTransactionLogSync(int totalCount, int batchSize, boolean withMarker, boolean isComplete) throws Exception { List<TransactionEdit> edits = TransactionEditUtil.createRandomEdits(totalCount); long timestamp = System.currentTimeMillis(); Configuration configuration = getConfiguration(); FileSystem fs = FileSystem.newInstance(FileSystem.getDefaultUri(configuration), configuration); SequenceFile.Writer writer = getSequenceFileWriter(configuration, fs, timestamp, withMarker); AtomicLong logSequence = new AtomicLong(); HDFSTransactionLog transactionLog = getHDFSTransactionLog(configuration, fs, timestamp); AbstractTransactionLog.Entry entry;/*from w ww . j av a2s . co m*/ for (int i = 0; i < totalCount - batchSize; i += batchSize) { if (withMarker) { writeNumWrites(writer, batchSize); } for (int j = 0; j < batchSize; j++) { entry = new AbstractTransactionLog.Entry(new LongWritable(logSequence.getAndIncrement()), edits.get(j)); writer.append(entry.getKey(), entry.getEdit()); } writer.syncFs(); } if (withMarker) { writeNumWrites(writer, batchSize); } for (int i = totalCount - batchSize; i < totalCount - 1; i++) { entry = new AbstractTransactionLog.Entry(new LongWritable(logSequence.getAndIncrement()), edits.get(i)); writer.append(entry.getKey(), entry.getEdit()); } entry = new AbstractTransactionLog.Entry(new LongWritable(logSequence.getAndIncrement()), edits.get(totalCount - 1)); if (isComplete) { writer.append(entry.getKey(), entry.getEdit()); } else { byte[] bytes = Longs.toByteArray(entry.getKey().get()); writer.appendRaw(bytes, 0, bytes.length, new SequenceFile.ValueBytes() { @Override public void writeUncompressedBytes(DataOutputStream outStream) throws IOException { byte[] test = new byte[] { 0x2 }; outStream.write(test, 0, 1); } @Override public void writeCompressedBytes(DataOutputStream outStream) throws IllegalArgumentException, IOException { // no-op } @Override public int getSize() { // mimic size longer than the actual byte array size written, so we would reach EOF return 12; } }); } writer.syncFs(); Closeables.closeQuietly(writer); // now let's try to read this log TransactionLogReader reader = transactionLog.getReader(); int syncedEdits = 0; while (reader.next() != null) { // testing reading the transaction edits syncedEdits++; } if (isComplete) { Assert.assertEquals(totalCount, syncedEdits); } else { Assert.assertEquals(totalCount - batchSize, syncedEdits); } }
From source file:co.cask.tephra.persist.HDFSTransactionStateStorage.java
License:Apache License
@Override protected void startUp() throws Exception { Preconditions.checkState(configuredSnapshotDir != null, "Snapshot directory is not configured. Please set " + TxConstants.Manager.CFG_TX_SNAPSHOT_DIR + " in configuration."); String hdfsUser = hConf.get(TxConstants.Manager.CFG_TX_HDFS_USER); if (hdfsUser == null || UserGroupInformation.isSecurityEnabled()) { if (hdfsUser != null && LOG.isDebugEnabled()) { LOG.debug("Ignoring configuration {}={}, running on secure Hadoop", TxConstants.Manager.CFG_TX_HDFS_USER, hdfsUser); }// www .j a v a2s . c om // NOTE: we can start multiple times this storage. As hdfs uses per-jvm cache, we want to create new fs instead // of getting closed one fs = FileSystem.newInstance(FileSystem.getDefaultUri(hConf), hConf); } else { fs = FileSystem.newInstance(FileSystem.getDefaultUri(hConf), hConf, hdfsUser); } snapshotDir = new Path(configuredSnapshotDir); LOG.info("Using snapshot dir " + snapshotDir); }
From source file:com.alibaba.jstorm.hdfs.common.security.AutoHDFS.java
License:Apache License
@SuppressWarnings("unchecked") protected byte[] getHadoopCredentials(Map conf) { try {/*from www. j a va 2s . c o m*/ if (UserGroupInformation.isSecurityEnabled()) { final Configuration configuration = new Configuration(); login(configuration); final String topologySubmitterUser = (String) conf.get(Config.TOPOLOGY_SUBMITTER_PRINCIPAL); final URI nameNodeURI = conf.containsKey(TOPOLOGY_HDFS_URI) ? new URI(conf.get(TOPOLOGY_HDFS_URI).toString()) : FileSystem.getDefaultUri(configuration); UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); final UserGroupInformation proxyUser = UserGroupInformation.createProxyUser(topologySubmitterUser, ugi); Credentials creds = (Credentials) proxyUser.doAs(new PrivilegedAction<Object>() { @Override public Object run() { try { FileSystem fileSystem = FileSystem.get(nameNodeURI, configuration); Credentials credential = proxyUser.getCredentials(); fileSystem.addDelegationTokens(hdfsPrincipal, credential); LOG.info("Delegation tokens acquired for user {}", topologySubmitterUser); return credential; } catch (IOException e) { throw new RuntimeException(e); } } }); ByteArrayOutputStream bao = new ByteArrayOutputStream(); ObjectOutputStream out = new ObjectOutputStream(bao); creds.write(out); out.flush(); out.close(); return bao.toByteArray(); } else { throw new RuntimeException("Security is not enabled for HDFS"); } } catch (Exception ex) { throw new RuntimeException("Failed to get delegation tokens.", ex); } }
From source file:com.cloudera.CacheTool.java
License:Apache License
public static void main(String[] args) throws Exception { conf = new Configuration(); conf.addResource(new Path("/home/james/hdfs-conf/hdfs-site.xml")); conf.addResource(new Path("/home/james/hdfs-conf/core-site.xml")); URI uri = FileSystem.getDefaultUri(conf); final FileSystem fs = FileSystem.get(uri, conf); for (int i = 0; i < 8000; i += 10) { final int i_copy = i; pool.submit(new Runnable() { public void run() { for (int j = 0; j < 10; j++) { try { createFile(fs, new Path("/home/james/large" + (i_copy + j)), 1024 * 1024); } catch (IOException ioe) { System.out.println(ioe); }/*from w ww . ja va2s . c o m*/ } } }); } pool.shutdown(); pool.awaitTermination(1, TimeUnit.DAYS); long start = Time.monotonicNow(); Random r = new Random(0); for (int i = 0; i < 100; i++) { FSDataInputStream fdis = fs.open(new Path("/home/james/large" + r.nextInt(8000)), 512); byte[] buffer = new byte[512]; for (int j = 0; j < 100; j++) { int offset = r.nextInt(1024 * 1024 - 511); fdis.read(offset, buffer, 0, 512); } } System.out.println("Time taken for 10000 random 512 byte reads: " + (Time.monotonicNow() - start) / 1000.0); }
From source file:com.cloudera.impala.common.FileSystemUtil.java
License:Apache License
public static DistributedFileSystem getDistributedFileSystem() throws IOException { return getDistributedFileSystem(new Path(FileSystem.getDefaultUri(CONF))); }
From source file:com.cloudera.impala.common.FileSystemUtil.java
License:Apache License
/** * Fully-qualifies the given path based on the FileSystem configuration. If the given * path is already fully qualified, a new Path object with the same location will be * returned.//from w ww . j a v a2 s. c om */ public static Path createFullyQualifiedPath(Path location) { return location.makeQualified(FileSystem.getDefaultUri(CONF), location); }
From source file:com.cloudera.oryx.common.servcomp.Namespaces.java
License:Open Source License
private Namespaces() { if (isLocalData()) { prefix = "file:"; } else {// w w w .j a v a2s. c o m URI defaultURI = FileSystem.getDefaultUri(OryxConfiguration.get()); String host = defaultURI.getHost(); Preconditions.checkNotNull(host, "Hadoop FS has no host in '%s'. Did you intend to set model.local-data=true?", defaultURI); int port = defaultURI.getPort(); if (port > 0) { prefix = "hdfs://" + host + ':' + port; } else { prefix = "hdfs://" + host; } } log.info("Namespace prefix: {}", prefix); }
From source file:com.ibm.bi.dml.api.DMLScript.java
License:Open Source License
/** * //from w ww .j a va 2 s . co m * @param config * @throws IOException * @throws DMLRuntimeException */ private static void checkSecuritySetup(DMLConfig config) throws IOException, DMLRuntimeException { //analyze local configuration String userName = System.getProperty("user.name"); HashSet<String> groupNames = new HashSet<String>(); try { //check existence, for backwards compatibility to < hadoop 0.21 if (UserGroupInformation.class.getMethod("getCurrentUser") != null) { String[] groups = UserGroupInformation.getCurrentUser().getGroupNames(); for (String g : groups) groupNames.add(g); } } catch (Exception ex) { } //analyze hadoop configuration JobConf job = ConfigurationManager.getCachedJobConf(); boolean localMode = InfrastructureAnalyzer.isLocalMode(job); String taskController = job.get("mapred.task.tracker.task-controller", "org.apache.hadoop.mapred.DefaultTaskController"); String ttGroupName = job.get("mapreduce.tasktracker.group", "null"); String perm = job.get(MRConfigurationNames.DFS_PERMISSIONS, "null"); //note: job.get("dfs.permissions.supergroup",null); URI fsURI = FileSystem.getDefaultUri(job); //determine security states boolean flagDiffUser = !(taskController.equals("org.apache.hadoop.mapred.LinuxTaskController") //runs map/reduce tasks as the current user || localMode // run in the same JVM anyway || groupNames.contains(ttGroupName)); //user in task tracker group boolean flagLocalFS = fsURI == null || fsURI.getScheme().equals("file"); boolean flagSecurity = perm.equals("yes"); LOG.debug("SystemML security check: " + "local.user.name = " + userName + ", " + "local.user.groups = " + ProgramConverter.serializeStringCollection(groupNames) + ", " + "mapred.job.tracker = " + job.get("mapred.job.tracker") + ", " + "mapred.task.tracker.task-controller = " + taskController + "," + "mapreduce.tasktracker.group = " + ttGroupName + ", " + "fs.default.name = " + ((fsURI != null) ? fsURI.getScheme() : "null") + ", " + MRConfigurationNames.DFS_PERMISSIONS + " = " + perm); //print warning if permission issues possible if (flagDiffUser && (flagLocalFS || flagSecurity)) { LOG.warn("Cannot run map/reduce tasks as user '" + userName + "'. Using tasktracker group '" + ttGroupName + "'."); } //validate external filenames working directories String localtmpdir = config.getTextValue(DMLConfig.LOCAL_TMP_DIR); String hdfstmpdir = config.getTextValue(DMLConfig.SCRATCH_SPACE); if (!LocalFileUtils.validateExternalFilename(localtmpdir, false)) throw new DMLRuntimeException("Invalid (non-trustworthy) local working directory."); if (!LocalFileUtils.validateExternalFilename(hdfstmpdir, true)) throw new DMLRuntimeException("Invalid (non-trustworthy) hdfs working directory."); }