Example usage for org.apache.hadoop.fs FileSystem getWorkingDirectory

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileSystem getWorkingDirectory.

Prototype

public abstract Path getWorkingDirectory();

Source Link

Document

Get the current working directory for the given FileSystem

Usage

From source file:io.dstream.tez.utils.HdfsSerializerUtils.java

License:Apache License

/**
 * Will serialize object to HDFS returning its {@link Path}.
 *
 * @param source/*from   ww  w . j a v a 2 s .  co m*/
 * @param fs
 * @param targetPath
 * @return
 */
public static Path serialize(Object source, FileSystem fs, Path targetPath) {
    Assert.notNull(targetPath, "'targetPath' must not be null");
    Assert.notNull(fs, "'fs' must not be null");
    Assert.notNull(source, "'source' must not be null");

    Path resultPath = targetPath.makeQualified(fs.getUri(), fs.getWorkingDirectory());
    OutputStream targetOutputStream = null;
    try {
        targetOutputStream = fs.create(targetPath);
        SerializationUtils.serialize(source, targetOutputStream);
    } catch (Exception e) {
        throw new IllegalStateException("Failed to serialize " + source + " to " + resultPath, e);
    }
    return resultPath;
}

---

From source file:io.gzinga.hadoop.TestHadoopGZipRandomAccess.java

License:Apache License

@Test
public void testGZipOutputStream() {
    try {/*from  ww  w . j a v a 2 s  .  com*/
        Configuration conf = new Configuration();
        conf.set("fs.defaultFS", "file:///");
        FileSystem fs = FileSystem.get(conf);
        fs.mkdirs(new Path("target/test"));
        GZipOutputStreamRandomAccess gzip = new GZipOutputStreamRandomAccess(
                fs.create(new Path("target/test/testfile")));
        byte[] str = "This is line\n".getBytes();
        for (int i = 1; i <= 10000; i++) {
            if (i % 100 == 0) {
                gzip.addOffset(i / 100l);
            }
            gzip.write(str);
        }
        Assert.assertEquals(gzip.getOffsetMap().size(), 100);
        gzip.close();
        fs.copyFromLocalFile(new Path(fs.getWorkingDirectory().toString() + "/target/test-classes/testfile1"),
                new Path("target/test/testfile1"));
        FSDataInputStream fin = fs.open(new Path("target/test/testfile"));
        long len = fs.getFileStatus(new Path("target/test/testfile")).getLen();
        SeekableGZipDataInputStream sin = new SeekableGZipDataInputStream(fin, len);
        Assert.assertTrue(GZipInputStreamRandomAccess.isGzipRandomOutputFile(sin));
        fin = fs.open(new Path("target/test/testfile1"));
        sin = new SeekableGZipDataInputStream(fin, len);
        Assert.assertFalse(GZipInputStreamRandomAccess.isGzipRandomOutputFile(sin));
        fin = fs.open(new Path("target/test/testfile"));
        sin = new SeekableGZipDataInputStream(fin, len);
        GZipInputStreamRandomAccess gzin = new GZipInputStreamRandomAccess(sin);
        Assert.assertEquals(gzin.getMetadata().size(), 100);
        Assert.assertTrue(gzin.getMetadata().containsKey(1l));
        Assert.assertTrue(gzin.getMetadata().containsKey(100l));
        Assert.assertFalse(gzin.getMetadata().containsKey(200l));
        gzin.jumpToIndex(50l);
        int count1 = 0;
        while (true) {
            int l = gzin.read();
            if (l == -1) {
                break;
            }
            count1++;
        }
        gzin.jumpToIndex(60l);
        int count2 = 0;
        while (true) {
            int l = gzin.read();
            if (l == -1) {
                break;
            }
            count2++;
        }
        Assert.assertTrue(count1 > count2);
        gzin.close();
    } catch (Exception e) {
        e.printStackTrace();
        Assert.fail();
    }
}

---

From source file:nl.basjes.hadoop.io.compress.TestSplittableCodecSeams.java

License:Apache License

/**
 * Write the specified number of records to file in test dir using codec.
 * Records are simply lines random ASCII
 *//*  www .j av a 2s  .c  o m*/
private static Path writeSplitTestFile(final Configuration conf,
        final Class<? extends SplittableCompressionCodec> codecClass, final long records,
        final int recordLength, final int trailingSizeJitter, final int randomizeEveryNChars)
        throws IOException {

    RAND.setSeed(1); // Make the tests better reproducable

    final FileSystem fs = FileSystem.getLocal(conf);
    final SplittableCompressionCodec codec = ReflectionUtils.newInstance(codecClass, conf);

    final Path wd = new Path(new Path(System.getProperty("test.build.data", "/tmp")).makeQualified(fs.getUri(),
            fs.getWorkingDirectory()), codec.getClass().getSimpleName());

    final Path file = new Path(wd,
            "test-" + records + "-" + recordLength + "-" + trailingSizeJitter + codec.getDefaultExtension());
    DataOutputStream out = null;
    final Compressor cmp = CodecPool.getCompressor(codec);
    try {
        out = new DataOutputStream(codec.createOutputStream(fs.create(file, true), cmp));

        for (long seq = 1; seq <= records; ++seq) {
            final String line = randomGibberish(
                    recordLength + (trailingSizeJitter > 0 ? RAND.nextInt(trailingSizeJitter) : 0),
                    randomizeEveryNChars) + "\n";
            // There must be a simpler way to output ACSII instead of 2 byte UNICODE
            out.writeBytes(new String(line.getBytes("UTF-8"), "US-ASCII"));
        }
    } finally {
        IOUtils.cleanup(LOG, out);
        CodecPool.returnCompressor(cmp);
    }
    return file;
}

---

From source file:org.anon.smart.d2cache.store.fileStore.hadoop.HadoopFileStoreTransaction.java

License:Open Source License

@Override
public void commit() throws CtxException {

    FileSystem hdfs = ((HadoopFileStoreConnection) _connection).getHadoopFS();

    assertion().assertNotNull(hdfs, "Hadoop FileSystem is null");

    String repo = hdfs.getWorkingDirectory().toUri().toString();

    for (Object fi : files.keySet()) {
        try {//from  ww  w  .  j  ava  2 s  .  co  m

            String filePath = (String) fi;
            String[] tmp = filePath.split("/");
            String fileName = tmp[tmp.length - 1];

            Path fldr = new Path(files.get(fi));
            if (!hdfs.exists(fldr))
                hdfs.mkdirs(fldr);

            hdfs.copyFromLocalFile(true, new Path(filePath), new Path(files.get(fi) + "/" + fileName));

        } catch (Exception e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        }
    }

}

---

From source file:org.apache.accumulo.core.file.rfile.PrintInfo.java

License:Apache License

@Override
public void execute(final String[] args) throws Exception {
    Opts opts = new Opts();
    opts.parseArgs(PrintInfo.class.getName(), args);
    if (opts.files.isEmpty()) {
        System.err.println("No files were given");
        System.exit(-1);//from  www.j a  va 2s  .  c  o  m
    }

    Configuration conf = new Configuration();
    for (String confFile : opts.configFiles) {
        log.debug("Adding Hadoop configuration file " + confFile);
        conf.addResource(new Path(confFile));
    }

    FileSystem hadoopFs = FileSystem.get(conf);
    FileSystem localFs = FileSystem.getLocal(conf);

    LogHistogram kvHistogram = new LogHistogram();

    KeyStats dataKeyStats = new KeyStats();
    KeyStats indexKeyStats = new KeyStats();

    for (String arg : opts.files) {
        Path path = new Path(arg);
        FileSystem fs;
        if (arg.contains(":"))
            fs = path.getFileSystem(conf);
        else {
            log.warn("Attempting to find file across filesystems. Consider providing URI instead of path");
            fs = hadoopFs.exists(path) ? hadoopFs : localFs; // fall back to local
        }
        System.out.println(
                "Reading file: " + path.makeQualified(fs.getUri(), fs.getWorkingDirectory()).toString());

        CachableBlockFile.Reader _rdr = new CachableBlockFile.Reader(fs, path, conf, null, null,
                SiteConfiguration.getInstance(DefaultConfiguration.getInstance()));
        Reader iter = new RFile.Reader(_rdr);
        MetricsGatherer<Map<String, ArrayList<VisibilityMetric>>> vmg = new VisMetricsGatherer();

        if (opts.vis || opts.hash)
            iter.registerMetrics(vmg);

        iter.printInfo();
        System.out.println();
        org.apache.accumulo.core.file.rfile.bcfile.PrintInfo.main(new String[] { arg });

        Map<String, ArrayList<ByteSequence>> localityGroupCF = null;

        if (opts.histogram || opts.dump || opts.vis || opts.hash || opts.keyStats) {
            localityGroupCF = iter.getLocalityGroupCF();

            FileSKVIterator dataIter;
            if (opts.useSample) {
                dataIter = iter.getSample();

                if (dataIter == null) {
                    System.out.println("ERROR : This rfile has no sample data");
                    return;
                }
            } else {
                dataIter = iter;
            }

            if (opts.keyStats) {
                FileSKVIterator indexIter = iter.getIndex();
                while (indexIter.hasTop()) {
                    indexKeyStats.add(indexIter.getTopKey());
                    indexIter.next();
                }
            }

            for (Entry<String, ArrayList<ByteSequence>> cf : localityGroupCF.entrySet()) {

                dataIter.seek(new Range((Key) null, (Key) null), cf.getValue(), true);
                while (dataIter.hasTop()) {
                    Key key = dataIter.getTopKey();
                    Value value = dataIter.getTopValue();
                    if (opts.dump) {
                        System.out.println(key + " -> " + value);
                        if (System.out.checkError())
                            return;
                    }
                    if (opts.histogram) {
                        kvHistogram.add(key.getSize() + value.getSize());
                    }
                    if (opts.keyStats) {
                        dataKeyStats.add(key);
                    }
                    dataIter.next();
                }
            }
        }

        iter.close();

        if (opts.vis || opts.hash) {
            System.out.println();
            vmg.printMetrics(opts.hash, "Visibility", System.out);
        }

        if (opts.histogram) {
            System.out.println();
            kvHistogram.print("");
        }

        if (opts.keyStats) {
            System.out.println();
            System.out.println("Statistics for keys in data :");
            dataKeyStats.print("\t");
            System.out.println();
            System.out.println("Statistics for keys in index :");
            indexKeyStats.print("\t");
        }
        // If the output stream has closed, there is no reason to keep going.
        if (System.out.checkError())
            return;
    }
}

---

From source file:org.apache.accumulo.server.util.TabletOperations.java

License:Apache License

public static String createTabletDirectory(VolumeManager fs, String tableId, Text endRow) {
    String lowDirectory;/*w ww . j a v a  2 s  .  c om*/

    UniqueNameAllocator namer = UniqueNameAllocator.getInstance();
    String volume = fs.choose(ServerConstants.getTablesDirs());

    while (true) {
        try {
            if (endRow == null) {
                lowDirectory = Constants.DEFAULT_TABLET_LOCATION;
                Path lowDirectoryPath = new Path(volume + "/" + tableId + "/" + lowDirectory);
                if (fs.exists(lowDirectoryPath) || fs.mkdirs(lowDirectoryPath)) {
                    FileSystem pathFs = fs.getVolumeByPath(lowDirectoryPath).getFileSystem();
                    return lowDirectoryPath.makeQualified(pathFs.getUri(), pathFs.getWorkingDirectory())
                            .toString();
                }
                log.warn("Failed to create " + lowDirectoryPath + " for unknown reason");
            } else {
                lowDirectory = "/" + Constants.GENERATED_TABLET_DIRECTORY_PREFIX + namer.getNextName();
                Path lowDirectoryPath = new Path(volume + "/" + tableId + "/" + lowDirectory);
                if (fs.exists(lowDirectoryPath))
                    throw new IllegalStateException("Dir exist when it should not " + lowDirectoryPath);
                if (fs.mkdirs(lowDirectoryPath)) {
                    FileSystem lowDirectoryFs = fs.getVolumeByPath(lowDirectoryPath).getFileSystem();
                    return lowDirectoryPath
                            .makeQualified(lowDirectoryFs.getUri(), lowDirectoryFs.getWorkingDirectory())
                            .toString();
                }
            }
        } catch (IOException e) {
            log.warn(e);
        }

        log.warn("Failed to create dir for tablet in table " + tableId + " in volume " + volume
                + " + will retry ...");
        UtilWaitThread.sleep(3000);

    }
}

---

From source file:org.apache.accumulo.tserver.tablet.Tablet.java

License:Apache License

private static String createTabletDirectory(VolumeManager fs, String tableId, Text endRow) {
    String lowDirectory;//from   ww  w  .  j av a 2s  .c  o m

    UniqueNameAllocator namer = UniqueNameAllocator.getInstance();
    String volume = fs.choose(Optional.of(tableId), ServerConstants.getBaseUris()) + Constants.HDFS_TABLES_DIR
            + Path.SEPARATOR;

    while (true) {
        try {
            if (endRow == null) {
                lowDirectory = Constants.DEFAULT_TABLET_LOCATION;
                Path lowDirectoryPath = new Path(volume + "/" + tableId + "/" + lowDirectory);
                if (fs.exists(lowDirectoryPath) || fs.mkdirs(lowDirectoryPath)) {
                    FileSystem pathFs = fs.getVolumeByPath(lowDirectoryPath).getFileSystem();
                    return lowDirectoryPath.makeQualified(pathFs.getUri(), pathFs.getWorkingDirectory())
                            .toString();
                }
                log.warn("Failed to create " + lowDirectoryPath + " for unknown reason");
            } else {
                lowDirectory = "/" + Constants.GENERATED_TABLET_DIRECTORY_PREFIX + namer.getNextName();
                Path lowDirectoryPath = new Path(volume + "/" + tableId + "/" + lowDirectory);
                if (fs.exists(lowDirectoryPath))
                    throw new IllegalStateException("Dir exist when it should not " + lowDirectoryPath);
                if (fs.mkdirs(lowDirectoryPath)) {
                    FileSystem lowDirectoryFs = fs.getVolumeByPath(lowDirectoryPath).getFileSystem();
                    return lowDirectoryPath
                            .makeQualified(lowDirectoryFs.getUri(), lowDirectoryFs.getWorkingDirectory())
                            .toString();
                }
            }
        } catch (IOException e) {
            log.warn(e);
        }

        log.warn("Failed to create dir for tablet in table " + tableId + " in volume " + volume
                + " + will retry ...");
        sleepUninterruptibly(3, TimeUnit.SECONDS);

    }
}

---

From source file:org.apache.blur.mapreduce.lib.BlurMapReduceUtil.java

License:Apache License

/**
 * Adds all the jars in the same path as the blur jar files.
 * //from  w ww .  ja  va2s  .  c om
 * @param conf
 * @throws IOException
 */
public static void addAllJarsInBlurLib(Configuration conf) throws IOException {
    FileSystem localFs = FileSystem.getLocal(conf);
    Set<String> jars = new HashSet<String>();
    jars.addAll(conf.getStringCollection("tmpjars"));

    String property = System.getProperty("java.class.path");
    String[] files = property.split("\\:");

    String blurLibPath = getPath("blur-", files);
    if (blurLibPath == null) {
        return;
    }
    List<String> pathes = getPathes(blurLibPath, files);
    for (String pathStr : pathes) {
        Path path = new Path(pathStr);
        if (!localFs.exists(path)) {
            LOG.warn("Could not validate jar file " + path);
            continue;
        }
        jars.add(path.makeQualified(localFs.getUri(), localFs.getWorkingDirectory()).toString());
    }
    if (jars.isEmpty()) {
        return;
    }
    conf.set("tmpjars", StringUtils.arrayToString(jars.toArray(new String[0])));
}

---

From source file:org.apache.blur.mapreduce.lib.BlurMapReduceUtil.java

License:Apache License

/**
 * Add the jars containing the given classes to the job's configuration such
 * that JobClient will ship them to the cluster and add them to the
 * DistributedCache./* w w  w .  ja  v  a 2s.  c  om*/
 */
public static void addDependencyJars(Configuration conf, Class<?>... classes) throws IOException {
    FileSystem localFs = FileSystem.getLocal(conf);
    Set<String> jars = new HashSet<String>();
    // Add jars that are already in the tmpjars variable
    jars.addAll(conf.getStringCollection("tmpjars"));

    // Add jars containing the specified classes
    for (Class<?> clazz : classes) {
        if (clazz == null) {
            continue;
        }

        String pathStr = findOrCreateJar(clazz);
        if (pathStr == null) {
            LOG.warn("Could not find jar for class " + clazz + " in order to ship it to the cluster.");
            continue;
        }
        Path path = new Path(pathStr);
        if (!localFs.exists(path)) {
            LOG.warn("Could not validate jar file " + path + " for class " + clazz);
            continue;
        }
        jars.add(path.makeQualified(localFs.getUri(), localFs.getWorkingDirectory()).toString());
    }
    if (jars.isEmpty()) {
        return;
    }

    conf.set("tmpjars", StringUtils.arrayToString(jars.toArray(new String[0])));
}

---

From source file:org.apache.blur.mapreduce.lib.CsvBlurMapper.java

License:Apache License

@Override
protected void setup(Context context) throws IOException, InterruptedException {
    super.setup(context);
    Configuration configuration = context.getConfiguration();
    _autoGenerateRecordIdAsHashOfData = isAutoGenerateRecordIdAsHashOfData(configuration);
    _autoGenerateRowIdAsHashOfData = isAutoGenerateRowIdAsHashOfData(configuration);
    if (_autoGenerateRecordIdAsHashOfData || _autoGenerateRowIdAsHashOfData) {
        try {/*ww  w  .j  a  v  a 2 s. c om*/
            _digest = MessageDigest.getInstance("MD5");
        } catch (NoSuchAlgorithmException e) {
            throw new IOException(e);
        }
    }
    _columnNameMap = getFamilyAndColumnNameMap(configuration);
    _separator = new String(Base64.decodeBase64(configuration.get(BLUR_CSV_SEPARATOR_BASE64, _separator)),
            UTF_8);
    _splitter = Splitter.on(_separator);
    Path fileCurrentlyProcessing = getCurrentFile(context);
    Collection<String> families = configuration.getStringCollection(BLUR_CSV_FAMILY_PATH_MAPPINGS_FAMILIES);
    OUTER: for (String family : families) {
        Collection<String> pathStrCollection = configuration
                .getStringCollection(BLUR_CSV_FAMILY_PATH_MAPPINGS_FAMILY_PREFIX + family);
        for (String pathStr : pathStrCollection) {
            Path path = new Path(pathStr);
            FileSystem fileSystem = path.getFileSystem(configuration);
            path = path.makeQualified(fileSystem.getUri(), fileSystem.getWorkingDirectory());
            if (isParent(path, fileCurrentlyProcessing)) {
                _familyFromPath = family;
                _familyNotInFile = true;
                break OUTER;
            }
        }
    }
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• »