Example usage for org.apache.hadoop.fs FileSystem newInstance

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileSystem newInstance.

Prototype

public static FileSystem newInstance(URI uri, Configuration config) throws IOException 

Source Link

Document

Returns the FileSystem for this URI's scheme and authority.

Usage

From source file:com.datatorrent.lib.bucket.BucketStoreTests.java

License:Open Source License

void setup(HdfsBucketStore<DummyEvent> store) {
    applicationPath = OperatorContextTestHelper.getUniqueApplicationPath(APPLICATION_PATH_PREFIX);
    bucketStore = store;//from w w w. j a v a  2 s.c  om
    bucketStore.setNoOfBuckets(TOTAL_BUCKETS);
    bucketStore.setWriteEventKeysOnly(true);
    bucketStore.setConfiguration(7, applicationPath, Sets.newHashSet(0), 0);
    bucketStore.setup();

    for (int bucketIdx = 0; bucketIdx < 2; bucketIdx++) {
        Map<Object, DummyEvent> bucketData = Maps.newHashMap();
        data.put(bucketIdx, bucketData);

        for (int i = 0; i < 10; i++) {
            DummyEvent event = new DummyEvent(i, System.currentTimeMillis());
            bucketData.put(event.getEventKey(), event);
        }
    }
    rootBucketPath = new Path(bucketStore.bucketRoot);
    try {
        fs = FileSystem.newInstance(rootBucketPath.toUri(), new Configuration());
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
}

---

From source file:com.datatorrent.lib.bucket.CategoricalBucketManagerTest.java

License:Open Source License

@After
public void teardown() throws IOException {
    manager.shutdownService();/*from   w  w w .j a  v  a  2 s . c  o  m*/
    Path root = new Path(applicationPath);
    FileSystem fs = FileSystem.newInstance(root.toUri(), new Configuration());
    fs.delete(root, true);
}

---

From source file:com.datatorrent.lib.bucket.ExpirableHdfsBucketStore.java

License:Open Source License

@Override
public void deleteExpiredBuckets(long time) throws IOException {
    Iterator<Long> iterator = windowToBuckets.keySet().iterator();
    for (; iterator.hasNext();) {
        long window = iterator.next();
        long timestamp = windowToTimestamp.get(window);
        if (timestamp < time) {
            Collection<Integer> indices = windowToBuckets.get(window);
            synchronized (indices) {
                if (indices.size() > 0) {
                    Path dataFilePath = new Path(bucketRoot + PATH_SEPARATOR + window);
                    FileSystem fs = FileSystem.newInstance(dataFilePath.toUri(), configuration);
                    try {
                        if (fs.exists(dataFilePath)) {
                            logger.debug("start delete {}", window);
                            fs.delete(dataFilePath, true);
                            logger.debug("end delete {}", window);
                        }/*  www  .  ja  v a  2s. c  om*/
                        for (int bucketIdx : indices) {
                            Map<Long, Long> offsetMap = bucketPositions[bucketIdx];
                            if (offsetMap != null) {
                                synchronized (offsetMap) {
                                    offsetMap.remove(window);
                                }
                            }
                        }
                    } finally {
                        fs.close();
                    }
                }
                windowToTimestamp.remove(window);
                iterator.remove();
            }
        }
    }
}

---

From source file:com.datatorrent.lib.bucket.HdfsBucketStore.java

License:Open Source License

/**
 * {@inheritDoc}/*from w w w.  ja  va  2 s.c o  m*/
 */
@Override
public void storeBucketData(long window, long timestamp, Map<Integer, Map<Object, T>> data) throws IOException {
    Path dataFilePath = new Path(bucketRoot + PATH_SEPARATOR + window);
    FileSystem fs = FileSystem.newInstance(dataFilePath.toUri(), configuration);
    FSDataOutputStream dataStream = fs.create(dataFilePath);

    Output output = new Output(dataStream);
    try {
        long offset = 0;
        for (int bucketIdx : data.keySet()) {
            Map<Object, T> bucketData = data.get(bucketIdx);

            if (eventKeyClass == null) {
                Map.Entry<Object, T> eventEntry = bucketData.entrySet().iterator().next();
                eventKeyClass = eventEntry.getKey().getClass();
                if (!writeEventKeysOnly) {
                    @SuppressWarnings("unchecked")
                    Class<T> lEventClass = (Class<T>) eventEntry.getValue().getClass();
                    eventClass = lEventClass;
                }
            }
            //Write the size of data and then data
            dataStream.writeInt(bucketData.size());
            for (Map.Entry<Object, T> entry : bucketData.entrySet()) {
                writeSerde.writeObject(output, entry.getKey());

                if (!writeEventKeysOnly) {
                    int posLength = output.position();
                    output.writeInt(0); //temporary place holder
                    writeSerde.writeObject(output, entry.getValue());
                    int posValue = output.position();
                    int valueLength = posValue - posLength - 4;
                    output.setPosition(posLength);
                    output.writeInt(valueLength);
                    output.setPosition(posValue);
                }
            }
            output.flush();
            if (bucketPositions[bucketIdx] == null) {
                bucketPositions[bucketIdx] = Maps.newHashMap();
            }
            windowToBuckets.put(window, bucketIdx);
            windowToTimestamp.put(window, timestamp);
            synchronized (bucketPositions[bucketIdx]) {
                bucketPositions[bucketIdx].put(window, offset);
            }
            offset = dataStream.getPos();
        }
    } finally {
        output.close();
        dataStream.close();
        fs.close();
    }
}

---

From source file:com.datatorrent.lib.bucket.HdfsBucketStore.java

License:Open Source License

/**
 * {@inheritDoc}//  w w  w.j a v  a2s.c o  m
 */
@Override
public void deleteBucket(int bucketIdx) throws IOException {
    Map<Long, Long> offsetMap = bucketPositions[bucketIdx];
    if (offsetMap != null) {
        for (Long window : offsetMap.keySet()) {
            Collection<Integer> indices = windowToBuckets.get(window);
            synchronized (indices) {
                boolean elementRemoved = indices.remove(bucketIdx);
                if (indices.isEmpty() && elementRemoved) {
                    Path dataFilePath = new Path(bucketRoot + PATH_SEPARATOR + window);
                    FileSystem fs = FileSystem.newInstance(dataFilePath.toUri(), configuration);
                    try {
                        if (fs.exists(dataFilePath)) {
                            logger.debug("start delete {}", window);
                            fs.delete(dataFilePath, true);
                            logger.debug("end delete {}", window);
                        }
                        windowToBuckets.removeAll(window);
                        windowToTimestamp.remove(window);
                    } finally {
                        fs.close();
                    }
                }
            }
        }
    }
    bucketPositions[bucketIdx] = null;
}

---

From source file:com.datatorrent.lib.dedup.DeduperBloomFilterTest.java

License:Open Source License

@After
public void teardown() {
    Path root = new Path(applicationPath);
    try {// ww w .  j  av  a  2 s  .  co m
        FileSystem fs = FileSystem.newInstance(root.toUri(), new Configuration());
        fs.delete(root, true);
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
}

---

From source file:com.datatorrent.lib.dedup.DeduperBucketEvictionTest.java

License:Open Source License

@AfterClass
public static void teardown() {
    Path root = new Path(applicationPath);
    try {/*  w  ww .  j  a v  a  2  s.co m*/
        FileSystem fs = FileSystem.newInstance(root.toUri(), new Configuration());
        fs.delete(root, true);
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
}

---

From source file:com.datatorrent.lib.dedup.DedupUsingCategoricalExpiryTest.java

License:Open Source License

@AfterClass
public static void teardown() {
    Path root = new Path(applicationPath);
    try {/*from  ww  w. ja  v a  2 s . com*/
        FileSystem fs = FileSystem.newInstance(root.toUri(), new Configuration());
        fs.delete(root, true);
        logger.debug("Deleted path: " + applicationPath);
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
}

---

From source file:com.datatorrent.lib.io.block.AbstractFSBlockReader.java

License:Apache License

/**
 * Override this method to change the FileSystem instance that is used by the operator.
 *
 * @return A FileSystem object.//from  w w  w  .  jav a 2 s  .c o m
 * @throws IOException
 */
protected FileSystem getFSInstance() throws IOException {
    if (basePath != null) {
        return FileSystem.newInstance(URI.create(basePath), configuration);
    } else {
        return FileSystem.newInstance(configuration);
    }
}

---

From source file:com.datatorrent.lib.io.block.BlockReader.java

License:Apache License

@Override
protected FileSystem getFSInstance() throws IOException {
    return FileSystem.newInstance(URI.create(uri), configuration);
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• »