Example usage for org.apache.hadoop.conf Configuration addResource

Introduction

In this page you can find the example usage for org.apache.hadoop.conf Configuration addResource.

Prototype

public void addResource(Configuration conf) 

Source Link

Document

Add a configuration resource.

Usage

From source file:net.java.jatextmining.JaCoOccurrence.java

License:Apache License

@Override
public int run(String[] args) throws Exception {
    Configuration conf = getConf();
    conf.addResource(CONF_PATH);
    procArgs(args);/*w w w  . j  a v  a  2  s.  c o m*/
    setPos(conf);
    cacheName = conf.get("jatextmining.cache");
    if (!reUse) {
        runJaCoOccurrence(conf, cacheName);
    }
    FileSystem fs = FileSystem.get(new URI(cacheName), conf);
    FileStatus[] status = fs.listStatus(new Path(cacheName));
    if (status == null) {
        printDfdbError(conf);
    }
    DistributedCache.addCacheFile(new URI(cacheName), conf);
    runJaCoOccurrenceWeighting(conf, cacheName);

    return 0;
}

---

From source file:net.java.jatextmining.JaWordCounter.java

License:Apache License

@Override
public int run(String[] args) throws Exception {
    Configuration conf = getConf();
    conf.addResource(CONF_PATH);
    procArgs(args);//from w ww . j a  va 2 s . com
    setPos(conf);
    String dfdb = conf.get("jatextmining.dfdb");
    if (weightingFlag && !dfReuse) {
        runCreateDFDB(conf, dfdb);
    }
    if (weightingFlag) {
        FileSystem fs = FileSystem.get(new URI(dfdb), conf);
        FileStatus[] status = fs.listStatus(new Path(dfdb));
        if (status == null) {
            printDfdbError(conf);
        }
        DistributedCache.addCacheFile(new URI(dfdb), conf);
    }
    runCount(conf);
    clearPos(conf);

    return 0;
}

---

From source file:net.java.jatextmining.lib.CoOccurrenceWeightingMapper.java

License:Apache License

@Override
public void setup(Context context) {
    Configuration conf = context.getConfiguration();
    String confPath = conf.get(CONF_PATH);
    conf.addResource(confPath);
    dfMap = new LinkedHashMap<String, Double>(Integer.valueOf(conf.get("jatextmining.dfHashSize")));
    try {/*from   ww w  . ja  va2  s. com*/
        Path[] cacheFiles = DistributedCache.getLocalCacheFiles(conf);
        if (cacheFiles != null) {
            for (Path cachePath : cacheFiles) {
                loadCacheFile(cachePath, context);
            }
        }
    } catch (IOException e) {
        e.printStackTrace();
    }
}

---

From source file:net.java.jatextmining.lib.CountReducer.java

License:Apache License

@Override
public final void setup(Context context) {
    Configuration conf = context.getConfiguration();
    if (conf.getBoolean("weighting", false)) {
        String confPath = conf.get(CONF_PATH);
        conf.addResource(confPath);
        dfMap = new LinkedHashMap<String, Double>(Integer.valueOf(conf.get("jatextmining.dfHashSize")));
        try {/*from w ww. j  ava  2s.  co m*/
            Path[] cacheFiles = DistributedCache.getLocalCacheFiles(conf);
            if (cacheFiles != null) {
                for (Path cachePath : cacheFiles) {
                    loadCacheFile(cachePath, context);
                }
            }
        } catch (IOException e) {
            e.printStackTrace();
        }
    }
    countMinNum = Integer.valueOf(conf.get("jatextmining.counterMinimumLimitNum"));
}

---

From source file:net.java.jatextmining.util.TokenizerSample.java

License:Apache License

@Override
public int run(final String[] args) {
    procArgs(args);/*www  .j a  v  a 2s.  c om*/
    if (in == null || out == null) {
        printUsage();
    }
    Configuration conf = getConf();
    conf.addResource(CONF_PATH);
    boolean rvBuf = true;
    try {
        rvBuf = runTokenizerSample(conf);
    } catch (IOException e) {
        e.printStackTrace();
    } catch (InterruptedException e) {
        e.printStackTrace();
    } catch (ClassNotFoundException e) {
        e.printStackTrace();
    }
    int rv = rvBuf ? 0 : 1;
    return rv;
}

---

From source file:nl.surfsara.warcexamples.hdfs.Headers.java

License:Apache License

@Override
public void run() {
    // PropertyConfigurator.configure("log4jconfig.properties");
    final Configuration conf = new Configuration();
    // The core-site.xml and hdfs-site.xml are cluster specific. If you wish to use this on other clusters adapt the files as needed.
    conf.addResource(
            Headers.class.getResourceAsStream("/nl/surfsara/warcexamples/hdfs/resources/core-site.xml"));
    conf.addResource(/*from   ww  w.j  a v a 2 s. c o m*/
            Headers.class.getResourceAsStream("/nl/surfsara/warcexamples/hdfs/resources/hdfs-site.xml"));

    conf.set("hadoop.security.authentication", "kerberos");
    conf.set("hadoop.security.authorization", "true");

    System.setProperty("java.security.krb5.realm", "CUA.SURFSARA.NL");
    System.setProperty("java.security.krb5.kdc", "kdc.hathi.surfsara.nl");

    UserGroupInformation.setConfiguration(conf);

    UserGroupInformation loginUser;
    try {
        loginUser = UserGroupInformation.getLoginUser();
        System.out.println("Logged in as: " + loginUser.getUserName());
        PrintHeaders printHeaders = new PrintHeaders(conf, path);
        loginUser.doAs(printHeaders);
    } catch (IOException e) {
        // Just dump the error..
        e.printStackTrace();
    }
}

---

From source file:nur.aini.hadoop.TestHDFS.java

public static FileSystem getHdfs(String hadooppath) {
    if (hdfs == null) {
        try {//from   www  .ja  v  a  2  s . c  o m
            Configuration conf = new Configuration();
            conf.addResource(new Path(hadooppath + "/conf/core-site.xml"));
            hdfs = FileSystem.get(conf);
        } catch (IOException ex) {
            System.err.print(ex.getMessage());
        }
    }
    return hdfs;
}

---

From source file:org.anon.smart.d2cache.fileStore.TestFileStores.java

License:Open Source License

@Test
public void testHadoopFS() throws Exception {

    prepare();/*from  w  w w . ja  v a2s  .  c o m*/

    System.setProperty("Smart.Development.Mode", "false");
    TestConfig tsConf = new TestConfig();
    D2Cache dc = D2CacheScheme.getCache(D2CacheScheme.scheme.filestore, "testStore", 1, tsConf);

    D2CacheTransaction tx = dc.startTransaction(UUID.randomUUID());

    StoreItem sItem1 = new StoreItem(null, fl1, "testGroup");
    StoreItem sItem2 = new StoreItem(null, fl2, "testGroup");

    tx.add(sItem1);
    tx.add(sItem2);

    tx.commit();

    Configuration conf = new Configuration();
    conf.addResource(new Path(System.getProperty("user.dir")
            + "/../sm.kernel/src/main/resources/dbscripts/hadoop-0.20.2-cdh3u5/conf/core-site.xml"));
    conf.addResource(new Path(System.getProperty("user.dir")
            + "/../sm.kernel/src/main/resources/dbscripts/hadoop-0.20.2-cdh3u5/conf/hdfs-site.xml"));

    FileStoreReader r = (FileStoreReader) dc.myReader();
    InputStream s1 = r.getFileAsStream(sItem1.getTruth().toString(), sItem1.group(), null);
    InputStream s2 = r.getFileAsStream(sItem2.getTruth().toString(), sItem2.group(), null);

    writeToFile("hFile1", s1);
    writeToFile("hFile2", s2);

    assertTrue(validate(masterFl, "hFile1"));
    assertTrue(validate(masterFl, "hFile2"));

    //deleteFile("hFile1");
    //deleteFile("hFile2");

}

---

From source file:org.apache.accumulo.cluster.standalone.StandaloneAccumuloCluster.java

License:Apache License

public Configuration getHadoopConfiguration() {
    String confDir = getHadoopConfDir();
    // Using CachedConfiguration will make repeatedly calling this method much faster
    final Configuration conf = CachedConfiguration.getInstance();
    conf.addResource(new Path(confDir, "core-site.xml"));
    // Need hdfs-site.xml for NN HA
    conf.addResource(new Path(confDir, "hdfs-site.xml"));
    return conf;/*from  w  ww.ja  va2s.co m*/
}

---

From source file:org.apache.accumulo.core.file.rfile.PrintInfo.java

License:Apache License

@Override
public void execute(final String[] args) throws Exception {
    Opts opts = new Opts();
    opts.parseArgs(PrintInfo.class.getName(), args);
    if (opts.files.isEmpty()) {
        System.err.println("No files were given");
        System.exit(-1);//  w ww .j  a  v a 2 s .c o  m
    }

    Configuration conf = new Configuration();
    for (String confFile : opts.configFiles) {
        log.debug("Adding Hadoop configuration file " + confFile);
        conf.addResource(new Path(confFile));
    }

    FileSystem hadoopFs = FileSystem.get(conf);
    FileSystem localFs = FileSystem.getLocal(conf);

    LogHistogram kvHistogram = new LogHistogram();

    KeyStats dataKeyStats = new KeyStats();
    KeyStats indexKeyStats = new KeyStats();

    for (String arg : opts.files) {
        Path path = new Path(arg);
        FileSystem fs;
        if (arg.contains(":"))
            fs = path.getFileSystem(conf);
        else {
            log.warn("Attempting to find file across filesystems. Consider providing URI instead of path");
            fs = hadoopFs.exists(path) ? hadoopFs : localFs; // fall back to local
        }
        System.out.println(
                "Reading file: " + path.makeQualified(fs.getUri(), fs.getWorkingDirectory()).toString());

        CachableBlockFile.Reader _rdr = new CachableBlockFile.Reader(fs, path, conf, null, null,
                SiteConfiguration.getInstance(DefaultConfiguration.getInstance()));
        Reader iter = new RFile.Reader(_rdr);
        MetricsGatherer<Map<String, ArrayList<VisibilityMetric>>> vmg = new VisMetricsGatherer();

        if (opts.vis || opts.hash)
            iter.registerMetrics(vmg);

        iter.printInfo();
        System.out.println();
        org.apache.accumulo.core.file.rfile.bcfile.PrintInfo.main(new String[] { arg });

        Map<String, ArrayList<ByteSequence>> localityGroupCF = null;

        if (opts.histogram || opts.dump || opts.vis || opts.hash || opts.keyStats) {
            localityGroupCF = iter.getLocalityGroupCF();

            FileSKVIterator dataIter;
            if (opts.useSample) {
                dataIter = iter.getSample();

                if (dataIter == null) {
                    System.out.println("ERROR : This rfile has no sample data");
                    return;
                }
            } else {
                dataIter = iter;
            }

            if (opts.keyStats) {
                FileSKVIterator indexIter = iter.getIndex();
                while (indexIter.hasTop()) {
                    indexKeyStats.add(indexIter.getTopKey());
                    indexIter.next();
                }
            }

            for (Entry<String, ArrayList<ByteSequence>> cf : localityGroupCF.entrySet()) {

                dataIter.seek(new Range((Key) null, (Key) null), cf.getValue(), true);
                while (dataIter.hasTop()) {
                    Key key = dataIter.getTopKey();
                    Value value = dataIter.getTopValue();
                    if (opts.dump) {
                        System.out.println(key + " -> " + value);
                        if (System.out.checkError())
                            return;
                    }
                    if (opts.histogram) {
                        kvHistogram.add(key.getSize() + value.getSize());
                    }
                    if (opts.keyStats) {
                        dataKeyStats.add(key);
                    }
                    dataIter.next();
                }
            }
        }

        iter.close();

        if (opts.vis || opts.hash) {
            System.out.println();
            vmg.printMetrics(opts.hash, "Visibility", System.out);
        }

        if (opts.histogram) {
            System.out.println();
            kvHistogram.print("");
        }

        if (opts.keyStats) {
            System.out.println();
            System.out.println("Statistics for keys in data :");
            dataKeyStats.print("\t");
            System.out.println();
            System.out.println("Statistics for keys in index :");
            indexKeyStats.print("\t");
        }
        // If the output stream has closed, there is no reason to keep going.
        if (System.out.checkError())
            return;
    }
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• 41
• 42
• 43
• 44
• 45
• 46
• 47
• 48
• 49
• 50
• »