Example usage for org.apache.hadoop.fs FileSystem createNewFile

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileSystem createNewFile.

Prototype

public boolean createNewFile(Path f) throws IOException 

Source Link

Document

Creates the given Path as a brand-new zero-length file.

Usage

From source file:org.apache.nutch.admin.management.CrawldbThread.java

License:Apache License

public void run() {

    Path runningSegment = null;/*from  w  ww. j  a  va  2  s  . c  o m*/
    Path runningDB = null;
    FileSystem fileSystem = null;
    try {
        this.fMessage = "crawldb.running";
        fileSystem = FileSystem.get(this.fConfiguration);
        runningSegment = new Path(this.fSegment, "crawldb.running");
        runningDB = new Path(this.fCrawldbFile, "crawldb.running");
        fileSystem.createNewFile(runningSegment);
        fileSystem.createNewFile(runningDB);
        // this.fCrawldb.update(this.fCrawldbFile, this.fSegment, true, true);
        Path[] segs = new Path[1];
        segs[0] = this.fSegment;
        this.fCrawldb.update(this.fCrawldbFile, segs, true, true);

    } catch (IOException e) {
        LOG.warning(e.toString());
        this.fMessage = e.toString();
    } finally {
        try {
            if (fileSystem != null) {
                fileSystem.delete(runningSegment);
                fileSystem.delete(runningDB);
            }
        } catch (IOException e) {
            LOG.warning(e.toString());
        }
    }
}

---

From source file:org.apache.nutch.admin.management.CreateFileThread.java

License:Apache License

public void run() {
    try {/*from   w  w w.  j  av  a2  s .com*/
        this.fMessage = "file.create.running";
        FileSystem system = FileSystem.get(this.fConfiguration);
        system.createNewFile(this.fFile);
    } catch (IOException e) {
        LOG.warning(e.toString());
        this.fMessage = e.toString();
    }
}

---

From source file:org.apache.nutch.admin.management.FetchThread.java

License:Apache License

public void run() {
    FileSystem fileSystem = null;
    Path running = null;//from   w  w w .  j a  v a2  s .c o  m
    try {
        this.fMessage = "fetch.running";
        fileSystem = FileSystem.get(this.fConfiguration);
        running = new Path(this.fSegment, "fetch.running");
        fileSystem.createNewFile(running);
        this.fFetcher.fetch(this.fSegment, this.fThreads, true);
        fileSystem.createNewFile(new Path(this.fSegment, "fetch.done"));
        if (this.fParsing) {
            fileSystem.createNewFile(new Path(this.fSegment, "parse.done"));
        }
    } catch (IOException e) {
        LOG.warning(e.toString());
        this.fMessage = e.toString();
    } finally {
        try {
            if (fileSystem != null) {
                fileSystem.delete(running);
            }
        } catch (IOException e) {
            LOG.warning(e.toString());
        }
    }
}

---

From source file:org.apache.nutch.admin.management.GenerateThread.java

License:Apache License

public void run() {
    if (!this.fError) {
        FileSystem fileSystem = null;
        Path runningSegment = null;
        Path runningDB = null;/*from   ww  w.ja v a 2s  .com*/
        try {
            this.fMessage = "generate.running";
            fileSystem = FileSystem.get(this.fConfiguration);
            runningSegment = new Path(this.fSegmentFolder, "generate.running");
            runningDB = new Path(this.fCrawldb, "generate.running");
            fileSystem.createNewFile(runningSegment);
            fileSystem.createNewFile(runningDB);
            long curTime = System.currentTimeMillis() + this.fDays * 1000L * 60 * 60 * 24;

            this.fGenerator.generate(this.fCrawldb, this.fSegmentFolder, this.fFetcher, this.fTopN, curTime,
                    true, true);

        } catch (IOException e) {
            LOG.warn(e.toString());
            this.fMessage = e.toString();
        } finally {
            try {
                if (fileSystem != null) {
                    fileSystem.delete(runningSegment);
                    fileSystem.delete(runningDB);
                }
            } catch (IOException e) {
                LOG.warn(e.toString());
            }
        }
    }
}

---

From source file:org.apache.nutch.admin.management.IndexThread.java

License:Apache License

public void run() {
    FileSystem fileSystem = null;
    try {/*from   w  w  w .j av  a2s .  c om*/
        this.fMessage = "index.running";
        fileSystem = FileSystem.get(this.fConfiguration);

        // create lock files in segments
        for (int i = 0; i < this.fSegments.length; i++) {
            Path file = this.fSegments[i];
            fileSystem.createNewFile(new Path(file, "index.running"));
        }

        // create lock files in linkdb
        fileSystem.createNewFile(new Path(this.fLinkdb, "index.running"));
        // create lock files in crawldb
        fileSystem.createNewFile(new Path(this.fCrawldb, "index.running"));

        // this.fIndexer.index(
        //     this.fIndexFolder, 
        //     this.fCrawldb, 
        //     this.fLinkdb,
        //     this.fSegments
        //     );
        List<Path> seglist = Arrays.asList(this.fSegments);
        this.fIndexer.index(this.fIndexFolder, this.fCrawldb, this.fLinkdb, seglist);

    } catch (IOException e) {
        LOG.warn(e.toString());
        this.fMessage = e.toString();
    } finally {
        try {
            if (fileSystem != null) {
                // delete lock files from segments
                for (int i = 0; i < this.fSegments.length; i++) {
                    Path file = this.fSegments[i];
                    fileSystem.delete(new Path(file, "index.running"));
                }
                // delete lock files from linkdb
                fileSystem.delete(new Path(this.fLinkdb, "index.running"));
                // delete lock files from crawldb
                fileSystem.delete(new Path(this.fCrawldb, "index.running"));
            }
        } catch (IOException e) {
            LOG.warn(e.toString());
        }
    }
}

---

From source file:org.apache.nutch.admin.management.LinkdbThread.java

License:Apache License

public void run() {
    FileSystem fileSystem = null;
    try {//from   w w  w .  j  a v  a 2s  . c o  m
        this.fMessage = "linkdb.running";
        fileSystem = FileSystem.get(this.fConfiguration);
        Path running = new Path(this.fLinkdbFile, "linkdb.running");
        // create lock files in segments
        for (int i = 0; i < this.fSegments.length; i++) {
            Path file = this.fSegments[i];
            fileSystem.createNewFile(new Path(file, "linkdb.running"));
        }
        // create lockfile in linkdb
        fileSystem.createNewFile(running);

        // this.fLinkdb.invert(this.fLinkdbFile, this.fSegments, true, true);
        this.fLinkdb.invert(this.fLinkdbFile, this.fSegments, true, true, false);

        for (int i = 0; i < this.fSegments.length; i++) {
            Path file = this.fSegments[i];
            fileSystem.createNewFile(new Path(file, "invert.done"));
        }
    } catch (IOException e) {
        LOG.warn(e.toString());
        this.fMessage = e.toString();
    } finally {
        try {
            if (fileSystem != null) {
                // delete lock files and create done files
                for (int i = 0; i < this.fSegments.length; i++) {
                    Path file = this.fSegments[i];
                    fileSystem.delete(new Path(file, "linkdb.running"));
                }
                // delete lock file from linkdb
                fileSystem.delete(new Path(this.fLinkdbFile, "linkdb.running"));
            }
        } catch (IOException e) {
            LOG.warn(e.toString());
        }
    }
}

---

From source file:org.apache.nutch.admin.management.ParseThread.java

License:Apache License

public void run() {
    FileSystem fileSystem = null;
    Path running = null;/*from www .  ja va  2s .c o m*/
    try {
        this.fMessage = "parse.running";
        fileSystem = FileSystem.get(this.fConfiguration);
        running = new Path(this.fSegment, "parse.running");
        fileSystem.createNewFile(running);
        this.fParser.parse(this.fSegment);
        fileSystem.createNewFile(new Path(this.fSegment, "parse.done"));
    } catch (IOException e) {
        LOG.warn(e.toString());
        this.fMessage = e.toString();
    } finally {
        try {
            if (fileSystem != null) {
                fileSystem.delete(running);
            }
        } catch (IOException e) {
            LOG.warn(e.toString());
        }
    }
}

---

From source file:org.apache.nutch.admin.scheduling.AdminCrawl.java

License:Apache License

private void index(Path index, Path crawldbFile, Path linkdbFile, Path[] segments, Configuration configuration)
        throws IOException {
    FileSystem fileSystem = FileSystem.get(configuration);
    // create running files in segments
    for (int i = 0; i < segments.length; i++) {
        Path file = segments[i];//w  w  w . j av a2 s  .  co  m
        fileSystem.createNewFile(new Path(file, "index.running"));
    }
    // create running files in linkdb
    fileSystem.createNewFile(new Path(linkdbFile, "index.running"));
    // create running files in crawldb
    fileSystem.createNewFile(new Path(crawldbFile, "index.running"));
    Indexer indexer = new Indexer(configuration);
    indexer.index(index, crawldbFile, linkdbFile, Arrays.asList(segments));
}

---

From source file:org.apache.nutch.admin.scheduling.AdminCrawl.java

License:Apache License

private void updateLinkDb(Path linkdbFile, Path[] segments, Configuration configuration) throws IOException {

    FileSystem fileSystem = FileSystem.get(configuration);

    Path running = new Path(linkdbFile, "linkdb.running");
    // create lock files in segments
    for (int i = 0; i < segments.length; i++) {
        Path file = segments[i];//from w  w w.  j a  v a 2s  .c  o m
        fileSystem.createNewFile(new Path(file, "linkdb.running"));
    }
    // create lockfile in linkdb
    fileSystem.createNewFile(running);

    LinkDb linkDb = new LinkDb(configuration);
    linkDb.invert(linkdbFile, segments, true, true, false);

    for (int i = 0; i < segments.length; i++) {
        Path file = segments[i];
        fileSystem.createNewFile(new Path(file, "invert.done"));
    }
}

---

From source file:org.apache.nutch.admin.scheduling.AdminCrawl.java

License:Apache License

private void updateCrawldb(Path crawldbFile, Path segment, Configuration configuration) throws IOException {
    FileSystem fileSystem = FileSystem.get(configuration);
    Path runningSegment = new Path(segment, "crawldb.running");
    Path runningDB = new Path(crawldbFile, "crawldb.running");
    fileSystem.createNewFile(runningSegment);
    fileSystem.createNewFile(runningDB);
    CrawlDb crawlDb = new CrawlDb(configuration);
    // crawlDb.update(crawldbFile, segment, true, true);
    Path[] segments = new Path[1];
    segments[0] = segment;/*from   ww w.j av  a  2 s .c  o m*/
    crawlDb.update(crawldbFile, segments, true, true);
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• »