Java tutorial
/* * To change this license header, choose License Headers in Project Properties. * To change this template file, choose Tools | Templates * and open the template in the editor. */ package lucene.searchengine; import java.io.BufferedReader; import java.io.File; import java.io.FileReader; import java.io.IOException; import java.io.InputStreamReader; import java.io.PrintWriter; import java.io.StringReader; import java.util.ArrayList; import java.util.HashMap; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.core.SimpleAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.StringField; import org.apache.lucene.document.TextField; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.Fields; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.MultiFields; import org.apache.lucene.index.Term; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.queryparser.classic.QueryParser; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.TopScoreDocCollector; import org.apache.lucene.store.FSDirectory; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.Version; import org.jsoup.Jsoup; /** * * @author devworks */ public class LuceneSearchEngine { //private static Analyzer analyzer = new StandardAnalyzer(Version.LUCENE_47); private static Analyzer sAnalyzer = new SimpleAnalyzer(Version.LUCENE_47); private IndexWriter writer; private ArrayList<File> queue = new ArrayList<File>(); public static void main(String[] args) throws IOException { System.out.println( "Enter the FULL path where the index will be created: (e.g. /Usr/index or c:\\temp\\index)"); String indexLocation = null; BufferedReader br = new BufferedReader(new InputStreamReader(System.in)); String s = br.readLine(); LuceneSearchEngine indexer = null; try { indexLocation = s; indexer = new LuceneSearchEngine(s); } catch (Exception ex) { System.out.println("Cannot create index..." + ex.getMessage()); System.exit(-1); } // =================================================== // read input from user until he enters q for quit // =================================================== while (!s.equalsIgnoreCase("q")) { try { System.out.println( "Enter the FULL path to add into the index (q=quit): (e.g. /home/mydir/docs or c:\\Users\\mydir\\docs)"); System.out.println("[Acceptable file types: .xml, .html, .html, .txt]"); s = br.readLine(); if (s.equalsIgnoreCase("q")) { break; } // try to add file into the index indexer.indexFileOrDirectory(s); } catch (Exception e) { System.out.println("Error indexing " + s + " : " + e.getMessage()); } } // =================================================== // after adding, we always have to call the // closeIndex, otherwise the index is not created // =================================================== indexer.closeIndex(); // ========================================================= // Now search // ========================================================= IndexReader reader = DirectoryReader.open(FSDirectory.open(new File(indexLocation))); //=========================================================== // GET Term frequency //=========================================================== // Creating a output file to store the term,term_frequency pairs. PrintWriter tfwriter = new PrintWriter("..\\term-frequency.csv"); Fields fields = MultiFields.getFields(reader); HashMap<String, Long> tfmap = new HashMap<String, Long>(); Terms terms = fields.terms("contents"); TermsEnum termsEnum = terms.iterator(null); BytesRef bref = null; while ((bref = termsEnum.next()) != null) { String term_name = new String(bref.bytes, bref.offset, bref.length); Term term_instance = new Term("contents", term_name); long termFrequency = reader.totalTermFreq(term_instance); tfmap.put(term_name, termFrequency); } System.out.println(tfmap.size()); for (String key : tfmap.keySet()) { tfwriter.write(key + "," + tfmap.get(key)); tfwriter.write("\n"); } tfwriter.close(); //==================================================================== // Code END to fetch term frequency //==================================================================== IndexSearcher searcher = new IndexSearcher(reader); s = ""; while (!s.equalsIgnoreCase("q")) { TopScoreDocCollector collector = TopScoreDocCollector.create(100, true); try { System.out.println("Enter the search query (q=quit):"); s = br.readLine(); if (s.equalsIgnoreCase("q")) { break; } Query q = new QueryParser(Version.LUCENE_47, "contents", sAnalyzer).parse(s); searcher.search(q, collector); ScoreDoc[] hits = collector.topDocs().scoreDocs; // 4. display results System.out.println("Found " + hits.length + " hits."); for (int i = 0; i < hits.length; ++i) { int docId = hits[i].doc; Document d = searcher.doc(docId); System.out.println((i + 1) + ". " + d.get("filename") + " score=" + hits[i].score); } // 5. term stats --> watch out for which "version" of the term // must be checked here instead! Term termInstance = new Term("contents", s); long termFreq = reader.totalTermFreq(termInstance); long docCount = reader.docFreq(termInstance); System.out.println(s + " Term Frequency " + termFreq + " - Document Frequency " + docCount); } catch (Exception e) { System.out.println("Error searching " + s + " : " + e.getMessage()); break; } } } /** * Constructor * * @param indexDir the name of the folder in which the index should be * created * @throws java.io.IOException when exception creating index. */ LuceneSearchEngine(String indexDir) throws IOException { FSDirectory dir = FSDirectory.open(new File(indexDir)); IndexWriterConfig config = new IndexWriterConfig(Version.LUCENE_47, sAnalyzer); writer = new IndexWriter(dir, config); } /** * Indexes a file or directory * * @param fileName the name of a text file or a folder we wish to add to the * index * @throws java.io.IOException when exception */ public void indexFileOrDirectory(String fileName) throws IOException { // =================================================== // gets the list of files in a folder (if user has submitted // the name of a folder) or gets a single file name (is user // has submitted only the file name) // =================================================== addFiles(new File(fileName)); int originalNumDocs = writer.numDocs(); for (File f : queue) { FileReader fr = null; try { Document doc = new Document(); // =================================================== // add contents of file // =================================================== fr = new FileReader(f); BufferedReader br = new BufferedReader(fr); String one_line = br.readLine(); StringBuilder sbldr = new StringBuilder(); while (one_line != null) { sbldr.append(one_line); sbldr.append("\n"); one_line = br.readLine(); } StringReader str_reader = new StringReader(Jsoup.parse(sbldr.toString()).text()); doc.add(new TextField("contents", str_reader)); doc.add(new StringField("path", f.getPath(), Field.Store.YES)); doc.add(new StringField("filename", f.getName(), Field.Store.YES)); writer.addDocument(doc); System.out.println("Added: " + f); } catch (Exception e) { System.out.println("Could not add: " + f); } finally { fr.close(); } } int newNumDocs = writer.numDocs(); System.out.println(""); System.out.println("************************"); System.out.println((newNumDocs - originalNumDocs) + " documents added."); System.out.println("************************"); queue.clear(); } private void addFiles(File file) { if (!file.exists()) { System.out.println(file + " does not exist."); } if (file.isDirectory()) { for (File f : file.listFiles()) { addFiles(f); } } else { String filename = file.getName().toLowerCase(); // =================================================== // Only index text files // =================================================== if (filename.endsWith(".htm") || filename.endsWith(".html") || filename.endsWith(".xml") || filename.endsWith(".txt")) { queue.add(file); } else { System.out.println("Skipped " + filename); } } } /** * Close the index. * * @throws java.io.IOException when exception closing */ public void closeIndex() throws IOException { writer.close(); } }