Java tutorial
/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.lucene.benchmark.byTask.feeds; import java.io.BufferedWriter; import java.io.IOException; import java.io.OutputStream; import java.io.OutputStreamWriter; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.util.Properties; import org.apache.commons.compress.compressors.CompressorStreamFactory; import org.apache.lucene.analysis.core.WhitespaceAnalyzer; import org.apache.lucene.benchmark.BenchmarkTestCase; import org.apache.lucene.benchmark.byTask.PerfRunData; import org.apache.lucene.benchmark.byTask.feeds.LineDocSource.HeaderLineParser; import org.apache.lucene.benchmark.byTask.feeds.LineDocSource.LineParser; import org.apache.lucene.benchmark.byTask.tasks.AddDocTask; import org.apache.lucene.benchmark.byTask.tasks.CloseIndexTask; import org.apache.lucene.benchmark.byTask.tasks.CreateIndexTask; import org.apache.lucene.benchmark.byTask.tasks.TaskSequence; import org.apache.lucene.benchmark.byTask.tasks.WriteLineDocTask; import org.apache.lucene.benchmark.byTask.utils.Config; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.Term; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.TopDocs; import org.apache.lucene.util.IOUtils; /** Tests the functionality of {@link LineDocSource}. */ public class LineDocSourceTest extends BenchmarkTestCase { private static final CompressorStreamFactory csFactory = new CompressorStreamFactory(); private void createBZ2LineFile(Path file, boolean addHeader) throws Exception { OutputStream out = Files.newOutputStream(file); out = csFactory.createCompressorOutputStream("bzip2", out); BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(out, StandardCharsets.UTF_8)); writeDocsToFile(writer, addHeader, null); writer.close(); } private void writeDocsToFile(BufferedWriter writer, boolean addHeader, Properties otherFields) throws IOException { if (addHeader) { writer.write(WriteLineDocTask.FIELDS_HEADER_INDICATOR); writer.write(WriteLineDocTask.SEP); writer.write(DocMaker.TITLE_FIELD); writer.write(WriteLineDocTask.SEP); writer.write(DocMaker.DATE_FIELD); writer.write(WriteLineDocTask.SEP); writer.write(DocMaker.BODY_FIELD); if (otherFields != null) { // additional field names in the header for (Object fn : otherFields.keySet()) { writer.write(WriteLineDocTask.SEP); writer.write(fn.toString()); } } writer.newLine(); } StringBuilder doc = new StringBuilder(); doc.append("title").append(WriteLineDocTask.SEP).append("date").append(WriteLineDocTask.SEP) .append(DocMaker.BODY_FIELD); if (otherFields != null) { // additional field values in the doc line for (Object fv : otherFields.values()) { doc.append(WriteLineDocTask.SEP).append(fv.toString()); } } writer.write(doc.toString()); writer.newLine(); } private void createRegularLineFile(Path file, boolean addHeader) throws Exception { OutputStream out = Files.newOutputStream(file); BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(out, StandardCharsets.UTF_8)); writeDocsToFile(writer, addHeader, null); writer.close(); } private void createRegularLineFileWithMoreFields(Path file, String... extraFields) throws Exception { OutputStream out = Files.newOutputStream(file); BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(out, StandardCharsets.UTF_8)); Properties p = new Properties(); for (String f : extraFields) { p.setProperty(f, f); } writeDocsToFile(writer, true, p); writer.close(); } private void doIndexAndSearchTest(Path file, Class<? extends LineParser> lineParserClass, String storedField) throws Exception { doIndexAndSearchTestWithRepeats(file, lineParserClass, 1, storedField); // no extra repetitions doIndexAndSearchTestWithRepeats(file, lineParserClass, 2, storedField); // 1 extra repetition doIndexAndSearchTestWithRepeats(file, lineParserClass, 4, storedField); // 3 extra repetitions } private void doIndexAndSearchTestWithRepeats(Path file, Class<? extends LineParser> lineParserClass, int numAdds, String storedField) throws Exception { IndexReader reader = null; IndexSearcher searcher = null; PerfRunData runData = null; try { Properties props = new Properties(); // LineDocSource specific settings. props.setProperty("docs.file", file.toAbsolutePath().toString()); if (lineParserClass != null) { props.setProperty("line.parser", lineParserClass.getName()); } // Indexing configuration. props.setProperty("analyzer", WhitespaceAnalyzer.class.getName()); props.setProperty("content.source", LineDocSource.class.getName()); props.setProperty("directory", "RAMDirectory"); props.setProperty("doc.stored", "true"); props.setProperty("doc.index.props", "true"); // Create PerfRunData Config config = new Config(props); runData = new PerfRunData(config); TaskSequence tasks = new TaskSequence(runData, "testBzip2", null, false); tasks.addTask(new CreateIndexTask(runData)); for (int i = 0; i < numAdds; i++) { tasks.addTask(new AddDocTask(runData)); } tasks.addTask(new CloseIndexTask(runData)); try { tasks.doLogic(); } finally { tasks.close(); } reader = DirectoryReader.open(runData.getDirectory()); searcher = newSearcher(reader); TopDocs td = searcher.search(new TermQuery(new Term("body", "body")), 10); assertEquals(numAdds, td.totalHits); assertNotNull(td.scoreDocs[0]); if (storedField == null) { storedField = DocMaker.BODY_FIELD; // added to all docs and satisfies field-name == value } assertEquals("Wrong field value", storedField, searcher.doc(0).get(storedField)); } finally { IOUtils.close(reader, runData); } } /* Tests LineDocSource with a bzip2 input stream. */ public void testBZip2() throws Exception { Path file = getWorkDir().resolve("one-line.bz2"); createBZ2LineFile(file, true); doIndexAndSearchTest(file, null, null); } public void testBZip2NoHeaderLine() throws Exception { Path file = getWorkDir().resolve("one-line.bz2"); createBZ2LineFile(file, false); doIndexAndSearchTest(file, null, null); } public void testRegularFile() throws Exception { Path file = getWorkDir().resolve("one-line"); createRegularLineFile(file, true); doIndexAndSearchTest(file, null, null); } public void testRegularFileSpecialHeader() throws Exception { Path file = getWorkDir().resolve("one-line"); createRegularLineFile(file, true); doIndexAndSearchTest(file, HeaderLineParser.class, null); } public void testRegularFileNoHeaderLine() throws Exception { Path file = getWorkDir().resolve("one-line"); createRegularLineFile(file, false); doIndexAndSearchTest(file, null, null); } public void testInvalidFormat() throws Exception { String[] testCases = new String[] { "", // empty line "title", // just title "title" + WriteLineDocTask.SEP, // title + SEP "title" + WriteLineDocTask.SEP + "body", // title + SEP + body // note that title + SEP + body + SEP is a valid line, which results in an // empty body }; for (int i = 0; i < testCases.length; i++) { Path file = getWorkDir().resolve("one-line"); BufferedWriter writer = Files.newBufferedWriter(file, StandardCharsets.UTF_8); writer.write(testCases[i]); writer.newLine(); writer.close(); expectThrows(Exception.class, () -> { doIndexAndSearchTest(file, null, null); }); } } /** Doc Name is not part of the default header */ public void testWithDocsName() throws Exception { Path file = getWorkDir().resolve("one-line"); createRegularLineFileWithMoreFields(file, DocMaker.NAME_FIELD); doIndexAndSearchTest(file, null, DocMaker.NAME_FIELD); } /** Use fields names that are not defined in Docmaker and so will go to Properties */ public void testWithProperties() throws Exception { Path file = getWorkDir().resolve("one-line"); String specialField = "mySpecialField"; createRegularLineFileWithMoreFields(file, specialField); doIndexAndSearchTest(file, null, specialField); } }