Java tutorial
/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.solr.hadoop; import java.io.ByteArrayOutputStream; import java.io.File; import java.io.PrintStream; import java.io.UnsupportedEncodingException; import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.Collections; import java.util.Locale; import org.apache.commons.io.FileUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.lucene.util.Constants; import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.cloud.AbstractZkTestCase; import org.apache.solr.hadoop.dedup.NoChangeUpdateConflictResolver; import org.apache.solr.hadoop.dedup.RetainMostRecentUpdateConflictResolver; import org.junit.After; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; public class MapReduceIndexerToolArgumentParserTest extends SolrTestCaseJ4 { private Configuration conf; private MapReduceIndexerTool.MyArgumentParser parser; private MapReduceIndexerTool.Options opts; private PrintStream oldSystemOut; private PrintStream oldSystemErr; private ByteArrayOutputStream bout; private ByteArrayOutputStream berr; private static final String RESOURCES_DIR = getFile("morphlines-core.marker").getParent(); private static final File MINIMR_INSTANCE_DIR = new File(RESOURCES_DIR + "/solr/minimr"); private static final String MORPHLINE_FILE = RESOURCES_DIR + "/test-morphlines/solrCellDocumentTypes.conf"; private final File solrHomeDirectory = createTempDir().toFile(); @BeforeClass public static void beforeClass() { assumeFalse("This test fails on Java 9 (https://issues.apache.org/jira/browse/SOLR-8876)", Constants.JRE_IS_MINIMUM_JAVA9); assumeFalse("Does not work on Windows, because it uses UNIX shell commands or POSIX paths", Constants.WINDOWS); assumeFalse( "This test fails on UNIX with Turkish default locale (https://issues.apache.org/jira/browse/SOLR-6387)", new Locale("tr").getLanguage().equals(Locale.getDefault().getLanguage())); } @Before public void setUp() throws Exception { super.setUp(); AbstractZkTestCase.SOLRHOME = solrHomeDirectory; FileUtils.copyDirectory(MINIMR_INSTANCE_DIR, solrHomeDirectory); conf = new Configuration(); parser = new MapReduceIndexerTool.MyArgumentParser(); opts = new MapReduceIndexerTool.Options(); oldSystemOut = System.out; bout = new ByteArrayOutputStream(); System.setOut(new PrintStream(bout, true, "UTF-8")); oldSystemErr = System.err; berr = new ByteArrayOutputStream(); System.setErr(new PrintStream(berr, true, "UTF-8")); } @After public void tearDown() throws Exception { super.tearDown(); System.setOut(oldSystemOut); System.setErr(oldSystemErr); } @Test public void testArgsParserTypicalUse() { String[] args = new String[] { "--input-list", "file:///tmp", "--morphline-file", MORPHLINE_FILE, "--morphline-id", "morphline_xyz", "--output-dir", "file:/tmp/foo", "--solr-home-dir", MINIMR_INSTANCE_DIR.getPath(), "--mappers", "10", "--reducers", "9", "--fanout", "8", "--max-segments", "7", "--shards", "1", "--verbose", "file:///home", "file:///dev", }; Integer res = parser.parseArgs(args, conf, opts); assertNull(res != null ? res.toString() : "", res); assertEquals(Collections.singletonList(new Path("file:///tmp")), opts.inputLists); assertEquals(new Path("file:/tmp/foo"), opts.outputDir); assertEquals(new File(MINIMR_INSTANCE_DIR.getPath()), opts.solrHomeDir); assertEquals(10, opts.mappers); assertEquals(9, opts.reducers); assertEquals(8, opts.fanout); assertEquals(7, opts.maxSegments); assertEquals(new Integer(1), opts.shards); assertEquals(null, opts.fairSchedulerPool); assertTrue(opts.isVerbose); assertEquals(Arrays.asList(new Path("file:///home"), new Path("file:///dev")), opts.inputFiles); assertEquals(RetainMostRecentUpdateConflictResolver.class.getName(), opts.updateConflictResolver); assertEquals(MORPHLINE_FILE, opts.morphlineFile.getPath()); assertEquals("morphline_xyz", opts.morphlineId); assertEmptySystemErrAndEmptySystemOut(); } @Test public void testArgsParserMultipleSpecsOfSameKind() { String[] args = new String[] { "--input-list", "file:///tmp", "--input-list", "file:///", "--morphline-file", MORPHLINE_FILE, "--output-dir", "file:/tmp/foo", "--solr-home-dir", MINIMR_INSTANCE_DIR.getPath(), "--shards", "1", "file:///home", "file:///dev", }; assertNull(parser.parseArgs(args, conf, opts)); assertEquals(Arrays.asList(new Path("file:///tmp"), new Path("file:///")), opts.inputLists); assertEquals(Arrays.asList(new Path("file:///home"), new Path("file:///dev")), opts.inputFiles); assertEquals(new Path("file:/tmp/foo"), opts.outputDir); assertEquals(new File(MINIMR_INSTANCE_DIR.getPath()), opts.solrHomeDir); assertEmptySystemErrAndEmptySystemOut(); } @Test public void testArgsParserTypicalUseWithEqualsSign() { String[] args = new String[] { "--input-list=file:///tmp", "--morphline-file", MORPHLINE_FILE, "--output-dir=file:/tmp/foo", "--solr-home-dir=" + MINIMR_INSTANCE_DIR.getPath(), "--mappers=10", "--shards", "1", "--verbose", "file:///home", "file:///dev", }; assertNull(parser.parseArgs(args, conf, opts)); assertEquals(Collections.singletonList(new Path("file:///tmp")), opts.inputLists); assertEquals(new Path("file:/tmp/foo"), opts.outputDir); assertEquals(new File(MINIMR_INSTANCE_DIR.getPath()), opts.solrHomeDir); assertEquals(10, opts.mappers); assertEquals(new Integer(1), opts.shards); assertEquals(null, opts.fairSchedulerPool); assertTrue(opts.isVerbose); assertEquals(Arrays.asList(new Path("file:///home"), new Path("file:///dev")), opts.inputFiles); assertEmptySystemErrAndEmptySystemOut(); } @Test public void testArgsParserMultipleSpecsOfSameKindWithEqualsSign() { String[] args = new String[] { "--input-list=file:///tmp", "--input-list=file:///", "--morphline-file", MORPHLINE_FILE, "--output-dir=file:/tmp/foo", "--solr-home-dir=" + MINIMR_INSTANCE_DIR.getPath(), "--shards", "1", "file:///home", "file:///dev", }; assertNull(parser.parseArgs(args, conf, opts)); assertEquals(Arrays.asList(new Path("file:///tmp"), new Path("file:///")), opts.inputLists); assertEquals(Arrays.asList(new Path("file:///home"), new Path("file:///dev")), opts.inputFiles); assertEquals(new Path("file:/tmp/foo"), opts.outputDir); assertEquals(new File(MINIMR_INSTANCE_DIR.getPath()), opts.solrHomeDir); assertEmptySystemErrAndEmptySystemOut(); } @Test public void testArgsParserHelp() throws UnsupportedEncodingException { String[] args = new String[] { "--help" }; assertEquals(new Integer(0), parser.parseArgs(args, conf, opts)); String helpText = new String(bout.toByteArray(), StandardCharsets.UTF_8); assertTrue(helpText.contains("MapReduce batch job driver that ")); assertTrue(helpText.contains("bin/hadoop command")); assertEquals(0, berr.toByteArray().length); } @Test public void testArgsParserOk() { String[] args = new String[] { "--input-list", "file:///tmp", "--morphline-file", MORPHLINE_FILE, "--output-dir", "file:/tmp/foo", "--solr-home-dir", MINIMR_INSTANCE_DIR.getPath(), "--shards", "1", }; assertNull(parser.parseArgs(args, conf, opts)); assertEquals(new Integer(1), opts.shards); assertEmptySystemErrAndEmptySystemOut(); } @Test public void testArgsParserUpdateConflictResolver() { String[] args = new String[] { "--input-list", "file:///tmp", "--morphline-file", MORPHLINE_FILE, "--output-dir", "file:/tmp/foo", "--solr-home-dir", MINIMR_INSTANCE_DIR.getPath(), "--shards", "1", "--update-conflict-resolver", NoChangeUpdateConflictResolver.class.getName(), }; assertNull(parser.parseArgs(args, conf, opts)); assertEquals(NoChangeUpdateConflictResolver.class.getName(), opts.updateConflictResolver); assertEmptySystemErrAndEmptySystemOut(); } @Test public void testArgsParserUnknownArgName() throws Exception { String[] args = new String[] { "--xxxxxxxxinputlist", "file:///tmp", "--morphline-file", MORPHLINE_FILE, "--output-dir", "file:/tmp/foo", "--solr-home-dir", MINIMR_INSTANCE_DIR.getPath(), "--shards", "1", }; assertArgumentParserException(args); } @Test public void testArgsParserFileNotFound1() throws Exception { String[] args = new String[] { "--input-list", "file:///tmp", "--morphline-file", MORPHLINE_FILE, "--output-dir", "file:/fileNotFound/foo", "--solr-home-dir", MINIMR_INSTANCE_DIR.getPath(), "--shards", "1", }; assertArgumentParserException(args); } @Test public void testArgsParserFileNotFound2() throws Exception { String[] args = new String[] { "--input-list", "file:///tmp", "--morphline-file", MORPHLINE_FILE, "--output-dir", "file:/tmp/foo", "--solr-home-dir", "/fileNotFound", "--shards", "1", }; assertArgumentParserException(args); } @Test public void testArgsParserIntOutOfRange() throws Exception { String[] args = new String[] { "--input-list", "file:///tmp", "--morphline-file", MORPHLINE_FILE, "--output-dir", "file:/tmp/foo", "--solr-home-dir", MINIMR_INSTANCE_DIR.getPath(), "--shards", "1", "--mappers", "-20" }; assertArgumentParserException(args); } @Test public void testArgsParserIllegalFanout() throws Exception { String[] args = new String[] { "--input-list", "file:///tmp", "--morphline-file", MORPHLINE_FILE, "--output-dir", "file:/tmp/foo", "--solr-home-dir", MINIMR_INSTANCE_DIR.getPath(), "--shards", "1", "--fanout", "1" // must be >= 2 }; assertArgumentParserException(args); } @Test public void testArgsParserSolrHomeMustContainSolrConfigFile() throws Exception { String[] args = new String[] { "--input-list", "file:///tmp", "--morphline-file", MORPHLINE_FILE, "--output-dir", "file:/tmp/foo", "--shards", "1", "--solr-home-dir", "/", }; assertArgumentParserException(args); } @Test public void testArgsShardUrlOk() { String[] args = new String[] { "--input-list", "file:///tmp", "--morphline-file", MORPHLINE_FILE, "--output-dir", "file:/tmp/foo", "--solr-home-dir", MINIMR_INSTANCE_DIR.getPath(), "--shard-url", "http://localhost:8983/solr/collection1", "--shard-url", "http://localhost:8983/solr/collection2", }; assertNull(parser.parseArgs(args, conf, opts)); assertEquals(Arrays.asList(Collections.singletonList("http://localhost:8983/solr/collection1"), Collections.singletonList("http://localhost:8983/solr/collection2")), opts.shardUrls); assertEquals(new Integer(2), opts.shards); assertEmptySystemErrAndEmptySystemOut(); } @Test public void testArgsShardUrlMustHaveAParam() throws Exception { String[] args = new String[] { "--input-list", "file:///tmp", "--morphline-file", MORPHLINE_FILE, "--output-dir", "file:/tmp/foo", "--solr-home-dir", MINIMR_INSTANCE_DIR.getPath(), "--shard-url", }; assertArgumentParserException(args); } @Test public void testArgsShardUrlAndShardsSucceeds() { String[] args = new String[] { "--input-list", "file:///tmp", "--morphline-file", MORPHLINE_FILE, "--output-dir", "file:/tmp/foo", "--solr-home-dir", MINIMR_INSTANCE_DIR.getPath(), "--shards", "1", "--shard-url", "http://localhost:8983/solr/collection1", }; assertNull(parser.parseArgs(args, conf, opts)); assertEmptySystemErrAndEmptySystemOut(); } @Test public void testArgsShardUrlNoGoLive() { String[] args = new String[] { "--input-list", "file:///tmp", "--morphline-file", MORPHLINE_FILE, "--output-dir", "file:/tmp/foo", "--solr-home-dir", MINIMR_INSTANCE_DIR.getPath(), "--shard-url", "http://localhost:8983/solr/collection1" }; assertNull(parser.parseArgs(args, conf, opts)); assertEmptySystemErrAndEmptySystemOut(); assertEquals(new Integer(1), opts.shards); } @Test public void testArgsShardUrlsAndZkhostAreMutuallyExclusive() throws Exception { String[] args = new String[] { "--input-list", "file:///tmp", "--morphline-file", MORPHLINE_FILE, "--output-dir", "file:/tmp/foo", "--solr-home-dir", MINIMR_INSTANCE_DIR.getPath(), "--shard-url", "http://localhost:8983/solr/collection1", "--shard-url", "http://localhost:8983/solr/collection1", "--zk-host", "http://localhost:2185", "--go-live" }; assertArgumentParserException(args); } @Test public void testArgsGoLiveAndSolrUrl() { String[] args = new String[] { "--input-list", "file:///tmp", "--morphline-file", MORPHLINE_FILE, "--output-dir", "file:/tmp/foo", "--solr-home-dir", MINIMR_INSTANCE_DIR.getPath(), "--shard-url", "http://localhost:8983/solr/collection1", "--shard-url", "http://localhost:8983/solr/collection1", "--go-live" }; Integer result = parser.parseArgs(args, conf, opts); assertNull(result); assertEmptySystemErrAndEmptySystemOut(); } @Test public void testArgsZkHostNoGoLive() throws Exception { String[] args = new String[] { "--input-list", "file:///tmp", "--morphline-file", MORPHLINE_FILE, "--output-dir", "file:/tmp/foo", "--solr-home-dir", MINIMR_INSTANCE_DIR.getPath(), "--zk-host", "http://localhost:2185", }; assertArgumentParserException(args); } @Test public void testArgsGoLiveZkHostNoCollection() throws Exception { String[] args = new String[] { "--input-list", "file:///tmp", "--morphline-file", MORPHLINE_FILE, "--output-dir", "file:/tmp/foo", "--solr-home-dir", MINIMR_INSTANCE_DIR.getPath(), "--zk-host", "http://localhost:2185", "--go-live" }; assertArgumentParserException(args); } @Test public void testArgsGoLiveNoZkHostOrSolrUrl() throws Exception { String[] args = new String[] { "--input-list", "file:///tmp", "--morphline-file", MORPHLINE_FILE, "--output-dir", "file:/tmp/foo", "--solr-home-dir", MINIMR_INSTANCE_DIR.getPath(), "--go-live" }; assertArgumentParserException(args); } @Test public void testNoSolrHomeDirOrZKHost() throws Exception { String[] args = new String[] { "--input-list", "file:///tmp", "--morphline-file", MORPHLINE_FILE, "--output-dir", "file:/tmp/foo", "--shards", "1", }; assertArgumentParserException(args); } @Test public void testZKHostNoSolrHomeDirOk() { String[] args = new String[] { "--input-list", "file:///tmp", "--morphline-file", MORPHLINE_FILE, "--output-dir", "file:/tmp/foo", "--zk-host", "http://localhost:2185", "--collection", "collection1", }; assertNull(parser.parseArgs(args, conf, opts)); assertEmptySystemErrAndEmptySystemOut(); } private void assertEmptySystemErrAndEmptySystemOut() { assertEquals(0, bout.toByteArray().length); assertEquals(0, berr.toByteArray().length); } private void assertArgumentParserException(String[] args) throws UnsupportedEncodingException { assertEquals("should have returned fail code", new Integer(1), parser.parseArgs(args, conf, opts)); assertEquals("no sys out expected:" + new String(bout.toByteArray(), StandardCharsets.UTF_8), 0, bout.toByteArray().length); String usageText; usageText = new String(berr.toByteArray(), StandardCharsets.UTF_8); assertTrue("should start with usage msg \"usage: hadoop \":" + usageText, usageText.startsWith("usage: hadoop ")); } }