Java tutorial
/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.ignite.internal.processors.hadoop.impl.igfs; import java.net.URI; import java.util.concurrent.atomic.AtomicInteger; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.ignite.Ignition; import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.hadoop.fs.v1.IgniteHadoopFileSystem; import org.apache.ignite.internal.IgniteInternalFuture; import org.apache.ignite.internal.IgnitionEx; import org.apache.ignite.internal.processors.resource.GridSpringResourceContext; import org.apache.ignite.internal.util.typedef.X; import org.apache.ignite.internal.util.typedef.internal.S; import org.apache.ignite.lang.IgniteBiTuple; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; /** * IGFS Hadoop file system Ignite client -based self test for PROXY mode. */ public class IgniteHadoopFileSystemClientBasedOpenTest extends GridCommonAbstractTest { /** Config root path. */ private static final String[] CONFIGS = { "modules/hadoop/src/test/config/hadoop-fs-open-test/grid-0.xml", "modules/hadoop/src/test/config/hadoop-fs-open-test/grid-1.xml", "modules/hadoop/src/test/config/hadoop-fs-open-test/grid-2.xml" }; /** Nodes types. */ private static NodeType[][] nodesTypes = new NodeType[][] { { NodeType.REMOTE, NodeType.REMOTE, NodeType.REMOTE }, { NodeType.LOCAL, NodeType.REMOTE, NodeType.REMOTE }, { NodeType.REMOTE, NodeType.LOCAL, NodeType.REMOTE }, { NodeType.LOCAL, NodeType.LOCAL, NodeType.REMOTE }, { NodeType.REMOTE, NodeType.REMOTE, NodeType.LOCAL }, { NodeType.LOCAL, NodeType.REMOTE, NodeType.LOCAL }, { NodeType.REMOTE, NodeType.LOCAL, NodeType.LOCAL }, { NodeType.LOCAL, NodeType.LOCAL, NodeType.LOCAL }, { NodeType.LOCAL_SAME_NAME, NodeType.REMOTE, NodeType.REMOTE }, { NodeType.REMOTE, NodeType.LOCAL_SAME_NAME, NodeType.REMOTE }, { NodeType.LOCAL_SAME_NAME, NodeType.LOCAL_SAME_NAME, NodeType.REMOTE }, { NodeType.REMOTE, NodeType.REMOTE, NodeType.LOCAL_SAME_NAME }, { NodeType.LOCAL_SAME_NAME, NodeType.REMOTE, NodeType.LOCAL_SAME_NAME }, { NodeType.REMOTE, NodeType.LOCAL_SAME_NAME, NodeType.LOCAL_SAME_NAME }, { NodeType.LOCAL_SAME_NAME, NodeType.LOCAL_SAME_NAME, NodeType.LOCAL_SAME_NAME }, }; /** Test threads count. */ private static final int THREADS_COUNT = 20; /** Iterations count. */ private static final int ITERATIONS = 10; /** Iterations count. */ private static boolean skipInProc; /** * @param idx Grid index. * @return Path to Ignite config file. */ private static String cfgPath(int idx) { return CONFIGS[idx]; } /** * @param idx Grid index. * @return Path to Ignite config file. */ private static String uri(int idx) { return "igfs://" + authority(idx) + '/'; } /** {@inheritDoc} */ @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception { IgniteBiTuple<IgniteConfiguration, GridSpringResourceContext> cfgPair = IgnitionEx .loadConfiguration(cfgPath(getTestIgniteInstanceIndex(gridName))); IgniteConfiguration cfg = cfgPair.get1(); cfg.setIgniteInstanceName(gridName); return cfg; } /** {@inheritDoc} */ @Override protected void afterTestsStopped() throws Exception { stopAllGrids(); super.afterTestsStopped(); } /** {@inheritDoc} */ @Override protected long getTestTimeout() { return 15 * 60_000; } /** * Create configuration for test. * * @param idx Grid index. * @return Configuration. */ protected Configuration configuration(int idx) { Configuration cfg = new Configuration(); cfg.set("fs.defaultFS", "igfs://" + authority(idx) + '/'); cfg.set("fs.igfs.impl", IgniteHadoopFileSystem.class.getName()); cfg.set("fs.AbstractFileSystem.igfs.impl", org.apache.ignite.hadoop.fs.v2.IgniteHadoopFileSystem.class.getName()); cfg.setBoolean("fs.igfs.impl.disable.cache", true); cfg.setStrings(String.format(HadoopIgfsUtils.PARAM_IGFS_ENDPOINT_IGNITE_CFG_PATH, authority(idx)), cfgPath(idx)); if (skipInProc) cfg.setBoolean(String.format(HadoopIgfsUtils.PARAM_IGFS_ENDPOINT_NO_EMBED, authority(idx)), true); return cfg; } /** * Create configuration for test. * * @return Configuration. */ protected Configuration configurationWrongIgfs() { Configuration cfg = new Configuration(); cfg.set("fs.defaultFS", "igfs://igfs-wrong-name@/"); cfg.set("fs.igfs.impl", IgniteHadoopFileSystem.class.getName()); cfg.set("fs.AbstractFileSystem.igfs.impl", org.apache.ignite.hadoop.fs.v2.IgniteHadoopFileSystem.class.getName()); cfg.setBoolean("fs.igfs.impl.disable.cache", true); cfg.setStrings(String.format(HadoopIgfsUtils.PARAM_IGFS_ENDPOINT_IGNITE_CFG_PATH, "igfs-wrong-name@"), cfgPath(0)); return cfg; } /** * @param idx Grid index. * @return IGFS authority. */ private static String authority(int idx) { return "igfs-" + idx + "@"; } /** * @throws Exception If failed. */ public void testFsOpenMultithreaded() throws Exception { skipInProc = false; checkFsOpenWithAllNodesTypes(); } /** * @throws Exception If failed. */ private void checkFsOpenWithAllNodesTypes() throws Exception { for (int i = 0; i < nodesTypes.length; ++i) { log.info("Begin test case for nodes: " + S.arrayToString(NodeType.class, nodesTypes[i])); startNodes(nodesTypes[i]); // Await clusters start Thread.sleep(10_000); try { checkFsOpenMultithreaded(); } finally { stopAllGrids(); } } } /** * @throws Exception If failed. */ public void testFsOpenMultithreadedSkipInProc() throws Exception { skipInProc = true; checkFsOpenWithAllNodesTypes(); } /** * @throws Exception If failed. */ public void testIgniteClientWithIgfsMisconfigure() throws Exception { startNodes(new NodeType[] { NodeType.REMOTE, NodeType.REMOTE, NodeType.REMOTE }); // Await clusters start Thread.sleep(10_000); try { try (FileSystem fs = FileSystem.get(new URI("igfs://igfs-wrong-name@"), configurationWrongIgfs())) { FSDataOutputStream out = fs.create(new Path("igfs://igfs-wrong-name@/file"), true); assert false : "Exception must be thrown"; } catch (Exception e) { assertTrue(e.getMessage() .contains("Ignite client node doesn't have IGFS with the given name: igfs-wrong-name")); } } finally { stopAllGrids(); } } /** * @param nodeTypes Types of server nodes. * @throws Exception If failed. */ public void startNodes(NodeType[] nodeTypes) throws Exception { assert nodeTypes.length == CONFIGS.length; for (int i = 0; i < CONFIGS.length; i++) { String name = getTestIgniteInstanceName(i); switch (nodeTypes[i]) { case REMOTE: startRemoteGrid(name, getConfiguration(name), null, null, false); break; case LOCAL: startGrid(name, getConfiguration(name)); break; case LOCAL_SAME_NAME: startGrid("IGFS-cli-" + i, getConfiguration(name)); break; } } } /** * @throws Exception If failed. */ public void checkFsOpenMultithreaded() throws Exception { X.println("Start hadoop client file system open test"); final AtomicInteger fileIdx = new AtomicInteger(); IgniteInternalFuture fut = multithreadedAsync(new Runnable() { @Override public void run() { for (int iter = 0; iter < ITERATIONS; iter++) { for (int i = 0; i < CONFIGS.length; i++) { try (FileSystem fs = FileSystem.get(new URI(uri(i)), configuration(i))) { FSDataOutputStream out = fs .create(new Path(uri(i) + "file" + fileIdx.getAndIncrement()), true); out.close(); } catch (Exception e) { e.printStackTrace(); assert false : "Unexpected exception"; } } } } }, THREADS_COUNT); fut.get(); } /** * Node type. */ enum NodeType { /** Remote node. */ REMOTE, /** Node in the test VM. */ LOCAL, /** Node in the test VM with the name equals to client node config. */ LOCAL_SAME_NAME } }