Java tutorial
/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.solr.hadoop; import java.io.File; import java.io.IOException; import java.lang.invoke.MethodHandles; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.Comparator; import java.util.List; import com.google.common.io.Files; import org.apache.commons.io.FileUtils; import org.apache.solr.cloud.ZkController; import org.apache.solr.common.SolrException; import org.apache.solr.common.cloud.Aliases; import org.apache.solr.common.cloud.ClusterState; import org.apache.solr.common.cloud.DocCollection; import org.apache.solr.common.cloud.Replica; import org.apache.solr.common.cloud.Slice; import org.apache.solr.common.cloud.SolrZkClient; import org.apache.solr.common.cloud.ZkConfigManager; import org.apache.solr.common.cloud.ZkCoreNodeProps; import org.apache.solr.common.cloud.ZkNodeProps; import org.apache.solr.common.cloud.ZkStateReader; import org.apache.solr.common.util.StrUtils; import org.apache.zookeeper.KeeperException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * Extracts SolrCloud information from ZooKeeper. */ final class ZooKeeperInspector { private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); public List<List<String>> extractShardUrls(String zkHost, String collection) { DocCollection docCollection = extractDocCollection(zkHost, collection); List<Slice> slices = getSortedSlices(docCollection.getSlices()); List<List<String>> solrUrls = new ArrayList<>(slices.size()); for (Slice slice : slices) { if (slice.getLeader() == null) { throw new IllegalArgumentException("Cannot find SolrCloud slice leader. " + "It looks like not all of your shards are registered in ZooKeeper yet"); } Collection<Replica> replicas = slice.getReplicas(); List<String> urls = new ArrayList<>(replicas.size()); for (Replica replica : replicas) { ZkCoreNodeProps props = new ZkCoreNodeProps(replica); urls.add(props.getCoreUrl()); } solrUrls.add(urls); } return solrUrls; } public DocCollection extractDocCollection(String zkHost, String collection) { if (collection == null) { throw new IllegalArgumentException("collection must not be null"); } SolrZkClient zkClient = getZkClient(zkHost); try (ZkStateReader zkStateReader = new ZkStateReader(zkClient)) { try { // first check for alias collection = checkForAlias(zkClient, collection); zkStateReader.createClusterStateWatchersAndUpdate(); } catch (Exception e) { throw new IllegalArgumentException( "Cannot find expected information for SolrCloud in ZooKeeper: " + zkHost, e); } try { return zkStateReader.getClusterState().getCollection(collection); } catch (SolrException e) { throw new IllegalArgumentException( "Cannot find collection '" + collection + "' in ZooKeeper: " + zkHost, e); } } finally { zkClient.close(); } } public SolrZkClient getZkClient(String zkHost) { if (zkHost == null) { throw new IllegalArgumentException("zkHost must not be null"); } SolrZkClient zkClient; try { zkClient = new SolrZkClient(zkHost, 30000); } catch (Exception e) { throw new IllegalArgumentException("Cannot connect to ZooKeeper: " + zkHost, e); } return zkClient; } public List<Slice> getSortedSlices(Collection<Slice> slices) { List<Slice> sorted = new ArrayList(slices); Collections.sort(sorted, (slice1, slice2) -> { Comparator c = new AlphaNumericComparator(); return c.compare(slice1.getName(), slice2.getName()); }); LOG.trace("Sorted slices: {}", sorted); return sorted; } /** * Returns config value given collection name * Borrowed heavily from Solr's ZKController. */ public String readConfigName(SolrZkClient zkClient, String collection) throws KeeperException, InterruptedException { if (collection == null) { throw new IllegalArgumentException("collection must not be null"); } String configName = null; // first check for alias collection = checkForAlias(zkClient, collection); String path = ZkStateReader.COLLECTIONS_ZKNODE + "/" + collection; if (LOG.isInfoEnabled()) { LOG.info("Load collection config from:" + path); } byte[] data = zkClient.getData(path, null, null, true); if (data != null) { ZkNodeProps props = ZkNodeProps.load(data); configName = props.getStr(ZkController.CONFIGNAME_PROP); } if (configName != null && !zkClient.exists(ZkConfigManager.CONFIGS_ZKNODE + "/" + configName, true)) { LOG.error("Specified config does not exist in ZooKeeper:" + configName); throw new IllegalArgumentException("Specified config does not exist in ZooKeeper:" + configName); } return configName; } private String checkForAlias(SolrZkClient zkClient, String collection) throws KeeperException, InterruptedException { byte[] aliasData = zkClient.getData(ZkStateReader.ALIASES, null, null, true); Aliases aliases = ClusterState.load(aliasData); String alias = aliases.getCollectionAlias(collection); if (alias != null) { List<String> aliasList = StrUtils.splitSmart(alias, ",", true); if (aliasList.size() > 1) { throw new IllegalArgumentException( "collection cannot be an alias that maps to multiple collections"); } collection = aliasList.get(0); } return collection; } /** * Download and return the config directory from ZK */ public File downloadConfigDir(SolrZkClient zkClient, String configName) throws IOException, InterruptedException, KeeperException { File dir = Files.createTempDir(); dir.deleteOnExit(); ZkConfigManager configManager = new ZkConfigManager(zkClient); configManager.downloadConfigDir(configName, dir.toPath()); File confDir = new File(dir, "conf"); if (!confDir.isDirectory()) { // create a temporary directory with "conf" subdir and mv the config in there. This is // necessary because of CDH-11188; solrctl does not generate nor accept directories with e.g. // conf/solrconfig.xml which is necessary for proper solr operation. This should work // even if solrctl changes. confDir = new File(Files.createTempDir().getAbsolutePath(), "conf"); confDir.getParentFile().deleteOnExit(); Files.move(dir, confDir); dir = confDir.getParentFile(); } FileUtils.writeStringToFile(new File(dir, "solr.xml"), "<solr><solrcloud></solrcloud></solr>", "UTF-8"); verifyConfigDir(confDir); return dir; } private void verifyConfigDir(File confDir) throws IOException { File solrConfigFile = new File(confDir, "solrconfig.xml"); if (!solrConfigFile.exists()) { throw new IOException("Detected invalid Solr config dir in ZooKeeper - Reason: File not found: " + solrConfigFile.getName()); } if (!solrConfigFile.isFile()) { throw new IOException("Detected invalid Solr config dir in ZooKeeper - Reason: Not a file: " + solrConfigFile.getName()); } if (!solrConfigFile.canRead()) { throw new IOException("Insufficient permissions to read file: " + solrConfigFile); } } }