Java tutorial
package org.apache.blur.utils; /** * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ import static org.apache.blur.metrics.MetricsConstants.BLUR; import static org.apache.blur.metrics.MetricsConstants.ORG_APACHE_BLUR; import static org.apache.blur.metrics.MetricsConstants.THRIFT_CALLS; import static org.apache.blur.utils.BlurConstants.BLUR_CONTROLLER_FILTERED_SERVER_CLASS; import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_FILTERED_SERVER_CLASS; import static org.apache.blur.utils.BlurConstants.HADOOP_CONF; import static org.apache.blur.utils.BlurConstants.SHARD_PREFIX; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.ObjectInputStream; import java.io.ObjectOutputStream; import java.io.Serializable; import java.io.UnsupportedEncodingException; import java.lang.management.ManagementFactory; import java.lang.reflect.Constructor; import java.lang.reflect.InvocationHandler; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.lang.reflect.Proxy; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.Comparator; import java.util.Date; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.Properties; import java.util.Set; import java.util.UUID; import java.util.concurrent.Callable; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicLongArray; import org.apache.blur.BlurConfiguration; import org.apache.blur.concurrent.ThreadWatcher; import org.apache.blur.log.Log; import org.apache.blur.log.LogFactory; import org.apache.blur.lucene.search.PrimeDocCache; import org.apache.blur.manager.BlurHighlighter; import org.apache.blur.manager.results.BlurResultComparator; import org.apache.blur.manager.results.BlurResultIterable; import org.apache.blur.manager.results.BlurResultPeekableIteratorComparator; import org.apache.blur.manager.results.PeekableIterator; import org.apache.blur.server.BlurServerContext; import org.apache.blur.server.ControllerServerContext; import org.apache.blur.server.FilteredBlurServer; import org.apache.blur.server.ShardServerContext; import org.apache.blur.thirdparty.thrift_0_9_0.TBase; import org.apache.blur.thirdparty.thrift_0_9_0.TException; import org.apache.blur.thirdparty.thrift_0_9_0.protocol.TJSONProtocol; import org.apache.blur.thirdparty.thrift_0_9_0.transport.TMemoryBuffer; import org.apache.blur.thrift.BException; import org.apache.blur.thrift.SortFieldComparator; import org.apache.blur.thrift.UserConverter; import org.apache.blur.thrift.generated.Blur.Iface; import org.apache.blur.thrift.generated.BlurException; import org.apache.blur.thrift.generated.BlurQuery; import org.apache.blur.thrift.generated.BlurResult; import org.apache.blur.thrift.generated.BlurResults; import org.apache.blur.thrift.generated.FetchResult; import org.apache.blur.thrift.generated.Record; import org.apache.blur.thrift.generated.RecordMutation; import org.apache.blur.thrift.generated.RecordMutationType; import org.apache.blur.thrift.generated.Row; import org.apache.blur.thrift.generated.RowMutation; import org.apache.blur.thrift.generated.RowMutationType; import org.apache.blur.thrift.generated.Selector; import org.apache.blur.thrift.generated.SortFieldResult; import org.apache.blur.thrift.util.ResetableTMemoryBuffer; import org.apache.blur.trace.Trace; import org.apache.blur.trace.Tracer; import org.apache.blur.user.User; import org.apache.blur.user.UserContext; import org.apache.blur.zookeeper.ZookeeperPathConstants; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.lucene.document.Document; import org.apache.lucene.index.AtomicReader; import org.apache.lucene.index.BaseCompositeReader; import org.apache.lucene.index.BaseCompositeReaderUtil; import org.apache.lucene.index.DocsEnum; import org.apache.lucene.index.Fields; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.SlowCompositeReaderWrapper; import org.apache.lucene.index.Term; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.search.DocIdSet; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.Filter; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.TopDocs; import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.OpenBitSet; import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.KeeperException; import org.apache.zookeeper.KeeperException.Code; import org.apache.zookeeper.WatchedEvent; import org.apache.zookeeper.Watcher; import org.apache.zookeeper.ZooDefs.Ids; import org.apache.zookeeper.ZooKeeper; import com.yammer.metrics.Metrics; import com.yammer.metrics.core.Histogram; import com.yammer.metrics.core.MetricName; public class BlurUtil { private static final String UTF_8 = "UTF-8"; public final static SortFieldComparator SORT_FIELD_COMPARATOR = new SortFieldComparator(); private static final Log REQUEST_LOG = LogFactory.getLog("REQUEST_LOG"); private static final Log RESPONSE_LOG = LogFactory.getLog("RESPONSE_LOG"); private static final Object[] EMPTY_OBJECT_ARRAY = new Object[] {}; private static final Class<?>[] EMPTY_PARAMETER_TYPES = new Class[] {}; private static final Log LOG = LogFactory.getLog(BlurUtil.class); private static final String UNKNOWN = "UNKNOWN"; public static final Comparator<? super PeekableIterator<BlurResult, BlurException>> HITS_PEEKABLE_ITERATOR_COMPARATOR = new BlurResultPeekableIteratorComparator(); public static final Comparator<? super BlurResult> HITS_COMPARATOR = new BlurResultComparator(); public static final Term PRIME_DOC_TERM = new Term(BlurConstants.PRIME_DOC, BlurConstants.PRIME_DOC_VALUE); static class LoggerArgsState { private static final int MAX_BUFFER_SIZE = 16384; private static final int MAX_BUILDER_SIZE = 16384; TJSONProtocol _tjsonProtocol; ResetableTMemoryBuffer _buffer; StringBuilder _builder = new StringBuilder(); final int _size; LoggerArgsState(int size) { _size = size; _buffer = new ResetableTMemoryBuffer(_size); _tjsonProtocol = new TJSONProtocol(_buffer); } void reset() { if (_buffer.getArray().length >= MAX_BUFFER_SIZE) { _buffer = new ResetableTMemoryBuffer(_size); _tjsonProtocol = new TJSONProtocol(_buffer); } if (_builder.length() >= MAX_BUILDER_SIZE) { _builder = new StringBuilder(); } } } @SuppressWarnings("unchecked") public static <T extends Iface> T lastChanceErrorHandling(final T t, Class<T> clazz) { InvocationHandler handler = new InvocationHandler() { @Override public Object invoke(Object proxy, Method method, Object[] args) throws Throwable { try { return method.invoke(t, args); } catch (InvocationTargetException e) { Throwable targetException = e.getTargetException(); if (targetException instanceof BlurException) { throw targetException; } else if (targetException instanceof TException) { throw targetException; } else { throw new BException( "Unknown error during call on method [{0}], this means that the method is handling exceptions correctly.", targetException, method.getName()); } } } }; return (T) Proxy.newProxyInstance(clazz.getClassLoader(), new Class[] { clazz }, handler); } @SuppressWarnings("unchecked") public static <T extends Iface> T recordMethodCallsAndAverageTimes(final T t, Class<T> clazz, final boolean controller) { final Map<String, Histogram> histogramMap = new ConcurrentHashMap<String, Histogram>(); Method[] declaredMethods = Iface.class.getDeclaredMethods(); for (Method m : declaredMethods) { String name = m.getName(); histogramMap.put(name, Metrics.newHistogram(new MetricName(ORG_APACHE_BLUR, BLUR, name, THRIFT_CALLS))); } final String prefix = new SimpleDateFormat("yyyyMMddHHmmss").format(new Date()); InvocationHandler handler = new InvocationHandler() { private final AtomicLong _requestCounter = new AtomicLong(); private ThreadLocal<LoggerArgsState> _loggerArgsState = new ThreadLocal<LoggerArgsState>() { @Override protected LoggerArgsState initialValue() { return new LoggerArgsState(1024); } }; @Override public Object invoke(Object proxy, Method method, Object[] args) throws Throwable { long requestNumber = _requestCounter.incrementAndGet(); String requestId = prefix + "-" + requestNumber; String tracingConnectionString; String connectionString; if (controller) { ControllerServerContext controllerServerContext = ControllerServerContext .getControllerServerContext(); if (controllerServerContext == null) { connectionString = "unknown"; tracingConnectionString = "unknown"; } else { connectionString = controllerServerContext.getConnectionString("\t"); tracingConnectionString = controllerServerContext.getConnectionString(":"); } } else { ShardServerContext shardServerContext = ShardServerContext.getShardServerContext(); if (shardServerContext == null) { connectionString = "unknown"; tracingConnectionString = "unknown"; } else { connectionString = shardServerContext.getConnectionString("\t"); tracingConnectionString = shardServerContext.getConnectionString(":"); } } String argsStr = null; long start = System.nanoTime(); String name = method.getName(); boolean error = false; LoggerArgsState loggerArgsState = null; Tracer trace = Trace.trace("thrift recv", Trace.param("method", method.getName()), Trace.param("connection", tracingConnectionString)); User user = UserContext.getUser(); boolean notSetUserMethod = isNotSetUserMethod(name); try { if (REQUEST_LOG.isInfoEnabled() && notSetUserMethod) { if (argsStr == null) { loggerArgsState = _loggerArgsState.get(); argsStr = getArgsStr(args, name, loggerArgsState); } REQUEST_LOG.info(getRequestLogMessage(requestId, connectionString, argsStr, name, user)); } return method.invoke(t, args); } catch (InvocationTargetException e) { error = true; throw e.getTargetException(); } finally { trace.done(); long end = System.nanoTime(); double ms = (end - start) / 1000000.0; if (RESPONSE_LOG.isInfoEnabled() && notSetUserMethod) { if (argsStr == null) { if (loggerArgsState == null) { loggerArgsState = _loggerArgsState.get(); } argsStr = getArgsStr(args, name, loggerArgsState); } if (error) { RESPONSE_LOG.info(getErrorResponseLogMessage(requestId, connectionString, argsStr, name, ms, user)); } else { RESPONSE_LOG.info(getSuccessfulResponseLogMessage(requestId, connectionString, argsStr, name, ms, user)); } } Histogram histogram = histogramMap.get(name); histogram.update((end - start) / 1000); if (loggerArgsState != null) { loggerArgsState.reset(); } } } private boolean isNotSetUserMethod(String name) { if (name.equals("setUser")) { return false; } return true; } }; return (T) Proxy.newProxyInstance(clazz.getClassLoader(), new Class[] { clazz }, handler); } public static String getArgsStr(Object[] args, String methodName, LoggerArgsState loggerArgsState) { String argsStr; if (methodName.equals("mutate")) { RowMutation rowMutation = (RowMutation) args[0]; if (rowMutation == null) { argsStr = "[null]"; } else { String table = rowMutation.getTable(); String rowId = rowMutation.getRowId(); if (table != null) { table = "\"" + table + "\""; } if (rowId != null) { rowId = "\"" + rowId + "\""; } argsStr = "[{\"table\":" + table + ",\"rowId\":" + rowId + "}]"; } } else if (methodName.equals("mutateBatch")) { argsStr = "[\"Batch Mutate\"]"; } else if (methodName.equals("bulkMutateAdd")) { argsStr = "[\"Bulk Mutate Add\"]"; } else if (methodName.equals("bulkMutateAddMultiple")) { argsStr = "[\"Bulk Mutate Add Multiple\"]"; } else { argsStr = getArgsStr(args, loggerArgsState); } return argsStr; } public static String getArgsStr(Object[] args, LoggerArgsState loggerArgsState) { if (args == null) { return null; } StringBuilder builder = loggerArgsState._builder; builder.setLength(0); for (Object o : args) { if (builder.length() == 0) { builder.append('['); } else { builder.append(','); } builder.append(getArgsStr(o, loggerArgsState)); } if (builder.length() != 0) { builder.append(']'); } return builder.toString(); } @SuppressWarnings("rawtypes") public static String getArgsStr(Object o, LoggerArgsState loggerArgsState) { if (o == null) { return null; } if (o instanceof TBase) { return getArgsStr((TBase) o, loggerArgsState); } return o.toString(); } @SuppressWarnings("rawtypes") public static String getArgsStr(TBase o, LoggerArgsState loggerArgsState) { ResetableTMemoryBuffer buffer = loggerArgsState._buffer; TJSONProtocol tjsonProtocol = loggerArgsState._tjsonProtocol; buffer.resetBuffer(); tjsonProtocol.reset(); try { o.write(tjsonProtocol); } catch (TException e) { LOG.error("Unknown error tyring to write object [{0}] to json.", e, o); } byte[] array = buffer.getArray(); int length = buffer.length(); return new String(array, 0, length); } public static String getRequestLogMessage(String requestId, String connectionString, String argsStr, String name, User user) { String u = "null"; if (user != null) { u = "\"" + user.getUsername() + "\""; } return "{\"id\":\"" + requestId + "\", \"con\":\"" + connectionString + "\", \"user\":" + u + ", \"meth\":\"" + name + "\", \"args\":" + argsStr + "}"; } public static String getResponseLogMessage(String requestId, String connectionString, String argsStr, String name, double ms, User user, boolean success) { String u = "null"; if (user != null) { u = "\"" + user.getUsername() + "\""; } String response; if (success) { response = "OK"; } else { response = "ERROR"; } return "{\"id\":\"" + requestId + "\", \"response\":\"" + response + "\", \"time\":" + Double.toString(ms) + ", \"con\":\"" + connectionString + "\", \"user\":" + u + ", \"meth\":\"" + name + "\", \"args\":" + argsStr + "}"; } public static String getSuccessfulResponseLogMessage(String requestId, String connectionString, String argsStr, String name, double ms, User user) { return getResponseLogMessage(requestId, connectionString, argsStr, name, ms, user, true); } public static String getErrorResponseLogMessage(String requestId, String connectionString, String argsStr, String name, double ms, User user) { return getResponseLogMessage(requestId, connectionString, argsStr, name, ms, user, false); } public static void setupZookeeper(ZooKeeper zookeeper) throws KeeperException, InterruptedException { setupZookeeper(zookeeper, null); } public synchronized static void setupZookeeper(ZooKeeper zookeeper, String cluster) throws KeeperException, InterruptedException { BlurUtil.createIfMissing(zookeeper, ZookeeperPathConstants.getBasePath()); BlurUtil.createIfMissing(zookeeper, ZookeeperPathConstants.getOnlineControllersPath()); BlurUtil.createIfMissing(zookeeper, ZookeeperPathConstants.getClustersPath()); if (cluster != null) { BlurUtil.createIfMissing(zookeeper, ZookeeperPathConstants.getClusterPath(cluster)); BlurUtil.createIfMissing(zookeeper, ZookeeperPathConstants.getSafemodePath(cluster)); BlurUtil.createIfMissing(zookeeper, ZookeeperPathConstants.getOnlineShardsPath(cluster)); BlurUtil.createIfMissing(zookeeper, ZookeeperPathConstants.getTablesPath(cluster)); } } public static void createIfMissing(ZooKeeper zookeeper, String path) throws KeeperException, InterruptedException { if (zookeeper.exists(path, false) == null) { try { zookeeper.create(path, null, Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); } catch (KeeperException e) { if (e.code() == Code.NODEEXISTS) { return; } throw e; } } } public static List<Long> getList(AtomicLongArray atomicLongArray) { if (atomicLongArray == null) { return null; } List<Long> counts = new ArrayList<Long>(atomicLongArray.length()); for (int i = 0; i < atomicLongArray.length(); i++) { counts.add(atomicLongArray.get(i)); } return counts; } public static void quietClose(Object... close) { if (close == null) { return; } for (Object object : close) { if (object != null) { close(object); } } } private static void close(Object object) { Class<? extends Object> clazz = object.getClass(); Method method; try { method = clazz.getMethod("close", EMPTY_PARAMETER_TYPES); } catch (SecurityException e) { throw new RuntimeException(e); } catch (NoSuchMethodException e) { return; } try { method.invoke(object, EMPTY_OBJECT_ARRAY); } catch (Exception e) { LOG.error("Error while trying to close object [{0}]", e, object); } } public static byte[] toBytes(Serializable serializable) { try { ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); ObjectOutputStream stream = new ObjectOutputStream(outputStream); stream.writeObject(serializable); stream.close(); return outputStream.toByteArray(); } catch (Exception e) { throw new RuntimeException(e); } } public static Serializable fromBytes(byte[] bs) { return fromBytes(bs, 0, bs.length); } public static Serializable fromBytes(byte[] bs, int offset, int length) { ObjectInputStream stream = null; try { stream = new ObjectInputStream(new ByteArrayInputStream(bs, offset, length)); return (Serializable) stream.readObject(); } catch (Exception e) { throw new RuntimeException(e); } finally { if (stream != null) { try { stream.close(); } catch (IOException e) { // eat } } } } public static List<Long> toList(AtomicLongArray atomicLongArray) { if (atomicLongArray == null) { return null; } int length = atomicLongArray.length(); List<Long> result = new ArrayList<Long>(length); for (int i = 0; i < length; i++) { result.add(atomicLongArray.get(i)); } return result; } public static AtomicLongArray getAtomicLongArraySameLengthAsList(List<?> list) { if (list == null) { return null; } return new AtomicLongArray(list.size()); } public static BlurResults convertToHits(BlurResultIterable hitsIterable, BlurQuery query, AtomicLongArray facetCounts, ExecutorService executor, Selector selector, final Iface iface, final String table) throws InterruptedException, ExecutionException, BlurException { BlurResults results = new BlurResults(); results.setTotalResults(hitsIterable.getTotalResults()); results.setShardInfo(hitsIterable.getShardInfo()); if (query.minimumNumberOfResults > 0) { Tracer trace = Trace.trace("convertToHits - skipTo", Trace.param("start", query.start)); hitsIterable.skipTo(query.start); trace.done(); BlurIterator<BlurResult, BlurException> iterator = hitsIterable.iterator(); for (int count = 0; count < query.fetch && iterator.hasNext(); count++) { results.addToResults(iterator.next()); } } if (results.results == null) { results.results = new ArrayList<BlurResult>(); } if (facetCounts != null) { results.facetCounts = BlurUtil.toList(facetCounts); } if (selector != null) { List<Future<FetchResult>> futures = new ArrayList<Future<FetchResult>>(); for (int i = 0; i < results.results.size(); i++) { BlurResult result = results.results.get(i); final Selector s = new Selector(selector); s.setLocationId(result.locationId); futures.add(executor.submit(new Callable<FetchResult>() { @Override public FetchResult call() throws Exception { return iface.fetchRow(table, s); } })); } for (int i = 0; i < results.results.size(); i++) { Future<FetchResult> future = futures.get(i); BlurResult result = results.results.get(i); result.setFetchResult(future.get()); } } results.query = query; results.query.selector = selector; return results; } public static void setStartTime(BlurQuery query) { if (query.startTime == 0) { query.startTime = System.currentTimeMillis(); } } public static String getVersion() { String path = "/META-INF/maven/org.apache.blur/blur-core/pom.properties"; InputStream inputStream = BlurUtil.class.getResourceAsStream(path); if (inputStream == null) { return UNKNOWN; } Properties prop = new Properties(); try { prop.load(inputStream); } catch (IOException e) { LOG.error("Unknown error while getting version.", e); return UNKNOWN; } Object verison = prop.get("version"); if (verison == null) { return UNKNOWN; } return verison.toString(); } public static void unlockForSafeMode(ZooKeeper zookeeper, String lockPath) throws InterruptedException, KeeperException { zookeeper.delete(lockPath, -1); LOG.info("Lock released."); } public static String lockForSafeMode(ZooKeeper zookeeper, String nodeName, String cluster) throws KeeperException, InterruptedException { LOG.info("Getting safe mode lock."); final Object lock = new Object(); String blurSafemodePath = ZookeeperPathConstants.getSafemodePath(cluster); String newPath = zookeeper.create(blurSafemodePath + "/safemode-", nodeName.getBytes(), Ids.OPEN_ACL_UNSAFE, CreateMode.EPHEMERAL_SEQUENTIAL); Watcher watcher = new Watcher() { @Override public void process(WatchedEvent event) { synchronized (lock) { lock.notifyAll(); } } }; while (true) { synchronized (lock) { List<String> children = new ArrayList<String>(zookeeper.getChildren(blurSafemodePath, watcher)); Collections.sort(children); if (newPath.equals(blurSafemodePath + "/" + children.get(0))) { LOG.info("Lock aquired."); return newPath; } else { lock.wait(BlurConstants.ZK_WAIT_TIME); } } } } public static String humanizeTime(long time, TimeUnit unit) { long seconds = unit.toSeconds(time); long hours = getHours(seconds); seconds = seconds - TimeUnit.HOURS.toSeconds(hours); long minutes = getMinutes(seconds); seconds = seconds - TimeUnit.MINUTES.toSeconds(minutes); return humanizeTime(hours, minutes, seconds); } public static String humanizeTime(long hours, long minutes, long seconds) { StringBuilder builder = new StringBuilder(); if (hours == 0 && minutes != 0) { addMinutes(builder, minutes); } else if (hours != 0) { addHours(builder, hours); addMinutes(builder, minutes); } addSeconds(builder, seconds); return builder.toString().trim(); } private static void addHours(StringBuilder builder, long hours) { builder.append(hours).append(" hours "); } private static void addMinutes(StringBuilder builder, long minutes) { builder.append(minutes).append(" minutes "); } private static void addSeconds(StringBuilder builder, long seconds) { builder.append(seconds).append(" seconds "); } private static long getMinutes(long seconds) { return seconds / TimeUnit.MINUTES.toSeconds(1); } private static long getHours(long seconds) { return seconds / TimeUnit.HOURS.toSeconds(1); } public static void createPath(ZooKeeper zookeeper, String path, byte[] data) throws KeeperException, InterruptedException { zookeeper.create(path, data, Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); } public static void setupFileSystem(String uri, int shardCount, Configuration configuration) throws IOException { ThreadWatcher.status("creating shard dirs", 0.0f); Path tablePath = new Path(uri); FileSystem fileSystem = tablePath.getFileSystem(configuration); if (createPath(fileSystem, tablePath)) { LOG.info("Table uri existed."); // validateShardCount(shardCount, fileSystem, tablePath); } ThreadWatcher.resetStatus(); } public static void validateShardCount(int shardCount, FileSystem fileSystem, Path tablePath) throws IOException { // Check that all the directories that should be are in fact there. for (int i = 0; i < shardCount; i++) { Path path = new Path(tablePath, ShardUtil.getShardName(BlurConstants.SHARD_PREFIX, i)); if (!fileSystem.exists(path)) { LOG.error("Path [{0}] for shard [{1}] does not exist.", path, i); throw new RuntimeException("Path [" + path + "] for shard [" + i + "] does not exist."); } } FileStatus[] listStatus = fileSystem.listStatus(tablePath); for (FileStatus fs : listStatus) { Path path = fs.getPath(); String name = path.getName(); if (name.startsWith(SHARD_PREFIX)) { int index = name.indexOf('-'); String shardIndexStr = name.substring(index + 1); int shardIndex = Integer.parseInt(shardIndexStr); if (shardIndex >= shardCount) { LOG.error("Number of directories in table path [" + path + "] exceeds definition of [" + shardCount + "] shard count."); throw new RuntimeException("Number of directories in table path [" + path + "] exceeds definition of [" + shardCount + "] shard count."); } } } } public static boolean createPath(FileSystem fileSystem, Path path) throws IOException { if (!fileSystem.exists(path)) { LOG.info("Path [{0}] does not exist, creating.", path); if (!fileSystem.mkdirs(path)) { LOG.error("Path [{0}] was NOT created, make sure that you have correct permissions.", path); throw new IOException("Path [{0}] was NOT created, make sure that you have correct permissions."); } return false; } return true; } public static int zeroCheck(int i, String message) { if (i < 1) { throw new RuntimeException(message); } return i; } public static <T> T nullCheck(T t, String message) { if (t == null) { throw new NullPointerException(message); } return t; } public static byte[] read(TBase<?, ?> base) { if (base == null) { return null; } TMemoryBuffer trans = new TMemoryBuffer(1024); TJSONProtocol protocol = new TJSONProtocol(trans); try { base.write(protocol); } catch (TException e) { throw new RuntimeException(e); } trans.close(); byte[] buf = new byte[trans.length()]; System.arraycopy(trans.getArray(), 0, buf, 0, trans.length()); return buf; } public static void write(byte[] data, TBase<?, ?> base) { nullCheck(null, "Data cannot be null."); TMemoryBuffer trans = new TMemoryBuffer(1024); TJSONProtocol protocol = new TJSONProtocol(trans); try { trans.write(data); base.read(protocol); } catch (TException e) { throw new RuntimeException(e); } trans.close(); } public static void removeAll(ZooKeeper zooKeeper, String path) throws KeeperException, InterruptedException { List<String> list = zooKeeper.getChildren(path, false); for (String p : list) { removeAll(zooKeeper, path + "/" + p); } LOG.info("Removing path [{0}]", path); zooKeeper.delete(path, -1); } public static void removeIndexFiles(String uri, Configuration configuration) throws IOException { Path tablePath = new Path(uri); FileSystem fileSystem = tablePath.getFileSystem(configuration); fileSystem.delete(tablePath, true); } public static RowMutation toRowMutation(String table, Row row) { RowMutation rowMutation = new RowMutation(); rowMutation.setRowId(row.getId()); rowMutation.setTable(table); rowMutation.setRowMutationType(RowMutationType.REPLACE_ROW); List<Record> records = row.getRecords(); for (Record record : records) { rowMutation.addToRecordMutations(toRecordMutation(record)); } return rowMutation; } public static RecordMutation toRecordMutation(Record record) { RecordMutation recordMutation = new RecordMutation(); recordMutation.setRecord(record); recordMutation.setRecordMutationType(RecordMutationType.REPLACE_ENTIRE_RECORD); return recordMutation; } public static int countDocuments(IndexReader reader, Term term) throws IOException { TermQuery query = new TermQuery(term); IndexSearcher indexSearcher = new IndexSearcher(reader); TopDocs topDocs = indexSearcher.search(query, 1); return topDocs.totalHits; } /** * NOTE: This is a potentially dangerous call, it will return all the * documents that match the term. * * @param selector * @param primeDocTerm * @param filter * @param totalRecords * @param highlighter * * @throws IOException */ @SuppressWarnings("unchecked") public static List<Document> fetchDocuments(IndexReader reader, ResetableDocumentStoredFieldVisitor fieldSelector, Selector selector, int maxHeap, String context, Term primeDocTerm, Filter filter, AtomicBoolean moreToFetch, AtomicInteger totalRecords, BlurHighlighter highlighter) throws IOException { if (highlighter == null) { highlighter = new BlurHighlighter(); } if (reader instanceof BaseCompositeReader) { BaseCompositeReader<IndexReader> indexReader = (BaseCompositeReader<IndexReader>) reader; List<? extends IndexReader> sequentialSubReaders = BaseCompositeReaderUtil .getSequentialSubReaders(indexReader); String locationId = selector.getLocationId(); int indexOf = locationId.indexOf('/'); if (indexOf < 0) { throw new IOException("Location id [" + locationId + "] not valid"); } int notAdjustedRequestedDocId = Integer.parseInt(locationId.substring(indexOf + 1)); int readerIndex = BaseCompositeReaderUtil.readerIndex(indexReader, notAdjustedRequestedDocId); int readerBase = BaseCompositeReaderUtil.readerBase(indexReader, readerIndex); int requestedDocId = notAdjustedRequestedDocId - readerBase; IndexReader orgReader = sequentialSubReaders.get(readerIndex); if (orgReader != null && orgReader instanceof AtomicReader) { AtomicReader atomicReader = (AtomicReader) orgReader; Bits liveDocs = atomicReader.getLiveDocs(); OpenBitSet bitSet = PrimeDocCache.getPrimeDocBitSet(primeDocTerm, atomicReader); int primeDocId = bitSet.prevSetBit(requestedDocId); int nextPrimeDoc = bitSet.nextSetBit(primeDocId + 1); int numberOfDocsInRow; if (nextPrimeDoc == -1) { numberOfDocsInRow = atomicReader.maxDoc() - primeDocId; } else { numberOfDocsInRow = nextPrimeDoc - primeDocId; } OpenBitSet docsInRowSpanToFetch = getDocsToFetch(atomicReader, selector, primeDocId, numberOfDocsInRow, liveDocs, filter, totalRecords); int start = selector.getStartRecord(); int maxDocsToFetch = selector.getMaxRecordsToFetch(); int startingPosition = getStartingPosition(docsInRowSpanToFetch, start); List<Document> docs = new ArrayList<Document>(); if (startingPosition < 0) { // nothing to fetch return docs; } int totalHeap = 0; Tracer trace2 = Trace.trace("fetching docs from index"); int cursor = 0; try { for (cursor = startingPosition; cursor < numberOfDocsInRow; cursor++) { if (maxDocsToFetch <= 0) { return docs; } if (totalHeap >= maxHeap) { LOG.warn( "Max heap size exceeded for this request [{0}] max [{1}] for [{2}] and selector [{3}]", totalHeap, maxHeap, context, selector); return docs; } if (docsInRowSpanToFetch.fastGet(cursor)) { maxDocsToFetch--; int docID = primeDocId + cursor; atomicReader.document(docID, fieldSelector); Document document = fieldSelector.getDocument(); if (highlighter.shouldHighlight()) { docs.add(highlighter.highlight(docID, document, atomicReader)); } else { docs.add(document); } totalHeap += fieldSelector.getSize(); fieldSelector.reset(); } } } finally { if (docsInRowSpanToFetch.nextSetBit(cursor) != -1) { moreToFetch.set(true); } trace2.done(); } return orderDocsBasedOnFamilyOrder(docs, selector); } else { throw new IOException("Expecting a atomic reader got a [" + orgReader + "]"); } } throw new IOException("IndexReader [" + reader + "] is not a basecompsitereader"); } private static List<Document> orderDocsBasedOnFamilyOrder(List<Document> docs, Selector selector) { List<String> orderOfFamiliesToFetch = selector.getOrderOfFamiliesToFetch(); if (orderOfFamiliesToFetch == null || orderOfFamiliesToFetch.isEmpty()) { return docs; } Set<String> columnFamiliesToFetch = selector.getColumnFamiliesToFetch(); if (columnFamiliesToFetch != null) { orderOfFamiliesToFetch.addAll(columnFamiliesToFetch); } Map<String, Set<String>> columnsToFetch = selector.getColumnsToFetch(); if (columnsToFetch != null) { orderOfFamiliesToFetch.addAll(columnsToFetch.keySet()); } final Map<String, Integer> familyOrdering = getFamilyOrdering(orderOfFamiliesToFetch); Collections.sort(docs, new Comparator<Document>() { @Override public int compare(Document o1, Document o2) { String f1 = o1.get(BlurConstants.FAMILY); String f2 = o2.get(BlurConstants.FAMILY); int order1 = familyOrdering.get(f1); int order2 = familyOrdering.get(f2); return order1 - order2; } }); return docs; } private static Map<String, Integer> getFamilyOrdering(List<String> columnFamiliesToFetch) { Map<String, Integer> ordering = new HashMap<String, Integer>(); int i = 0; for (String family : columnFamiliesToFetch) { Integer integer = ordering.get(family); if (integer == null) { ordering.put(family, i); i++; } } return ordering; } private static int getStartingPosition(OpenBitSet docsInRowSpanToFetch, int start) { int docStartingPosition = docsInRowSpanToFetch.nextSetBit(0); int offset = 0; while (offset < start) { docStartingPosition = docsInRowSpanToFetch.nextSetBit(docStartingPosition + 1); offset++; } return docStartingPosition; } private static OpenBitSet getDocsToFetch(AtomicReader atomicReader, Selector selector, int primeDocRowId, int numberOfDocsInRow, Bits liveDocs, Filter filter, AtomicInteger totalRecords) throws IOException { Set<String> alreadyProcessed = new HashSet<String>(); OpenBitSet bits = new OpenBitSet(numberOfDocsInRow); OpenBitSet mask = null; if (filter != null) { DocIdSet docIdSet = filter.getDocIdSet(atomicReader.getContext(), liveDocs); mask = getMask(docIdSet, primeDocRowId, numberOfDocsInRow); } Set<String> columnFamiliesToFetch = selector.getColumnFamiliesToFetch(); boolean fetchAll = true; if (columnFamiliesToFetch != null) { fetchAll = false; applyFamilies(alreadyProcessed, bits, columnFamiliesToFetch, atomicReader, primeDocRowId, numberOfDocsInRow, liveDocs); } Map<String, Set<String>> columnsToFetch = selector.getColumnsToFetch(); if (columnsToFetch != null) { fetchAll = false; applyColumns(alreadyProcessed, bits, columnsToFetch, atomicReader, primeDocRowId, numberOfDocsInRow, liveDocs); } if (fetchAll) { bits.set(0, numberOfDocsInRow); } if (mask != null) { bits.intersect(mask); } totalRecords.set((int) bits.cardinality()); return bits; } private static OpenBitSet getMask(DocIdSet docIdSet, int primeDocRowId, int numberOfDocsInRow) throws IOException { OpenBitSet mask = new OpenBitSet(numberOfDocsInRow); DocIdSetIterator iterator = docIdSet.iterator(); if (iterator == null) { return mask; } int docId = iterator.advance(primeDocRowId); int end = numberOfDocsInRow + primeDocRowId; while (docId < end) { mask.set(docId - primeDocRowId); docId = iterator.nextDoc(); } return mask; } private static void applyColumns(Set<String> alreadyProcessed, OpenBitSet bits, Map<String, Set<String>> columnsToFetch, AtomicReader atomicReader, int primeDocRowId, int numberOfDocsInRow, Bits liveDocs) throws IOException { for (String family : columnsToFetch.keySet()) { if (!alreadyProcessed.contains(family)) { applyFamily(bits, family, atomicReader, primeDocRowId, numberOfDocsInRow, liveDocs); alreadyProcessed.add(family); } } } private static void applyFamilies(Set<String> alreadyProcessed, OpenBitSet bits, Set<String> columnFamiliesToFetch, AtomicReader segmentReader, int primeDocRowId, int numberOfDocsInRow, Bits liveDocs) throws IOException { for (String family : columnFamiliesToFetch) { if (!alreadyProcessed.contains(family)) { applyFamily(bits, family, segmentReader, primeDocRowId, numberOfDocsInRow, liveDocs); alreadyProcessed.add(family); } } } private static void applyFamily(OpenBitSet bits, String family, AtomicReader atomicReader, int primeDocRowId, int numberOfDocsInRow, Bits liveDocs) throws IOException { Fields fields = atomicReader.fields(); Terms terms = fields.terms(BlurConstants.FAMILY); TermsEnum iterator = terms.iterator(null); BytesRef text = new BytesRef(family); int lastDocId = primeDocRowId + numberOfDocsInRow; if (iterator.seekExact(text, true)) { DocsEnum docs = iterator.docs(liveDocs, null, DocsEnum.FLAG_NONE); int doc = primeDocRowId; while ((doc = docs.advance(doc)) < lastDocId) { bits.set(doc - primeDocRowId); } } } public static AtomicReader getAtomicReader(IndexReader reader) throws IOException { return SlowCompositeReaderWrapper.wrap(reader); } public static String getPid() { return ManagementFactory.getRuntimeMXBean().getName(); } public static <T, E extends Exception> BlurIterator<T, E> convert(final Iterator<T> iterator) { return new BlurIterator<T, E>() { private long _position = 0; @Override public boolean hasNext() throws E { return iterator.hasNext(); } @Override public T next() throws E { _position++; return iterator.next(); } @Override public long getPosition() throws E { return _position; } }; } public static void validateWritableDirectory(FileSystem fileSystem, Path tablePath) throws IOException { String tmpDir = UUID.randomUUID().toString(); String tmpFile = UUID.randomUUID().toString(); Path path = new Path(tablePath, tmpDir); if (!fileSystem.mkdirs(path)) { throw new IOException("Could not create new directory in [" + tablePath + "] "); } Path filePath = new Path(path, tmpFile); try { fileSystem.create(filePath).close(); } catch (IOException e) { throw new IOException("Could not create new filr in [" + path + "] "); } fileSystem.delete(path, true); } @SuppressWarnings("unchecked") public static Iface wrapFilteredBlurServer(BlurConfiguration configuration, Iface iface, boolean shard) { String className; if (!shard) { className = configuration.get(BLUR_CONTROLLER_FILTERED_SERVER_CLASS); } else { className = configuration.get(BLUR_SHARD_FILTERED_SERVER_CLASS); } if (className != null && !className.isEmpty()) { try { Class<? extends FilteredBlurServer> clazz = (Class<? extends FilteredBlurServer>) Class .forName(className); Constructor<? extends FilteredBlurServer> constructor = clazz .getConstructor(new Class[] { BlurConfiguration.class, Iface.class, Boolean.TYPE }); return constructor.newInstance(new Object[] { configuration, iface, shard }); } catch (ClassNotFoundException e) { throw new RuntimeException(e); } catch (NoSuchMethodException e) { throw new RuntimeException(e); } catch (SecurityException e) { throw new RuntimeException(e); } catch (InstantiationException e) { throw new RuntimeException(e); } catch (IllegalAccessException e) { throw new RuntimeException(e); } catch (IllegalArgumentException e) { throw new RuntimeException(e); } catch (InvocationTargetException e) { throw new RuntimeException(e); } } return iface; } public static Iface runWithUser(final Iface iface, final boolean controller) { InvocationHandler handler = new InvocationHandler() { @Override public Object invoke(Object proxy, Method method, Object[] args) throws Throwable { if (method.getName().equals("setUser")) { try { return method.invoke(iface, args); } catch (InvocationTargetException e) { throw e.getTargetException(); } } BlurServerContext context = getServerContext(controller); if (context == null) { try { return method.invoke(iface, args); } catch (InvocationTargetException e) { throw e.getTargetException(); } } UserContext.setUser(UserConverter.toUserFromThrift(context.getUser())); try { return method.invoke(iface, args); } catch (InvocationTargetException e) { throw e.getTargetException(); } finally { UserContext.reset(); } } private BlurServerContext getServerContext(boolean controller) { if (controller) { return ControllerServerContext.getControllerServerContext(); } return ShardServerContext.getShardServerContext(); } }; return (Iface) Proxy.newProxyInstance(Iface.class.getClassLoader(), new Class[] { Iface.class }, handler); } public static Iface runTrace(final Iface iface, final boolean controller) { InvocationHandler handler = new InvocationHandler() { @Override public Object invoke(Object proxy, Method method, Object[] args) throws Throwable { if (method.getName().equals("startTrace")) { try { return method.invoke(iface, args); } catch (InvocationTargetException e) { throw e.getTargetException(); } } BlurServerContext context = getServerContext(controller); if (context == null) { try { return method.invoke(iface, args); } catch (InvocationTargetException e) { throw e.getTargetException(); } } String rootTraceId = context.getTraceRootId(); if (rootTraceId != null) { Trace.setupTrace(rootTraceId, context.getTraceRequestId()); } try { return method.invoke(iface, args); } catch (InvocationTargetException e) { throw e.getTargetException(); } finally { Trace.tearDownTrace(); context.resetTraceIds(); } } private BlurServerContext getServerContext(boolean controller) { if (controller) { return ControllerServerContext.getControllerServerContext(); } return ShardServerContext.getShardServerContext(); } }; return (Iface) Proxy.newProxyInstance(Iface.class.getClassLoader(), new Class[] { Iface.class }, handler); } public static List<SortFieldResult> convertToSortFields(Object[] fields) { if (fields == null) { return null; } if (fields.length == 0) { return Arrays.asList(); } List<SortFieldResult> result = new ArrayList<SortFieldResult>(); for (Object o : fields) { result.add(convertToSortField(o)); } return result; } public static SortFieldResult convertToSortField(Object o) { if (o == null) { return SortFieldResult.nullValue(true); } else if (o instanceof BytesRef) { return SortFieldResult.binaryValue(getBytes((BytesRef) o)); } else if (o instanceof String) { return SortFieldResult.stringValue((String) o); } else if (o instanceof Double) { return SortFieldResult.doubleValue((Double) o); } else if (o instanceof Integer) { return SortFieldResult.intValue((Integer) o); } else if (o instanceof Long) { return SortFieldResult.longValue((Long) o); } else if (o instanceof byte[]) { return SortFieldResult.binaryValue((byte[]) o); } throw new RuntimeException("Object [" + o + "] is not supported"); } private static byte[] getBytes(BytesRef ref) { byte[] buf = new byte[ref.length]; System.arraycopy(ref.bytes, ref.offset, buf, 0, ref.length); return buf; } public static String toStringBinary(final byte[] b, int off, int len) { StringBuilder result = new StringBuilder(); String str = toString(b, off, len); for (int i = 0; i < str.length(); ++i) { int ch = str.charAt(i) & 0xFF; if ((ch >= '0' && ch <= '9') || (ch >= 'A' && ch <= 'Z') || (ch >= 'a' && ch <= 'z') || " `~!@#$%^&*()-_=+[]{}\\|;:'\",.<>/? ".indexOf(ch) >= 0) { result.append(str.charAt(i)); } else { result.append(String.format("\\x%02X", ch)); } } return result.toString(); } public static String toString(byte[] b, int off, int len) { try { return new String(b, off, len, UTF_8); } catch (UnsupportedEncodingException e) { throw new RuntimeException(e); } } public static Configuration newHadoopConfiguration(BlurConfiguration blurConfiguration) { return addHdfsConfig(new Configuration(), blurConfiguration); } public static Configuration addHdfsConfig(Configuration configuration, BlurConfiguration blurConfiguration) { configuration.addResource("hdfs-default.xml"); configuration.addResource("hdfs-site.xml"); if (blurConfiguration != null) { Map<String, String> properties = blurConfiguration.getProperties(); for (Entry<String, String> e : properties.entrySet()) { String key = e.getKey(); if (key.startsWith(HADOOP_CONF)) { String hadoopKey = key.substring(HADOOP_CONF.length()); String hadoopValue = e.getValue(); LOG.info("Adding hadoop configuration item [{0}] [{1}]", hadoopKey, hadoopValue); configuration.set(hadoopKey, hadoopValue); } } } return configuration; } }