org.graylog2.indexer.indices.Indices.java Source code

Java tutorial

Introduction

Here is the source code for org.graylog2.indexer.indices.Indices.java

Source

/**
 * This file is part of Graylog.
 *
 * Graylog is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * Graylog is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with Graylog.  If not, see <http://www.gnu.org/licenses/>.
 */
package org.graylog2.indexer.indices;

import com.github.joschi.jadconfig.util.Duration;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.eventbus.EventBus;
import com.google.common.primitives.Ints;
import com.google.common.reflect.TypeToken;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.google.gson.JsonArray;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import io.searchbox.client.JestClient;
import io.searchbox.client.JestResult;
import io.searchbox.cluster.Health;
import io.searchbox.cluster.State;
import io.searchbox.core.Bulk;
import io.searchbox.core.BulkResult;
import io.searchbox.core.Cat;
import io.searchbox.core.CatResult;
import io.searchbox.core.Search;
import io.searchbox.core.SearchResult;
import io.searchbox.core.SearchScroll;
import io.searchbox.core.search.aggregation.FilterAggregation;
import io.searchbox.core.search.aggregation.MaxAggregation;
import io.searchbox.core.search.aggregation.MinAggregation;
import io.searchbox.core.search.aggregation.TermsAggregation;
import io.searchbox.indices.CloseIndex;
import io.searchbox.indices.CreateIndex;
import io.searchbox.indices.DeleteIndex;
import io.searchbox.indices.Flush;
import io.searchbox.indices.ForceMerge;
import io.searchbox.indices.OpenIndex;
import io.searchbox.indices.Stats;
import io.searchbox.indices.aliases.AddAliasMapping;
import io.searchbox.indices.aliases.AliasMapping;
import io.searchbox.indices.aliases.GetAliases;
import io.searchbox.indices.aliases.ModifyAliases;
import io.searchbox.indices.aliases.RemoveAliasMapping;
import io.searchbox.indices.settings.GetSettings;
import io.searchbox.indices.settings.UpdateSettings;
import io.searchbox.indices.template.DeleteTemplate;
import io.searchbox.indices.template.PutTemplate;
import io.searchbox.params.Parameters;
import io.searchbox.params.SearchType;
import org.apache.http.client.config.RequestConfig;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.aggregations.AggregationBuilders;
import org.elasticsearch.search.aggregations.bucket.filter.FilterAggregationBuilder;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.search.sort.SortBuilders;
import org.elasticsearch.search.sort.SortParseElement;
import org.graylog2.audit.AuditActor;
import org.graylog2.audit.AuditEventSender;
import org.graylog2.indexer.ElasticsearchException;
import org.graylog2.indexer.IndexMapping;
import org.graylog2.indexer.IndexMappingFactory;
import org.graylog2.indexer.IndexNotFoundException;
import org.graylog2.indexer.IndexSet;
import org.graylog2.indexer.cluster.jest.JestUtils;
import org.graylog2.indexer.gson.GsonUtils;
import org.graylog2.indexer.indexset.IndexSetConfig;
import org.graylog2.indexer.indices.events.IndicesClosedEvent;
import org.graylog2.indexer.indices.events.IndicesDeletedEvent;
import org.graylog2.indexer.indices.events.IndicesReopenedEvent;
import org.graylog2.indexer.indices.stats.IndexStatistics;
import org.graylog2.indexer.messages.Messages;
import org.graylog2.indexer.searches.IndexRangeStats;
import org.graylog2.plugin.Message;
import org.graylog2.plugin.system.NodeId;
import org.joda.time.DateTime;
import org.joda.time.DateTimeZone;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import javax.annotation.Nullable;
import javax.inject.Inject;
import javax.inject.Singleton;
import javax.validation.constraints.NotNull;
import java.io.IOException;
import java.lang.reflect.Type;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.function.Function;
import java.util.stream.Collectors;
import java.util.stream.Stream;

import static java.util.stream.Collectors.toList;
import static java.util.stream.Collectors.toSet;
import static org.elasticsearch.search.builder.SearchSourceBuilder.searchSource;
import static org.graylog2.audit.AuditEventTypes.ES_INDEX_CREATE;
import static org.graylog2.indexer.gson.GsonUtils.asInteger;
import static org.graylog2.indexer.gson.GsonUtils.asJsonArray;
import static org.graylog2.indexer.gson.GsonUtils.asJsonObject;
import static org.graylog2.indexer.gson.GsonUtils.asLong;
import static org.graylog2.indexer.gson.GsonUtils.asString;

@Singleton
public class Indices {
    private static final Logger LOG = LoggerFactory.getLogger(Indices.class);
    private static final String REOPENED_INDEX_SETTING = "graylog2_reopened";

    private final JestClient jestClient;
    private final Gson gson;
    private final IndexMappingFactory indexMappingFactory;
    private final Messages messages;
    private final NodeId nodeId;
    private final AuditEventSender auditEventSender;
    private final EventBus eventBus;

    @Inject
    public Indices(JestClient jestClient, Gson gson, IndexMappingFactory indexMappingFactory, Messages messages,
            NodeId nodeId, AuditEventSender auditEventSender, EventBus eventBus) {
        this.jestClient = jestClient;
        this.gson = gson;
        this.indexMappingFactory = indexMappingFactory;
        this.messages = messages;
        this.nodeId = nodeId;
        this.auditEventSender = auditEventSender;
        this.eventBus = eventBus;
    }

    public void move(String source, String target) {
        // TODO: This method should use the Re-index API: https://www.elastic.co/guide/en/elasticsearch/reference/5.3/docs-reindex.html
        final String query = SearchSourceBuilder.searchSource().query(QueryBuilders.matchAllQuery()).size(350)
                .sort(SortBuilders.fieldSort(SortParseElement.DOC_FIELD_NAME)).toString();

        final Search request = new Search.Builder(query).setParameter(Parameters.SCROLL, "10s").addIndex(source)
                .build();

        final SearchResult searchResult = JestUtils.execute(jestClient, request,
                () -> "Couldn't process search query response");

        final String scrollId = Optional.of(searchResult.getJsonObject())
                .map(json -> asString(json.get("_scroll_id")))
                .orElseThrow(() -> new ElasticsearchException("Couldn't find scroll ID in search query response"));

        final Type type = new TypeToken<Map<String, Object>>() {
        }.getType();
        while (true) {
            final SearchScroll scrollRequest = new SearchScroll.Builder(scrollId, "1m").build();
            final JestResult scrollResult = JestUtils.execute(jestClient, scrollRequest,
                    () -> "Couldn't process result of scroll query");
            final JsonArray scrollHits = Optional.of(scrollResult.getJsonObject())
                    .map(json -> asJsonObject(json.get("hits"))).map(hits -> asJsonArray(hits.get("hits")))
                    .orElse(new JsonArray());

            // No more hits.
            if (scrollHits.size() == 0) {
                break;
            }

            final Bulk.Builder bulkRequestBuilder = new Bulk.Builder();
            for (JsonElement jsonElement : scrollHits) {
                final Map<String, Object> doc = Optional.ofNullable(asJsonObject(jsonElement))
                        .map(hitsJson -> asJsonObject(hitsJson.get("_source")))
                        .map(sourceJson -> gson.<Map<String, Object>>fromJson(sourceJson, type))
                        .orElse(Collections.emptyMap());
                final String id = (String) doc.remove("_id");

                bulkRequestBuilder.addAction(messages.prepareIndexRequest(target, doc, id));
            }

            bulkRequestBuilder.setParameter(Parameters.CONSISTENCY, "one");

            final BulkResult bulkResult = JestUtils.execute(jestClient, bulkRequestBuilder.build(),
                    () -> "Couldn't bulk index messages into index " + target);

            final boolean hasFailedItems = !bulkResult.getFailedItems().isEmpty();
            LOG.info("Moving index <{}> to <{}>: Bulk indexed {} messages, took {} ms, failures: {}", source,
                    target, bulkResult.getItems().size(), asLong(bulkResult.getJsonObject().get("took")),
                    hasFailedItems);

            if (hasFailedItems) {
                throw new ElasticsearchException("Failed to move a message. Check your indexer log.");
            }
        }
    }

    public void delete(String indexName) {
        JestUtils.execute(jestClient, new DeleteIndex.Builder(indexName).build(),
                () -> "Couldn't delete index " + indexName);
        eventBus.post(IndicesDeletedEvent.create(indexName));
    }

    public void close(String indexName) {
        JestUtils.execute(jestClient, new CloseIndex.Builder(indexName).build(),
                () -> "Couldn't close index " + indexName);
        eventBus.post(IndicesClosedEvent.create(indexName));
    }

    public long numberOfMessages(String indexName) throws IndexNotFoundException {
        return indexStats(indexName).map(index -> asJsonObject(index.get("primaries")))
                .map(primaries -> asJsonObject(primaries.get("docs"))).map(docs -> asLong(docs.get("count")))
                .orElse(0L);
    }

    private Map<String, JsonElement> getAllWithShardLevel(final Collection<String> indices) {
        final Stats request = new Stats.Builder().addIndex(indices).setParameter("level", "shards").build();
        final JestResult jestResult = JestUtils.execute(jestClient, request,
                () -> "Couldn't fetch index stats of indices " + indices);
        final JsonObject responseJson = jestResult.getJsonObject();
        final int failedShards = Optional.ofNullable(responseJson).map(json -> asJsonObject(json.get("_shards")))
                .map(json -> asInteger(json.get("failed"))).orElse(0);

        if (failedShards > 0) {
            throw new ElasticsearchException("Index stats response contains failed shards, response is incomplete");
        }

        return Optional.ofNullable(responseJson).map(json -> asJsonObject(json.get("indices")))
                .map(GsonUtils::entrySetAsMap).orElse(Collections.emptyMap());
    }

    public Map<String, JsonElement> getIndexStats(final IndexSet indexSet) {
        return getIndexStats(Collections.singleton(indexSet.getIndexWildcard()));
    }

    public Map<String, JsonElement> getIndexStats(final Collection<String> indices) {
        final Stats request = new Stats.Builder().addIndex(indices).docs(true).store(true).build();

        final JestResult jestResult = JestUtils.execute(jestClient, request,
                () -> "Couldn't check stats of indices " + indices);

        return Optional.ofNullable(asJsonObject(jestResult.getJsonObject()))
                .map(json -> asJsonObject(json.get("indices"))).map(GsonUtils::entrySetAsMap)
                .orElse(Collections.emptyMap());
    }

    private Optional<JsonObject> indexStats(final String indexName) {
        final Stats request = new Stats.Builder().addIndex(indexName).build();

        final JestResult jestResult = JestUtils.execute(jestClient, request,
                () -> "Couldn't check stats of index " + indexName);

        return Optional.ofNullable(asJsonObject(jestResult.getJsonObject()))
                .map(json -> asJsonObject(json.get("indices")))
                .map(indices -> asJsonObject(indices.get(indexName)));
    }

    private Optional<JsonObject> indexStatsWithShardLevel(final String indexName) {
        final Stats request = new Stats.Builder().addIndex(indexName).setParameter("level", "shards")
                .ignoreUnavailable(true).build();

        final JestResult jestResult = JestUtils.execute(jestClient, request,
                () -> "Couldn't check stats of index " + indexName);

        return Optional.ofNullable(asJsonObject(jestResult.getJsonObject()))
                .map(json -> asJsonObject(json.get("indices")))
                .map(indices -> asJsonObject(indices.get(indexName)));
    }

    /**
     * Check if a given name is an existing index.
     * @param indexName Name of the index to check presence for.
     * @return {@code true} if indexName is an existing index, {@code false} if it is non-existing or an alias.
     */
    public boolean exists(String indexName) {
        try {
            final JestResult result = jestClient.execute(new GetSettings.Builder().addIndex(indexName).build());
            if (!result.isSucceeded()) {
                return false;
            }
            return Optional.of(result.getJsonObject()).map(GsonUtils::entrySetAsMap)
                    .map(map -> map.containsKey(indexName)).orElse(false);
        } catch (IOException e) {
            throw new ElasticsearchException("Couldn't check existence of index " + indexName, e);
        }
    }

    /**
     * Check if a given name is an existing alias.
     * @param alias Name of the alias to check presence for.
     * @return {@code true} if alias is an existing alias, {@code false} if it is non-existing or an index.
     */
    public boolean aliasExists(String alias) {
        try {
            final JestResult result = jestClient.execute(new GetSettings.Builder().addIndex(alias).build());
            if (!result.isSucceeded()) {
                return false;
            }
            return Optional.of(result.getJsonObject()).map(GsonUtils::entrySetAsMap)
                    .map(map -> !map.containsKey(alias)).orElse(false);
        } catch (IOException e) {
            throw new ElasticsearchException("Couldn't check existence of alias " + alias, e);
        }
    }

    @NotNull
    public Map<String, Set<String>> getIndexNamesAndAliases(String indexPattern) {
        // only request indices matching the name or pattern in `indexPattern` and only get the alias names for each index,
        // not the settings or mappings
        final GetAliases request = new GetAliases.Builder().addIndex(indexPattern).build();

        final JestResult jestResult = JestUtils.execute(jestClient, request,
                () -> "Couldn't collect aliases for index pattern " + indexPattern);

        final ImmutableMap.Builder<String, Set<String>> indexAliasesBuilder = ImmutableMap.builder();
        for (Map.Entry<String, JsonElement> entry : jestResult.getJsonObject().entrySet()) {
            final JsonObject aliasMetaData = asJsonObject(entry.getValue());
            if (aliasMetaData != null) {
                final ImmutableSet.Builder<String> aliasesBuilder = ImmutableSet.builder();
                for (Map.Entry<String, JsonElement> aliasesEntry : aliasMetaData.entrySet()) {
                    aliasesBuilder.add(aliasesEntry.getKey());
                }
                indexAliasesBuilder.put(entry.getKey(), aliasesBuilder.build());
            }
        }

        return indexAliasesBuilder.build();
    }

    public Optional<String> aliasTarget(String alias) throws TooManyAliasesException {
        final GetAliases request = new GetAliases.Builder().build();
        final JestResult jestResult = JestUtils.execute(jestClient, request,
                () -> "Couldn't collect indices for alias " + alias);

        // The ES return value of this has an awkward format: The first key of the hash is the target index. Thanks.
        final ImmutableSet.Builder<String> indicesBuilder = ImmutableSet.builder();
        for (Map.Entry<String, JsonElement> entry : jestResult.getJsonObject().entrySet()) {
            final String indexName = entry.getKey();
            Optional.of(entry.getValue()).map(GsonUtils::asJsonObject)
                    .map(json -> asJsonObject(json.get("aliases"))).map(JsonObject::entrySet)
                    .filter(aliases -> !aliases.isEmpty())
                    .filter(aliases -> aliases.stream().anyMatch(aliasEntry -> aliasEntry.getKey().equals(alias)))
                    .ifPresent(x -> indicesBuilder.add(indexName));
        }

        final Set<String> indices = indicesBuilder.build();
        if (indices.size() > 1) {
            throw new TooManyAliasesException(indices);
        }

        return indices.stream().findFirst();
    }

    private void ensureIndexTemplate(IndexSet indexSet) {
        final IndexSetConfig indexSetConfig = indexSet.getConfig();
        final String templateName = indexSetConfig.indexTemplateName();
        final IndexMapping indexMapping = indexMappingFactory.createIndexMapping();
        final Map<String, Object> template = indexMapping.messageTemplate(indexSet.getIndexWildcard(),
                indexSetConfig.indexAnalyzer(), -1);

        final PutTemplate request = new PutTemplate.Builder(templateName, template).build();

        final JestResult jestResult = JestUtils.execute(jestClient, request,
                () -> "Unable to create index template " + templateName);

        if (jestResult.isSucceeded()) {
            LOG.info("Successfully created index template {}", templateName);
        }
    }

    public void deleteIndexTemplate(IndexSet indexSet) {
        final String templateName = indexSet.getConfig().indexTemplateName();
        final JestResult result = JestUtils.execute(jestClient, new DeleteTemplate.Builder(templateName).build(),
                () -> "Unable to delete the Graylog index template " + templateName);
        if (result.isSucceeded()) {
            LOG.info("Successfully deleted index template {}", templateName);
        }
    }

    public boolean create(String indexName, IndexSet indexSet) {
        return create(indexName, indexSet, Collections.emptyMap());
    }

    public boolean create(String indexName, IndexSet indexSet, Map<String, Object> customSettings) {
        final Map<String, Object> settings = new HashMap<>();
        settings.put("number_of_shards", indexSet.getConfig().shards());
        settings.put("number_of_replicas", indexSet.getConfig().replicas());
        settings.putAll(customSettings);

        final CreateIndex request = new CreateIndex.Builder(indexName).settings(settings).build();

        // Make sure our index template exists before creating an index!
        ensureIndexTemplate(indexSet);

        final JestResult jestResult;
        try {
            jestResult = jestClient.execute(request);
        } catch (IOException e) {
            throw new ElasticsearchException("Couldn't create index " + indexName, e);
        }

        final boolean succeeded = jestResult.isSucceeded();
        if (succeeded) {
            auditEventSender.success(AuditActor.system(nodeId), ES_INDEX_CREATE,
                    ImmutableMap.of("indexName", indexName));
        } else {
            LOG.warn("Couldn't create index {}. Error: {}", indexName, jestResult.getErrorMessage());
            auditEventSender.failure(AuditActor.system(nodeId), ES_INDEX_CREATE,
                    ImmutableMap.of("indexName", indexName));
        }
        return succeeded;
    }

    public Map<String, Set<String>> getAllMessageFieldsForIndices(final String[] writeIndexWildcards) {
        final String indices = String.join(",", writeIndexWildcards);
        final State request = new State.Builder().indices(indices).withMetadata().build();

        final JestResult jestResult = JestUtils.execute(jestClient, request,
                () -> "Couldn't read cluster state for indices " + indices);

        final JsonObject indicesJson = getClusterStateIndicesMetadata(jestResult.getJsonObject());
        final ImmutableMap.Builder<String, Set<String>> fields = ImmutableMap.builder();
        for (Map.Entry<String, JsonElement> entry : indicesJson.entrySet()) {
            final String indexName = entry.getKey();
            final Set<String> fieldNames = Optional.ofNullable(asJsonObject(entry.getValue()))
                    .map(index -> asJsonObject(index.get("mappings")))
                    .map(mappings -> asJsonObject(mappings.get(IndexMapping.TYPE_MESSAGE)))
                    .map(messageType -> asJsonObject(messageType.get("properties"))).map(JsonObject::entrySet)
                    .map(Set::stream).orElseGet(Stream::empty).map(Map.Entry::getKey).collect(toSet());

            if (!fieldNames.isEmpty()) {
                fields.put(indexName, fieldNames);
            }
        }

        return fields.build();
    }

    public Set<String> getAllMessageFields(final String[] writeIndexWildcards) {
        final Map<String, Set<String>> fieldsForIndices = getAllMessageFieldsForIndices(writeIndexWildcards);
        final ImmutableSet.Builder<String> result = ImmutableSet.builder();
        for (Set<String> fields : fieldsForIndices.values()) {
            result.addAll(fields);
        }
        return result.build();
    }

    public void setReadOnly(String index) {
        // https://www.elastic.co/guide/en/elasticsearch/reference/2.4/indices-update-settings.html
        final Map<String, Object> settings = ImmutableMap.of("index",
                ImmutableMap.of("blocks", ImmutableMap.of("write", true, // Block writing.
                        "read", false, // Allow reading.
                        "metadata", false) // Allow getting metadata.
                ));

        final UpdateSettings request = new UpdateSettings.Builder(settings).addIndex(index).build();
        JestUtils.execute(jestClient, request, () -> "Couldn't set index " + index + " to read-only");
    }

    public void flush(String index) {
        JestUtils.execute(jestClient, new Flush.Builder().addIndex(index).force().build(),
                () -> "Couldn't flush index " + index);
    }

    public Map<String, Object> reopenIndexSettings() {
        return ImmutableMap.of("index", ImmutableMap.of(REOPENED_INDEX_SETTING, true));
    }

    public void reopenIndex(String index) {
        // Mark this index as re-opened. It will never be touched by retention.
        final Map<String, Object> settings = reopenIndexSettings();
        final UpdateSettings request = new UpdateSettings.Builder(settings).addIndex(index).build();

        JestUtils.execute(jestClient, request, () -> "Couldn't update settings of index " + index);

        // Open index.
        openIndex(index);
    }

    private void openIndex(String index) {
        JestUtils.execute(jestClient, new OpenIndex.Builder(index).build(), () -> "Couldn't open index " + index);
        eventBus.post(IndicesReopenedEvent.create(index));
    }

    public boolean isReopened(String indexName) {
        final JestResult jestResult = JestUtils.execute(jestClient, new State.Builder().withMetadata().build(),
                () -> "Couldn't read cluster state for index " + indexName);

        final JsonObject indexJson = Optional.ofNullable(asJsonObject(jestResult.getJsonObject()))
                .map(response -> asJsonObject(response.get("metadata")))
                .map(metadata -> asJsonObject(metadata.get("indices")))
                .map(indices -> getIndexSettings(indices, indexName)).orElse(new JsonObject());

        return checkForReopened(indexJson);
    }

    public Map<String, Boolean> areReopened(Collection<String> indices) {
        final JestResult jestResult = JestUtils.execute(jestClient, new State.Builder().withMetadata().build(),
                () -> "Couldn't read cluster state for indices " + indices);

        final JsonObject indicesJson = getClusterStateIndicesMetadata(jestResult.getJsonObject());
        return indices.stream().collect(Collectors.toMap(Function.identity(),
                index -> checkForReopened(getIndexSettings(indicesJson, index))));
    }

    private JsonObject getIndexSettings(JsonObject indicesJson, String index) {
        return Optional.ofNullable(asJsonObject(indicesJson)).map(indices -> asJsonObject(indices.get(index)))
                .map(idx -> asJsonObject(idx.get("settings"))).map(settings -> asJsonObject(settings.get("index")))
                .orElse(new JsonObject());
    }

    private boolean checkForReopened(@Nullable JsonObject indexSettings) {
        return Optional.ofNullable(indexSettings).map(settings -> asString(settings.get(REOPENED_INDEX_SETTING))) // WTF, why is this a string?
                .map(Boolean::parseBoolean).orElse(false);
    }

    public Set<String> getClosedIndices(final Collection<String> indices) {
        final JsonArray catIndices = catIndices(indices, "index", "status");

        final ImmutableSet.Builder<String> closedIndices = ImmutableSet.builder();
        for (JsonElement jsonElement : catIndices) {
            if (jsonElement.isJsonObject()) {
                final JsonObject jsonObject = jsonElement.getAsJsonObject();
                final String index = GsonUtils.asString(jsonObject.get("index"));
                final String status = GsonUtils.asString(jsonObject.get("status"));
                if (index != null && "close".equals(status)) {
                    closedIndices.add(index);
                }
            }
        }

        return closedIndices.build();
    }

    public Set<String> getClosedIndices(final IndexSet indexSet) {
        return getClosedIndices(Collections.singleton(indexSet.getIndexWildcard()));
    }

    /**
     * Retrieve the response for the <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/cat-indices.html">cat indices</a> request from Elasticsearch.
     *
     * @param fields The fields to show, see <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/cat-indices.html">cat indices API</a>.
     * @return A {@link JsonArray} with the result of the cat indices request.
     */
    private JsonArray catIndices(Collection<String> indices, String... fields) {
        final String fieldNames = String.join(",", fields);
        final Cat request = new Cat.IndicesBuilder().addIndex(indices).setParameter("h", fieldNames).build();
        final CatResult response = JestUtils.execute(jestClient, request,
                () -> "Unable to read information for indices " + indices);
        return Optional.of(response.getJsonObject()).map(json -> GsonUtils.asJsonArray(json.get("result")))
                .orElse(new JsonArray());
    }

    private JsonObject getClusterStateIndicesMetadata(JsonObject clusterStateJson) {
        return Optional.ofNullable(clusterStateJson).map(json -> asJsonObject(json.get("metadata")))
                .map(metadata -> asJsonObject(metadata.get("indices"))).orElse(new JsonObject());
    }

    public Set<String> getReopenedIndices(final Collection<String> indices) {
        final String indexList = String.join(",", indices);
        final State request = new State.Builder().withMetadata().indices(indexList).build();

        final JestResult jestResult = JestUtils.execute(jestClient, request,
                () -> "Couldn't read cluster state for reopened indices " + indices);
        final JsonObject indicesJson = getClusterStateIndicesMetadata(jestResult.getJsonObject());
        final ImmutableSet.Builder<String> reopenedIndices = ImmutableSet.builder();

        for (Map.Entry<String, JsonElement> entry : indicesJson.entrySet()) {
            final String indexName = entry.getKey();
            final JsonElement value = entry.getValue();
            if (value.isJsonObject()) {
                final JsonObject indexSettingsJson = value.getAsJsonObject();
                final JsonObject indexSettings = getIndexSettings(indexSettingsJson, indexName);
                if (checkForReopened(indexSettings)) {
                    reopenedIndices.add(indexName);
                }
            }
        }

        return reopenedIndices.build();
    }

    public Set<String> getReopenedIndices(final IndexSet indexSet) {
        return getReopenedIndices(Collections.singleton(indexSet.getIndexWildcard()));
    }

    public Optional<IndexStatistics> getIndexStats(String index) {
        return indexStatsWithShardLevel(index).map(indexStats -> buildIndexStatistics(index, indexStats));
    }

    private IndexStatistics buildIndexStatistics(String index, JsonObject indexStats) {
        return IndexStatistics.create(index, indexStats);
    }

    public Optional<Long> getStoreSizeInBytes(String index) {
        final Stats request = new Stats.Builder().addIndex(index).store(true).build();

        final JestResult jestResult = JestUtils.execute(jestClient, request,
                () -> "Couldn't check store stats of index " + index);

        return Optional.ofNullable(jestResult.getJsonObject()).map(json -> asJsonObject(json.get("indices")))
                .map(json -> asJsonObject(json.get(index))).map(json -> asJsonObject(json.get("primaries")))
                .map(json -> asJsonObject(json.get("store"))).map(json -> asLong(json.get("size_in_bytes")));
    }

    public Set<IndexStatistics> getIndicesStats(final IndexSet indexSet) {
        return getIndicesStats(Collections.singleton(indexSet.getIndexWildcard()));
    }

    public Set<IndexStatistics> getIndicesStats(final Collection<String> indices) {
        final ImmutableSet.Builder<IndexStatistics> result = ImmutableSet.builder();
        for (Map.Entry<String, JsonElement> entry : getAllWithShardLevel(indices).entrySet()) {
            final String index = entry.getKey();
            Optional.ofNullable(asJsonObject(entry.getValue()))
                    .map(indexStats -> buildIndexStatistics(index, indexStats)).ifPresent(result::add);
        }

        return result.build();
    }

    public void cycleAlias(String aliasName, String targetIndex) {
        final AddAliasMapping addAliasMapping = new AddAliasMapping.Builder(targetIndex, aliasName).build();
        JestUtils.execute(jestClient, new ModifyAliases.Builder(addAliasMapping).build(),
                () -> "Couldn't point alias " + aliasName + " to index " + targetIndex);
    }

    public void cycleAlias(String aliasName, String targetIndex, String oldIndex) {
        final AliasMapping addAliasMapping = new AddAliasMapping.Builder(targetIndex, aliasName).build();
        final AliasMapping removeAliasMapping = new RemoveAliasMapping.Builder(oldIndex, aliasName).build();
        final ModifyAliases request = new ModifyAliases.Builder(Arrays.asList(removeAliasMapping, addAliasMapping))
                .build();

        JestUtils.execute(jestClient, request, () -> "Couldn't switch alias " + aliasName + " from index "
                + oldIndex + " to index " + targetIndex);
    }

    public void removeAliases(String alias, Set<String> indices) {
        final AliasMapping removeAliasMapping = new RemoveAliasMapping.Builder(ImmutableList.copyOf(indices), alias)
                .build();
        final ModifyAliases request = new ModifyAliases.Builder(removeAliasMapping).build();
        JestUtils.execute(jestClient, request, () -> "Couldn't remove alias " + alias + " from indices " + indices);
    }

    public void optimizeIndex(String index, int maxNumSegments, Duration timeout) {
        final RequestConfig requestConfig = RequestConfig.custom()
                .setSocketTimeout(Ints.saturatedCast(timeout.toMilliseconds())).build();

        final ForceMerge request = new ForceMerge.Builder().addIndex(index).maxNumSegments(maxNumSegments)
                .flush(true).onlyExpungeDeletes(false).build();

        JestUtils.execute(jestClient, requestConfig, request, () -> "Couldn't force merge index " + index);
    }

    public Health.Status waitForRecovery(String index) {
        return waitForStatus(index, Health.Status.YELLOW);
    }

    private Health.Status waitForStatus(String index, Health.Status clusterHealthStatus) {
        LOG.debug("Waiting until index health status of index {} is {}", index, clusterHealthStatus);
        final Health request = new Health.Builder().addIndex(index).waitForStatus(clusterHealthStatus).build();
        final JestResult jestResult = JestUtils.execute(jestClient, request,
                () -> "Couldn't read health status for index " + index);

        final String status = jestResult.getJsonObject().get("status").getAsString();
        return Health.Status.valueOf(status.toUpperCase(Locale.ENGLISH));
    }

    public Optional<DateTime> indexCreationDate(String index) {
        final GetSettings request = new GetSettings.Builder().addIndex(index).ignoreUnavailable(true).build();
        final JestResult jestResult = JestUtils.execute(jestClient, request,
                () -> "Couldn't read settings of index " + index);

        return Optional.of(jestResult.getJsonObject()).map(json -> asJsonObject(json.get(index)))
                .map(json -> asJsonObject(json.get("settings"))).map(json -> asJsonObject(json.get("index")))
                .map(json -> asString(json.get("creation_date"))) // WTF, why is this a string?
                .map(Long::parseLong).map(creationDate -> new DateTime(creationDate, DateTimeZone.UTC));
    }

    /**
     * Calculate min and max message timestamps in the given index.
     *
     * @param index Name of the index to query.
     * @return the timestamp stats in the given index, or {@code null} if they couldn't be calculated.
     * @see org.elasticsearch.search.aggregations.metrics.stats.Stats
     */
    public IndexRangeStats indexRangeStatsOfIndex(String index) {
        final FilterAggregationBuilder builder = AggregationBuilders.filter("agg")
                .filter(QueryBuilders.existsQuery(Message.FIELD_TIMESTAMP))
                .subAggregation(AggregationBuilders.min("ts_min").field(Message.FIELD_TIMESTAMP))
                .subAggregation(AggregationBuilders.max("ts_max").field(Message.FIELD_TIMESTAMP))
                .subAggregation(AggregationBuilders.terms("streams").field(Message.FIELD_STREAMS));
        final String query = searchSource().aggregation(builder).size(0).toString();

        final Search request = new Search.Builder(query).addIndex(index)
                .setSearchType(SearchType.DFS_QUERY_THEN_FETCH).ignoreUnavailable(true).build();

        if (LOG.isDebugEnabled()) {
            final Gson gson = new GsonBuilder().setPrettyPrinting().create();
            LOG.debug("Index range query: _search/{}: {}", index, request.getData(gson));
        }

        final SearchResult result = JestUtils.execute(jestClient, request,
                () -> "Couldn't build index range of index " + index);

        final FilterAggregation f = result.getAggregations().getFilterAggregation("agg");
        if (f == null) {
            throw new IndexNotFoundException(
                    "Couldn't build index range of index " + index + " because it doesn't exist.");
        } else if (f.getCount() == 0L) {
            LOG.debug("No documents with attribute \"timestamp\" found in index <{}>", index);
            return IndexRangeStats.EMPTY;
        }

        final MinAggregation minAgg = f.getMinAggregation("ts_min");
        final DateTime min = new DateTime(minAgg.getMin().longValue(), DateTimeZone.UTC);
        final MaxAggregation maxAgg = f.getMaxAggregation("ts_max");
        final DateTime max = new DateTime(maxAgg.getMax().longValue(), DateTimeZone.UTC);
        // make sure we return an empty list, so we can differentiate between old indices that don't have this information
        // and newer ones that simply have no streams.
        final TermsAggregation streams = f.getTermsAggregation("streams");
        final List<String> streamIds = streams.getBuckets().stream().map(TermsAggregation.Entry::getKeyAsString)
                .collect(toList());

        return IndexRangeStats.create(min, max, streamIds);
    }
}