Java tutorial
/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.usergrid.tools; import java.util.*; import java.util.concurrent.atomic.AtomicInteger; import com.datastax.driver.core.ConsistencyLevel; import com.netflix.astyanax.connectionpool.exceptions.ConnectionException; import com.netflix.astyanax.model.Column; import com.netflix.astyanax.util.RangeBuilder; import org.apache.usergrid.persistence.Entity; import org.apache.usergrid.persistence.EntityManager; import org.apache.usergrid.persistence.collection.serialization.MvccEntitySerializationStrategy; import org.apache.usergrid.persistence.collection.serialization.UniqueValueSerializationStrategy; import org.apache.usergrid.persistence.collection.serialization.UniqueValueSet; import org.apache.usergrid.persistence.collection.serialization.impl.*; import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl; import org.apache.usergrid.persistence.model.entity.SimpleId; import org.apache.usergrid.persistence.model.field.StringField; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.Option; import org.apache.commons.cli.OptionBuilder; import org.apache.commons.cli.Options; import org.apache.usergrid.persistence.core.astyanax.MultiTenantColumnFamily; import org.apache.usergrid.persistence.core.astyanax.ScopedRowKey; import org.apache.usergrid.persistence.core.astyanax.ScopedRowKeySerializer; public class UniqueValueScanner extends ToolBase { private static final Logger logger = LoggerFactory.getLogger(UniqueValueScanner.class); private static final String APPLICATION_ARG = "app"; private static final String ENTITY_TYPE_ARG = "entityType"; private static final String ENTITY_NAME_ARG = "entityName"; private static final String ENTITY_FIELD_TYPE_ARG = "fieldType"; //copied shamelessly from unique value serialization strat. private static final ScopedRowKeySerializer<TypeField> ROW_KEY_SER = new ScopedRowKeySerializer<>( UniqueTypeFieldRowKeySerializer.get()); private final EntityVersionSerializer ENTITY_VERSION_SER = new EntityVersionSerializer(); private final MultiTenantColumnFamily<ScopedRowKey<TypeField>, EntityVersion> CF_UNIQUE_VALUES = new MultiTenantColumnFamily<>( "Unique_Values_V2", ROW_KEY_SER, ENTITY_VERSION_SER); private com.netflix.astyanax.Keyspace keyspace; private MvccEntitySerializationStrategy mvccEntitySerializationStrategy; private UniqueValueSerializationStrategy uniqueValueSerializationStrategy; private EntityManager em; @Override @SuppressWarnings("static-access") public Options createOptions() { Options options = super.createOptions(); Option appOption = OptionBuilder.withArgName(APPLICATION_ARG).hasArg().isRequired(true) .withDescription("application id").create(APPLICATION_ARG); options.addOption(appOption); Option collectionOption = OptionBuilder.withArgName(ENTITY_TYPE_ARG).hasArg().isRequired(true) .withDescription("collection name").create(ENTITY_TYPE_ARG); options.addOption(collectionOption); Option specificEntityNameOption = OptionBuilder.withArgName(ENTITY_NAME_ARG).hasArg().isRequired(false) .withDescription("specific entity name").create(ENTITY_NAME_ARG); options.addOption(specificEntityNameOption); Option fieldTypeOption = OptionBuilder.withArgName(ENTITY_FIELD_TYPE_ARG).hasArg().isRequired(false) .withDescription("field type").create(ENTITY_FIELD_TYPE_ARG); options.addOption(fieldTypeOption); return options; } /* * (non-Javadoc) * * @see * org.apache.usergrid.tools.ToolBase#runTool(org.apache.commons.cli.CommandLine) */ @Override public void runTool(CommandLine line) throws Exception { startSpring(); UUID appToFilter = null; if (!line.getOptionValue(APPLICATION_ARG).isEmpty()) { appToFilter = UUID.fromString(line.getOptionValue(APPLICATION_ARG)); } logger.info("Staring Tool: UniqueValueScanner"); logger.info("Using Cassandra consistency level: {}", System.getProperty("usergrid.read.cl", "CL_LOCAL_QUORUM")); keyspace = injector.getInstance(com.netflix.astyanax.Keyspace.class); mvccEntitySerializationStrategy = injector.getInstance(MvccEntitySerializationStrategy.class); uniqueValueSerializationStrategy = injector.getInstance(UniqueValueSerializationStrategy.class); String fieldType = line.getOptionValue(ENTITY_FIELD_TYPE_ARG) != null ? line.getOptionValue(ENTITY_FIELD_TYPE_ARG) : "name"; String entityType = line.getOptionValue(ENTITY_TYPE_ARG); String entityName = line.getOptionValue(ENTITY_NAME_ARG); AtomicInteger count = new AtomicInteger(0); if (entityName != null && !entityName.isEmpty()) { if (appToFilter == null) { throw new RuntimeException("Cannot execute UniqueValueScanner with specific entity without the " + "application UUID for which the entity should exist."); } if (entityType == null) { throw new RuntimeException("Cannot execute UniqueValueScanner without the entity type (singular " + "collection name)."); } logger.info("Running entity unique load only"); //do stuff w/o read repair UniqueValueSet uniqueValueSet = uniqueValueSerializationStrategy.load( new ApplicationScopeImpl(new SimpleId(appToFilter, "application")), ConsistencyLevel.valueOf(System.getProperty("usergrid.read.cl", "LOCAL_QUORUM")), entityType, Collections.singletonList(new StringField(fieldType, entityName)), false); StringBuilder stringBuilder = new StringBuilder(); stringBuilder.append("["); uniqueValueSet.forEach(uniqueValue -> { String entry = "fieldName=" + uniqueValue.getField().getName() + ", fieldValue=" + uniqueValue.getField().getValue() + ", uuid=" + uniqueValue.getEntityId().getUuid() + ", type=" + uniqueValue.getEntityId().getType() + ", version=" + uniqueValue.getEntityVersion(); stringBuilder.append("{").append(entry).append("},"); }); stringBuilder.deleteCharAt(stringBuilder.length() - 1); stringBuilder.append("]"); logger.info("Returned unique value set from serialization load = {}", stringBuilder.toString()); } else { logger.info("Running entity unique scanner only"); // scan through all unique values and log some info Iterator<com.netflix.astyanax.model.Row<ScopedRowKey<TypeField>, EntityVersion>> rows = null; try { rows = keyspace.prepareQuery(CF_UNIQUE_VALUES) .setConsistencyLevel(com.netflix.astyanax.model.ConsistencyLevel .valueOf(System.getProperty("usergrid.read.cl", "CL_LOCAL_QUORUM"))) .getAllRows().withColumnRange(new RangeBuilder().setLimit(1000).build()).execute() .getResult().iterator(); } catch (ConnectionException e) { logger.error("Error connecting to cassandra", e); } UUID finalAppToFilter = appToFilter; if (rows != null) { rows.forEachRemaining(row -> { count.incrementAndGet(); if (count.get() % 1000 == 0) { logger.info("Scanned {} rows in {}", count.get(), CF_UNIQUE_VALUES.getName()); } final String fieldName = row.getKey().getKey().getField().getName(); final String fieldValue = row.getKey().getKey().getField().getValue().toString(); final String scopeType = row.getKey().getScope().getType(); final UUID scopeUUID = row.getKey().getScope().getUuid(); if (!fieldName.equalsIgnoreCase(fieldType) || (finalAppToFilter != null && !finalAppToFilter.equals(scopeUUID))) { // do nothing } else { // if we have more than 1 column, let's check for a duplicate if (row.getColumns() != null && row.getColumns().size() > 1) { final List<EntityVersion> values = new ArrayList<>(row.getColumns().size()); Iterator<Column<EntityVersion>> columns = row.getColumns().iterator(); columns.forEachRemaining(column -> { final EntityVersion entityVersion = column.getName(); logger.trace(scopeType + ": " + scopeUUID + ", " + fieldName + ": " + fieldValue + ", " + "entity type: " + entityVersion.getEntityId().getType() + ", " + "entity uuid: " + entityVersion.getEntityId().getUuid()); if (entityType != null && entityVersion.getEntityId().getType().equalsIgnoreCase(entityType)) { // add the first value into the list if (values.size() == 0) { values.add(entityVersion); } else { if (!values.get(0).getEntityId().getUuid() .equals(entityVersion.getEntityId().getUuid())) { values.add(entityVersion); logger.error( "Duplicate found for field [{}={}]. Entry 1: [{}], Entry 2: [{}]", fieldName, fieldValue, values.get(0).getEntityId(), entityVersion.getEntityId()); } } } }); } } }); } else { logger.warn("No rows returned from table: {}", CF_UNIQUE_VALUES.getName()); } } } }