Java tutorial
/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.usergrid.tools; import java.text.DateFormat; import java.text.SimpleDateFormat; import java.util.*; import com.google.common.base.Optional; import com.netflix.astyanax.MutationBatch; import org.apache.usergrid.corepersistence.pipeline.read.ResultsPage; import org.apache.usergrid.corepersistence.results.EntityQueryExecutor; import org.apache.usergrid.corepersistence.results.IdQueryExecutor; import org.apache.usergrid.corepersistence.service.CollectionSearch; import org.apache.usergrid.corepersistence.service.CollectionService; import org.apache.usergrid.corepersistence.util.CpNamingUtils; import org.apache.usergrid.persistence.*; import org.apache.usergrid.persistence.core.scope.ApplicationScope; import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl; import org.apache.usergrid.persistence.graph.*; import org.apache.usergrid.persistence.graph.impl.SimpleEdge; import org.apache.usergrid.persistence.graph.impl.SimpleMarkedEdge; import org.apache.usergrid.persistence.graph.impl.SimpleSearchByEdgeType; import org.apache.usergrid.persistence.graph.serialization.EdgeSerialization; import org.apache.usergrid.persistence.index.utils.UUIDUtils; import org.apache.usergrid.persistence.model.entity.*; import org.apache.usergrid.persistence.model.entity.Entity; import org.apache.usergrid.persistence.schema.CollectionInfo; import org.apache.usergrid.utils.InflectionUtils; import static org.apache.commons.lang.StringUtils.isBlank; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.Option; import org.apache.commons.cli.OptionBuilder; import org.apache.commons.cli.Options; import static org.apache.usergrid.persistence.Schema.getDefaultSchema; public class CollectionIterator extends ToolBase { private static final Logger logger = LoggerFactory.getLogger(CollectionIterator.class); private static final String APPLICATION_ARG = "app"; private static final String ENTITY_TYPE_ARG = "entityType"; private static final String REMOVE_CONNECTIONS_ARG = "removeConnections"; private static final String LATEST_TIMESTAMP_ARG = "latestTimestamp"; private static final String EARLIEST_TIMESTAMP_ARG = "earliestTimestamp"; private static final String SECONDS_IN_PAST_ARG = "secondsInPast"; private static final Long DEFAULT_SECONDS_IN_PAST = 60L * 60L; // hour private EntityManager em; @Override @SuppressWarnings("static-access") public Options createOptions() { Options options = super.createOptions(); Option appOption = OptionBuilder.withArgName(APPLICATION_ARG).hasArg().isRequired(true) .withDescription("application id").create(APPLICATION_ARG); options.addOption(appOption); Option collectionOption = OptionBuilder.withArgName(ENTITY_TYPE_ARG).hasArg().isRequired(true) .withDescription("singular collection name").create(ENTITY_TYPE_ARG); options.addOption(collectionOption); Option removeConnectionsOption = OptionBuilder.withArgName(REMOVE_CONNECTIONS_ARG).hasArg() .isRequired(false).withDescription("remove orphaned connections").create(REMOVE_CONNECTIONS_ARG); options.addOption(removeConnectionsOption); Option earliestTimestampOption = OptionBuilder.withArgName(EARLIEST_TIMESTAMP_ARG).hasArg() .isRequired(false).withDescription("earliest timestamp to delete").create(EARLIEST_TIMESTAMP_ARG); options.addOption(earliestTimestampOption); Option latestTimestampOption = OptionBuilder.withArgName(LATEST_TIMESTAMP_ARG).hasArg().isRequired(false) .withDescription("latest timestamp to delete").create(LATEST_TIMESTAMP_ARG); options.addOption(latestTimestampOption); Option secondsInPastOption = OptionBuilder.withArgName(SECONDS_IN_PAST_ARG).hasArg().isRequired(false) .withDescription("how many seconds old orphan must be to be deleted").create(SECONDS_IN_PAST_ARG); options.addOption(secondsInPastOption); return options; } /* * (non-Javadoc) * * @see * org.apache.usergrid.tools.ToolBase#runTool(org.apache.commons.cli.CommandLine) */ @Override public void runTool(CommandLine line) throws Exception { startSpring(); String applicationOption = line.getOptionValue(APPLICATION_ARG); String entityTypeOption = line.getOptionValue(ENTITY_TYPE_ARG); String removeConnectionsOption = line.getOptionValue(REMOVE_CONNECTIONS_ARG); String earliestTimestampOption = line.getOptionValue(EARLIEST_TIMESTAMP_ARG); String latestTimestampOption = line.getOptionValue(LATEST_TIMESTAMP_ARG); String secondsInPastOption = line.getOptionValue(SECONDS_IN_PAST_ARG); if (isBlank(applicationOption)) { throw new RuntimeException("Application ID not provided."); } final UUID app = UUID.fromString(line.getOptionValue(APPLICATION_ARG)); if (isBlank(entityTypeOption)) { throw new RuntimeException("Entity type (singular collection name) not provided."); } String entityType = entityTypeOption; final boolean removeOrphans = !isBlank(removeConnectionsOption) && removeConnectionsOption.toLowerCase().equals("yes"); if (!isBlank(secondsInPastOption) && !isBlank(latestTimestampOption)) { throw new RuntimeException("Can't specify both latest timestamp and seconds in past options."); } long earliest = 0L; if (!isBlank(earliestTimestampOption)) { try { earliest = Long.parseLong(earliestTimestampOption); } catch (Exception e) { throw new RuntimeException("Cannot convert earliest timestamp to long: " + earliestTimestampOption); } } final long earliestTimestamp = earliest; long currentTimestamp = System.currentTimeMillis(); // default to DEFAULT_SECONDS_IN_PAST long latest = currentTimestamp - (DEFAULT_SECONDS_IN_PAST * 1000L); if (!isBlank(latestTimestampOption)) { try { latest = Long.parseLong(latestTimestampOption); } catch (Exception e) { throw new RuntimeException("Cannot convert latest timestamp to long: " + latestTimestampOption); } } else if (!isBlank(secondsInPastOption)) { try { long secondsInPast = Long.parseLong(secondsInPastOption); latest = currentTimestamp - (secondsInPast * 1000L); } catch (Exception e) { throw new RuntimeException("Cannot convert seconds in past to long: " + secondsInPastOption); } } final long latestTimestamp = latest; logger.info("Starting Tool: CollectionIterator"); logger.info("Orphans {} be deleted", removeOrphans ? "WILL" : "will not"); logger.info("Timestamp range {} to {}", Long.toString(earliestTimestamp), Long.toString(latestTimestamp)); logger.info("Using Cassandra consistency level: {}", System.getProperty("usergrid.read.cl", "CL_LOCAL_QUORUM")); em = emf.getEntityManager(app); EntityRef headEntity = new SimpleEntityRef("application", app); CollectionService collectionService = injector.getInstance(CollectionService.class); String collectionName = InflectionUtils.pluralize(entityType); String simpleEdgeType = CpNamingUtils.getEdgeTypeFromCollectionName(collectionName); logger.info("simpleEdgeType: {}", simpleEdgeType); ApplicationScope applicationScope = new ApplicationScopeImpl(new SimpleId(app, "application")); Id applicationScopeId = applicationScope.getApplication(); logger.info("applicationScope.getApplication(): {}", applicationScopeId); EdgeSerialization edgeSerialization = injector.getInstance(EdgeSerialization.class); Query query = new Query(); query.setCollection(collectionName); query.setLimit(1000); com.google.common.base.Optional<String> queryString = com.google.common.base.Optional.absent(); CollectionInfo collection = getDefaultSchema().getCollection(headEntity.getType(), collectionName); GraphManagerFactory gmf = injector.getInstance(GraphManagerFactory.class); GraphManager gm = gmf.createEdgeManager(applicationScope); final SimpleSearchByEdgeType search = new SimpleSearchByEdgeType(applicationScopeId, simpleEdgeType, Long.MAX_VALUE, SearchByEdgeType.Order.DESCENDING, Optional.absent(), false); gm.loadEdgesFromSource(search).map(markedEdge -> { UUID uuid = markedEdge.getTargetNode().getUuid(); try { EntityRef entityRef = new SimpleEntityRef(entityType, uuid); org.apache.usergrid.persistence.Entity retrieved = em.get(entityRef); long timestamp = 0; String dateString = "NOT TIME-BASED"; if (UUIDUtils.isTimeBased(uuid)) { timestamp = UUIDUtils.getTimestampInMillis(uuid); Date uuidDate = new Date(timestamp); DateFormat df = new SimpleDateFormat(); df.setTimeZone(TimeZone.getTimeZone("GMT")); dateString = df.format(uuidDate) + " GMT"; } if (retrieved != null) { logger.info("{} - {} - entity data found", uuid, dateString); } else { if (removeOrphans && timestamp >= earliestTimestamp && timestamp <= latestTimestamp) { logger.info("{} - {} - entity data NOT found, REMOVING", uuid, dateString); try { //em.removeItemFromCollection(headEntity, collectionName, entityRef ); logger.info("entityRef.asId(): {}", entityRef.asId()); MutationBatch batch = edgeSerialization.deleteEdge(applicationScope, markedEdge, UUIDUtils.newTimeUUID()); logger.info("BATCH: {}", batch); batch.execute(); } catch (Exception e) { logger.error("{} - exception while trying to remove orphaned connection, {}", uuid, e.getMessage()); } } else if (removeOrphans) { logger.info( "{} - {} ({}) - entity data NOT found, not removing because timestamp not in range", uuid, dateString, timestamp); } else { logger.info("{} - {} ({}) - entity data NOT found", uuid, dateString, timestamp); } } } catch (Exception e) { logger.error("{} - exception while trying to load entity data, {} ", uuid, e.getMessage()); } return markedEdge; }).toBlocking().lastOrDefault(null); } }