List of usage examples for java.util.concurrent TimeUnit DAYS
TimeUnit DAYS
To view the source code for java.util.concurrent TimeUnit DAYS.
Click Source Link
From source file:models.NotificationEvent.java
public static void scheduleDeleteOldNotifications() { if (EventConstants.KEEP_TIME_IN_DAYS > 0) { Akka.system().scheduler().schedule(Duration.create(1, TimeUnit.MINUTES), Duration.create(1, TimeUnit.DAYS), new Runnable() { @Override/*from w w w.j ava 2s . c o m*/ public void run() { Date threshold = DateTime.now().minusDays(EventConstants.KEEP_TIME_IN_DAYS).toDate(); List<NotificationEvent> olds = find.where().lt("created", threshold).findList(); for (NotificationEvent old : olds) { old.delete(); } } }, Akka.system().dispatcher()); } }
From source file:org.apache.nifi.avro.AvroTypeUtil.java
/** * Convert an Avro object to a normal Java objects for further processing. * The counter-part method which convert a raw value to an Avro object is {@link #convertToAvroObject(Object, Schema, String, Charset)} *//*from w w w . j a v a 2s. co m*/ private static Object normalizeValue(final Object value, final Schema avroSchema, final String fieldName) { if (value == null) { return null; } switch (avroSchema.getType()) { case INT: { final LogicalType logicalType = avroSchema.getLogicalType(); if (logicalType == null) { return value; } final String logicalName = logicalType.getName(); if (LOGICAL_TYPE_DATE.equals(logicalName)) { // date logical name means that the value is number of days since Jan 1, 1970 return new java.sql.Date(TimeUnit.DAYS.toMillis((int) value)); } else if (LOGICAL_TYPE_TIME_MILLIS.equals(logicalName)) { // time-millis logical name means that the value is number of milliseconds since midnight. return new java.sql.Time((int) value); } break; } case LONG: { final LogicalType logicalType = avroSchema.getLogicalType(); if (logicalType == null) { return value; } final String logicalName = logicalType.getName(); if (LOGICAL_TYPE_TIME_MICROS.equals(logicalName)) { return new java.sql.Time(TimeUnit.MICROSECONDS.toMillis((long) value)); } else if (LOGICAL_TYPE_TIMESTAMP_MILLIS.equals(logicalName)) { return new java.sql.Timestamp((long) value); } else if (LOGICAL_TYPE_TIMESTAMP_MICROS.equals(logicalName)) { return new java.sql.Timestamp(TimeUnit.MICROSECONDS.toMillis((long) value)); } break; } case UNION: if (value instanceof GenericData.Record) { final GenericData.Record avroRecord = (GenericData.Record) value; return normalizeValue(value, avroRecord.getSchema(), fieldName); } return convertUnionFieldValue(value, avroSchema, schema -> normalizeValue(value, schema, fieldName), fieldName); case RECORD: final GenericData.Record record = (GenericData.Record) value; final Schema recordSchema = record.getSchema(); final List<Field> recordFields = recordSchema.getFields(); final Map<String, Object> values = new HashMap<>(recordFields.size()); for (final Field field : recordFields) { final Object avroFieldValue = record.get(field.name()); final Object fieldValue = normalizeValue(avroFieldValue, field.schema(), fieldName + "/" + field.name()); values.put(field.name(), fieldValue); } final RecordSchema childSchema = AvroTypeUtil.createSchema(recordSchema, false); return new MapRecord(childSchema, values); case BYTES: final ByteBuffer bb = (ByteBuffer) value; final LogicalType logicalType = avroSchema.getLogicalType(); if (logicalType != null && LOGICAL_TYPE_DECIMAL.equals(logicalType.getName())) { return new Conversions.DecimalConversion().fromBytes(bb, avroSchema, logicalType); } return AvroTypeUtil.convertByteArray(bb.array()); case FIXED: final GenericFixed fixed = (GenericFixed) value; return AvroTypeUtil.convertByteArray(fixed.bytes()); case ENUM: return value.toString(); case NULL: return null; case STRING: return value.toString(); case ARRAY: if (value instanceof List) { final List<?> list = (List<?>) value; final Object[] valueArray = new Object[list.size()]; for (int i = 0; i < list.size(); i++) { final Schema elementSchema = avroSchema.getElementType(); valueArray[i] = normalizeValue(list.get(i), elementSchema, fieldName + "[" + i + "]"); } return valueArray; } else { final GenericData.Array<?> array = (GenericData.Array<?>) value; final Object[] valueArray = new Object[array.size()]; for (int i = 0; i < array.size(); i++) { final Schema elementSchema = avroSchema.getElementType(); valueArray[i] = normalizeValue(array.get(i), elementSchema, fieldName + "[" + i + "]"); } return valueArray; } case MAP: final Map<?, ?> avroMap = (Map<?, ?>) value; final Map<String, Object> map = new HashMap<>(avroMap.size()); for (final Map.Entry<?, ?> entry : avroMap.entrySet()) { Object obj = entry.getValue(); if (obj instanceof Utf8 || obj instanceof CharSequence) { obj = obj.toString(); } final String key = entry.getKey().toString(); obj = normalizeValue(obj, avroSchema.getValueType(), fieldName + "[" + key + "]"); map.put(key, obj); } return map; } return value; }
From source file:org.opendaylight.controller.cluster.raft.RaftActorServerConfigurationSupportTest.java
@Test public void testAddServerForwardedToLeader() { LOG.info("testAddServerForwardedToLeader starting"); setupNewFollower();/* w w w .ja v a 2 s. co m*/ DefaultConfigParamsImpl configParams = new DefaultConfigParamsImpl(); configParams.setHeartBeatInterval(new FiniteDuration(1, TimeUnit.DAYS)); ActorRef leaderActor = actorFactory.createActor(MessageCollectorActor.props(), actorFactory.generateActorId(LEADER_ID)); TestActorRef<MockRaftActor> followerRaftActor = actorFactory.createTestActor(MockRaftActor.builder() .id(FOLLOWER_ID).peerAddresses(ImmutableMap.of(LEADER_ID, leaderActor.path().toString())) .config(configParams).persistent(Optional.of(false)).props() .withDispatcher(Dispatchers.DefaultDispatcherId()), actorFactory.generateActorId(FOLLOWER_ID)); followerRaftActor.underlyingActor().waitForInitializeBehaviorComplete(); followerRaftActor.tell(new AppendEntries(1, LEADER_ID, 0, 1, Collections.<ReplicatedLogEntry>emptyList(), -1, -1, (short) 0), leaderActor); followerRaftActor.tell(new AddServer(NEW_SERVER_ID, newFollowerRaftActor.path().toString(), true), testKit.getRef()); expectFirstMatching(leaderActor, AddServer.class); LOG.info("testAddServerForwardedToLeader ending"); }
From source file:org.voltdb.iv2.LeaderAppointer.java
public void shutdown() { try {//from www . java 2 s .c o m m_es.execute(new Runnable() { @Override public void run() { try { m_iv2appointees.shutdown(); m_iv2masters.shutdown(); for (BabySitter watcher : m_partitionWatchers.values()) { watcher.shutdown(); } } catch (Exception e) { // don't care, we're going down } } }); m_es.shutdown(); m_es.awaitTermination(356, TimeUnit.DAYS); } catch (InterruptedException e) { tmLog.warn("Unexpected interrupted exception", e); } }
From source file:org.opendaylight.controller.cluster.raft.RaftActorTest.java
@Test public void testFakeSnapshotsForLeaderWithInInitiateSnapshots() throws Exception { new JavaTestKit(getSystem()) { {/* w w w.jav a 2 s. c o m*/ String persistenceId = factory.generateActorId("leader-"); String follower1Id = factory.generateActorId("follower-"); String follower2Id = factory.generateActorId("follower-"); ActorRef followerActor1 = factory.createActor(Props.create(MessageCollectorActor.class), follower1Id); ActorRef followerActor2 = factory.createActor(Props.create(MessageCollectorActor.class), follower2Id); DefaultConfigParamsImpl config = new DefaultConfigParamsImpl(); config.setHeartBeatInterval(new FiniteDuration(1, TimeUnit.DAYS)); config.setIsolatedLeaderCheckInterval(new FiniteDuration(1, TimeUnit.DAYS)); DataPersistenceProvider dataPersistenceProvider = mock(DataPersistenceProvider.class); Map<String, String> peerAddresses = new HashMap<>(); peerAddresses.put(follower1Id, followerActor1.path().toString()); peerAddresses.put(follower2Id, followerActor2.path().toString()); TestActorRef<MockRaftActor> mockActorRef = factory.createTestActor( MockRaftActor.props(persistenceId, peerAddresses, config, dataPersistenceProvider), persistenceId); MockRaftActor leaderActor = mockActorRef.underlyingActor(); leaderActor.getRaftActorContext().setCommitIndex(9); leaderActor.getRaftActorContext().setLastApplied(9); leaderActor.getRaftActorContext().getTermInformation().update(1, persistenceId); leaderActor.waitForInitializeBehaviorComplete(); Leader leader = new Leader(leaderActor.getRaftActorContext()); leaderActor.setCurrentBehavior(leader); assertEquals(RaftState.Leader, leaderActor.getCurrentBehavior().state()); // create 5 entries in the log MockRaftActorContext.MockReplicatedLogBuilder logBuilder = new MockRaftActorContext.MockReplicatedLogBuilder(); leaderActor.getRaftActorContext().setReplicatedLog(logBuilder.createEntries(5, 10, 1).build()); //set the snapshot index to 4 , 0 to 4 are snapshotted leaderActor.getRaftActorContext().getReplicatedLog().setSnapshotIndex(4); //setting replicatedToAllIndex = 9, for the log to clear leader.setReplicatedToAllIndex(9); assertEquals(5, leaderActor.getReplicatedLog().size()); assertEquals(RaftState.Leader, leaderActor.getCurrentBehavior().state()); leaderActor.onReceiveCommand(new AppendEntriesReply(follower1Id, 1, true, 9, 1, (short) 0)); assertEquals(5, leaderActor.getReplicatedLog().size()); assertEquals(RaftState.Leader, leaderActor.getCurrentBehavior().state()); // set the 2nd follower nextIndex to 1 which has been snapshotted leaderActor.onReceiveCommand(new AppendEntriesReply(follower2Id, 1, true, 0, 1, (short) 0)); assertEquals(5, leaderActor.getReplicatedLog().size()); assertEquals(RaftState.Leader, leaderActor.getCurrentBehavior().state()); // simulate a real snapshot leaderActor.onReceiveCommand(SendHeartBeat.INSTANCE); assertEquals(5, leaderActor.getReplicatedLog().size()); assertEquals( String.format("expected to be Leader but was %s. Current Leader = %s ", leaderActor.getCurrentBehavior().state(), leaderActor.getLeaderId()), RaftState.Leader, leaderActor.getCurrentBehavior().state()); //reply from a slow follower does not initiate a fake snapshot leaderActor.onReceiveCommand(new AppendEntriesReply(follower2Id, 1, true, 9, 1, (short) 0)); assertEquals("Fake snapshot should not happen when Initiate is in progress", 5, leaderActor.getReplicatedLog().size()); ByteString snapshotBytes = fromObject(Arrays.asList(new MockRaftActorContext.MockPayload("foo-0"), new MockRaftActorContext.MockPayload("foo-1"), new MockRaftActorContext.MockPayload("foo-2"), new MockRaftActorContext.MockPayload("foo-3"), new MockRaftActorContext.MockPayload("foo-4"))); leaderActor.onReceiveCommand(new CaptureSnapshotReply(snapshotBytes.toByteArray())); assertTrue(leaderActor.getRaftActorContext().getSnapshotManager().isCapturing()); assertEquals("Real snapshot didn't clear the log till replicatedToAllIndex", 0, leaderActor.getReplicatedLog().size()); //reply from a slow follower after should not raise errors leaderActor.onReceiveCommand(new AppendEntriesReply(follower2Id, 1, true, 5, 1, (short) 0)); assertEquals(0, leaderActor.getReplicatedLog().size()); } }; }
From source file:com.jivesoftware.os.upena.deployable.UpenaMain.java
public void run(String[] args) throws Exception { HealthFactory.initialize(BindInterfaceToConfiguration::bindDefault, new HealthCheckRegistry() { @Override//from w ww .j a va 2 s . co m public void register(HealthChecker<?> healthChecker) { } @Override public void unregister(HealthChecker<?> healthChecker) { } }); Properties buildProperties = new Properties(); String upenaVersion = ""; try { buildProperties.load(UpenaMain.class.getClassLoader().getResourceAsStream("build.properties")); upenaVersion = buildProperties.getProperty("my.version", "") + " " + buildProperties.getProperty("my.timestamp", "") + " sha:" + buildProperties.getProperty("git.commit.id", ""); } catch (Exception x) { LOG.warn("Failed to locate build.properties"); } String workingDir = System.getProperty("user.dir"); long start = System.currentTimeMillis(); Exception failed = null; while (start + TimeUnit.SECONDS.toMillis(10) > System.currentTimeMillis()) { try { File lockFile = new File(workingDir, "onlyLetOneRunningAtATime"); lockFile.createNewFile(); FileChannel.open(lockFile.toPath(), StandardOpenOption.WRITE).lock(); failed = null; break; } catch (Exception x) { failed = x; LOG.warn("Failed to acquire lock on onlyLetOneRunningAtATime", x); Thread.sleep(1000); } } if (failed != null) { throw failed; } JDIAPI jvmapi = null; try { jvmapi = new JDIAPI(); } catch (NoClassDefFoundError x) { LOG.warn( "Failed to local tools.jar. Please manually add to classpath. Breakpoint debugger will be disabled."); } String hostname = args[0]; int loopbackPort = Integer.parseInt(System.getProperty("amza.loopback.port", "1174")); int port = Integer.parseInt(System.getProperty("amza.port", "1175")); String multicastGroup = System.getProperty("amza.discovery.group", "225.4.5.6"); int multicastPort = Integer.parseInt(System.getProperty("amza.discovery.port", "1123")); String clusterDiscoveryName = (args.length > 1 ? args[1] : null); String datacenter = System.getProperty("host.datacenter", "unknownDatacenter"); String rack = System.getProperty("host.rack", "unknownRack"); String publicHost = System.getProperty("public.host.name", hostname); UpenaRingHost ringHost = new UpenaRingHost(hostname, port); // TODO include rackId // todo need a better way to create writer id. int writerId = new Random().nextInt(512); TimestampedOrderIdProvider orderIdProvider = new OrderIdProviderImpl( new ConstantWriterIdProvider(writerId)); ObjectMapper mapper = new ObjectMapper(); mapper.configure(SerializationFeature.INDENT_OUTPUT, true); mapper.configure(SerializationFeature.FAIL_ON_EMPTY_BEANS, false); mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); RowsStorageProvider rowsStorageProvider = rowsStorageProvider(orderIdProvider); boolean sslEnable = Boolean.parseBoolean(System.getProperty("ssl.enabled", "true")); String sslKeystorePassword = System.getProperty("ssl.keystore.password", "password"); String sslKeystorePath = System.getProperty("ssl.keystore.path", "./certs/sslKeystore"); String sslKeyStoreAlias = System.getProperty("ssl.keystore.alias", "upenanode").toLowerCase(); boolean sslAutoGenerateSelfSignedCert = Boolean .parseBoolean(System.getProperty("ssl.keystore.autoGenerate", "true")); File sslKeystore = new File(sslKeystorePath); if (sslEnable) { SelfSigningCertGenerator selfSigningCertGenerator = new SelfSigningCertGenerator(); if (sslKeystore.exists()) { if (!selfSigningCertGenerator.validate(sslKeyStoreAlias, sslKeystorePassword, sslKeystore)) { LOG.error("SSL keystore validation failed. keyStoreAlias:{} sslKeystore:{}", sslKeyStoreAlias, sslKeystore); System.exit(1); } } else { sslKeystore.getParentFile().mkdirs(); if (sslAutoGenerateSelfSignedCert) { selfSigningCertGenerator.create(sslKeyStoreAlias, sslKeystorePassword, sslKeystore); } else { LOG.error("Failed to locate mandatory sslKeystore:{}", sslKeystore); System.exit(1); } } } String consumerKey = System.getProperty("upena.consumerKey", clusterDiscoveryName); if (consumerKey == null) { consumerKey = "upena"; LOG.warn("Please provide a stronger consumerKey via -Dupena.consumerKey"); } String finalConsumerKey = consumerKey; String secret = System.getProperty("upena.secret"); if (secret == null) { secret = "secret"; LOG.warn("Please provide a stronger secret via -Dupena.secret"); } String finalSecret = secret; OAuthSigner authSigner = (request) -> { CommonsHttpOAuthConsumer oAuthConsumer = new CommonsHttpOAuthConsumer(finalConsumerKey, finalSecret); oAuthConsumer.setMessageSigner(new HmacSha1MessageSigner()); oAuthConsumer.setTokenWithSecret(finalConsumerKey, finalSecret); return oAuthConsumer.sign(request); }; UpenaSSLConfig upenaSSLConfig = new UpenaSSLConfig(sslEnable, sslAutoGenerateSelfSignedCert, authSigner); UpenaAmzaService upenaAmzaService = null; if (!new File("./state").exists()) { UpdatesSender changeSetSender = new HttpUpdatesSender(sslEnable, sslAutoGenerateSelfSignedCert, authSigner); UpdatesTaker tableTaker = new HttpUpdatesTaker(sslEnable, sslAutoGenerateSelfSignedCert, authSigner); UpenaAmzaServiceConfig upenaAmzaServiceConfig = new UpenaAmzaServiceConfig(); upenaAmzaService = new UpenaAmzaServiceInitializer().initialize(upenaAmzaServiceConfig, orderIdProvider, new com.jivesoftware.os.upena.amza.storage.FstMarshaller( FSTConfiguration.getDefaultConfiguration()), rowsStorageProvider, rowsStorageProvider, rowsStorageProvider, changeSetSender, tableTaker, Optional.<SendFailureListener>absent(), Optional.<UpenaTakeFailureListener>absent(), (changes) -> { }); /*upenaAmzaService.start(ringHost, upenaAmzaServiceConfig.resendReplicasIntervalInMillis, upenaAmzaServiceConfig.applyReplicasIntervalInMillis, upenaAmzaServiceConfig.takeFromNeighborsIntervalInMillis, upenaAmzaServiceConfig.checkIfCompactionIsNeededIntervalInMillis, upenaAmzaServiceConfig.compactTombstoneIfOlderThanNMillis);*/ LOG.info("-----------------------------------------------------------------------"); LOG.info("| OLD Amza Service Online"); LOG.info("-----------------------------------------------------------------------"); } else { LOG.info("-----------------------------------------------------------------------"); LOG.info("| OLD Amza Data is decomissionable"); LOG.info("-----------------------------------------------------------------------"); } BAInterner baInterner = new BAInterner(); AtomicReference<Callable<RingTopology>> topologyProvider = new AtomicReference<>(); // bit of a hack InstanceDescriptor instanceDescriptor = new InstanceDescriptor(datacenter, rack, "", "", "", "", "", "", "", "", 0, "", "", "", 0L, true); ConnectionDescriptorsProvider noAuthConnectionsProvider = (connectionDescriptorsRequest, expectedReleaseGroup) -> { try { RingTopology systemRing = topologyProvider.get().call(); List<ConnectionDescriptor> descriptors = Lists.newArrayList(Iterables.transform(systemRing.entries, input -> new ConnectionDescriptor(instanceDescriptor, sslEnable, false, new HostPort(input.ringHost.getHost(), input.ringHost.getPort()), Collections.emptyMap(), Collections.emptyMap()))); return new ConnectionDescriptorsResponse(200, Collections.emptyList(), "", descriptors, connectionDescriptorsRequest.getRequestUuid()); } catch (Exception e) { throw new RuntimeException(e); } }; TenantsServiceConnectionDescriptorProvider<String> noAuthConnectionPoolProvider = new TenantsServiceConnectionDescriptorProvider<>( Executors.newScheduledThreadPool(1), "", noAuthConnectionsProvider, "", "", 10_000); // TODO config noAuthConnectionPoolProvider.start(); ConnectionDescriptorsProvider connectionsProvider = (connectionDescriptorsRequest, expectedReleaseGroup) -> { try { RingTopology systemRing = topologyProvider.get().call(); List<ConnectionDescriptor> descriptors = Lists.newArrayList(Iterables.transform(systemRing.entries, input -> new ConnectionDescriptor(instanceDescriptor, sslEnable, true, new HostPort(input.ringHost.getHost(), input.ringHost.getPort()), Collections.emptyMap(), Collections.emptyMap()))); return new ConnectionDescriptorsResponse(200, Collections.emptyList(), "", descriptors, connectionDescriptorsRequest.getRequestUuid()); } catch (Exception e) { throw new RuntimeException(e); } }; TenantsServiceConnectionDescriptorProvider<String> connectionPoolProvider = new TenantsServiceConnectionDescriptorProvider<>( Executors.newScheduledThreadPool(1), "", connectionsProvider, "", "", 10_000); // TODO config connectionPoolProvider.start(); HttpDeliveryClientHealthProvider clientHealthProvider = new HttpDeliveryClientHealthProvider("", null, "", 5000, 100); TenantRoutingHttpClientInitializer<String> nonSigningClientInitializer = new TenantRoutingHttpClientInitializer<>( null); TenantAwareHttpClient<String> systemTakeClient = nonSigningClientInitializer .builder(noAuthConnectionPoolProvider, // TODO config clientHealthProvider) .deadAfterNErrors(10).checkDeadEveryNMillis(10_000).maxConnections(1_000) .socketTimeoutInMillis(60_000).build(); // TODO expose to conf TenantAwareHttpClient<String> stripedTakeClient = nonSigningClientInitializer .builder(noAuthConnectionPoolProvider, // TODO config clientHealthProvider) .deadAfterNErrors(10).checkDeadEveryNMillis(10_000).maxConnections(1_000) .socketTimeoutInMillis(60_000).build(); // TODO expose to conf TenantRoutingHttpClientInitializer<String> tenantRoutingHttpClientInitializer = new TenantRoutingHttpClientInitializer<>( new OAuthSignerProvider(() -> authSigner)); TenantAwareHttpClient<String> ringClient = tenantRoutingHttpClientInitializer .builder(connectionPoolProvider, // TODO config clientHealthProvider) .deadAfterNErrors(10).checkDeadEveryNMillis(10_000).maxConnections(1_000) .socketTimeoutInMillis(60_000).build(); // TODO expose to conf AmzaStats amzaStats = new AmzaStats(); AmzaService amzaService = startAmza(workingDir, amzaStats, baInterner, writerId, new RingHost(datacenter, rack, ringHost.getHost(), ringHost.getPort()), new RingMember(ringHost.getHost() + ":" + ringHost.getPort()), authSigner, systemTakeClient, stripedTakeClient, ringClient, topologyProvider, clusterDiscoveryName, multicastGroup, multicastPort); EmbeddedClientProvider embeddedClientProvider = new EmbeddedClientProvider(amzaService); LOG.info("-----------------------------------------------------------------------"); LOG.info("| Amza Service Online"); LOG.info("-----------------------------------------------------------------------"); ObjectMapper storeMapper = new ObjectMapper(); mapper.configure(SerializationFeature.FAIL_ON_EMPTY_BEANS, false); mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); UpenaConfigStore upenaConfigStore = new UpenaConfigStore(orderIdProvider, storeMapper, upenaAmzaService, amzaService, embeddedClientProvider); LOG.info("-----------------------------------------------------------------------"); LOG.info("| Upena Config Store Online"); LOG.info("-----------------------------------------------------------------------"); ExecutorService instanceChangedThreads = Executors.newFixedThreadPool(32); AtomicReference<UbaService> ubaServiceReference = new AtomicReference<>(); UpenaStore upenaStore = new UpenaStore(storeMapper, upenaAmzaService, (instanceChanges) -> { instanceChangedThreads.submit(() -> { UbaService got = ubaServiceReference.get(); if (got != null) { try { got.instanceChanged(instanceChanges); } catch (Exception ex) { ex.printStackTrace(); } } }); }, (changes) -> { }, (change) -> { LOG.info("TODO: tie into conductor. " + change); }, amzaService, embeddedClientProvider); //upenaStore.attachWatchers(); ChaosService chaosService = new ChaosService(upenaStore); SecureRandom random = new SecureRandom(); PasswordStore passwordStore = (key) -> { String password = System.getProperty("sauth.keystore.password"); if (password == null) { File passwordFile = new File(workingDir, "keystore/" + key + ".key"); if (passwordFile.exists()) { password = Files.toString(passwordFile, StandardCharsets.UTF_8); } else { passwordFile.getParentFile().mkdirs(); password = new BigInteger(130, random).toString(32); Files.write(password, passwordFile, StandardCharsets.UTF_8); } } return password; }; SessionStore sessionStore = new SessionStore(TimeUnit.MINUTES.toMillis(60), TimeUnit.MINUTES.toMillis(30)); AtomicReference<UpenaHealth> upenaHealthProvider = new AtomicReference<>(); InstanceHealthly instanceHealthly = (key, version) -> { UpenaHealth upenaHealth = upenaHealthProvider.get(); if (upenaHealth == null) { return false; } ConcurrentMap<RingHost, NodeHealth> ringHostNodeHealth = upenaHealth.buildClusterHealth(); for (NodeHealth nodeHealth : ringHostNodeHealth.values()) { for (NannyHealth nannyHealth : nodeHealth.nannyHealths) { if (nannyHealth.instanceDescriptor.instanceKey.equals(key.getKey())) { return nannyHealth.serviceHealth.fullyOnline ? nannyHealth.serviceHealth.version.equals(version) : false; } } } return false; }; UpenaService upenaService = new UpenaService(passwordStore, sessionStore, upenaStore, chaosService, instanceHealthly); LOG.info("-----------------------------------------------------------------------"); LOG.info("| Upena Service Online"); LOG.info("-----------------------------------------------------------------------"); File defaultPathToRepo = new File(new File(System.getProperty("user.dir"), ".m2"), "repository"); PathToRepo localPathToRepo = new PathToRepo( new File(System.getProperty("pathToRepo", defaultPathToRepo.getAbsolutePath()))); RepositoryProvider repositoryProvider = new RepositoryProvider(localPathToRepo); Host host = new Host(publicHost, datacenter, rack, ringHost.getHost(), ringHost.getPort(), workingDir, null, null); HostKey hostKey = new HostKeyProvider().getNodeKey(upenaStore.hosts, host); String hostInstanceId = System.getProperty("host.instance.id", hostKey.getKey()); host = new Host(publicHost, datacenter, rack, ringHost.getHost(), ringHost.getPort(), workingDir, hostInstanceId, null); UbaLog ubaLog = (what, why, how) -> { try { upenaStore.record("Uba", what, System.currentTimeMillis(), why, ringHost.getHost() + ":" + ringHost.getPort(), how); } catch (Exception x) { x.printStackTrace(); // Hmm lame } }; OktaLog oktaLog = (who, what, why, how) -> { try { upenaStore.record("okta:" + who, what, System.currentTimeMillis(), why, ringHost.getHost() + ":" + ringHost.getPort(), how); } catch (Exception x) { x.printStackTrace(); // Hmm lame } }; OktaCredentialsMatcher.oktaLog = oktaLog; OktaRealm.oktaLog = oktaLog; UpenaClient upenaClient = new UpenaClient() { @Override public InstanceDescriptorsResponse instanceDescriptor( InstanceDescriptorsRequest instanceDescriptorsRequest) throws Exception { return upenaService.instanceDescriptors(instanceDescriptorsRequest); } @Override public void updateKeyPair(String instanceKey, String publicKey) throws Exception { Instance i = upenaStore.instances.get(new InstanceKey(instanceKey)); if (i != null) { LOG.info("Updating publicKey for {}", instanceKey); upenaStore.instances.update(new InstanceKey(instanceKey), new Instance(i.clusterKey, i.hostKey, i.serviceKey, i.releaseGroupKey, i.instanceId, i.enabled, i.locked, publicKey, i.restartTimestampGMTMillis, i.ports)); } } }; final UbaService ubaService = new UbaServiceInitializer().initialize(passwordStore, upenaClient, repositoryProvider, hostKey.getKey(), workingDir, new UbaCoordinate(datacenter, rack, publicHost, host.hostName, "localhost", loopbackPort), null, ubaLog); UpenaHealth upenaHealth = new UpenaHealth(amzaService, upenaSSLConfig, upenaConfigStore, ubaService, new RingHost(datacenter, rack, ringHost.getHost(), ringHost.getPort()), hostKey); upenaHealthProvider.set(upenaHealth); DiscoveredRoutes discoveredRoutes = new DiscoveredRoutes(); ShiroRequestHelper shiroRequestHelper = new ShiroRequestHelper(TimeUnit.DAYS.toMillis(1)); // TODO expose Sys prop? String shiroConfigLocation = System.getProperty("shiro.ini.location", "classpath:shiro.ini"); // classpath:oktashiro.ini UpenaJerseyEndpoints jerseyEndpoints = new UpenaJerseyEndpoints(shiroConfigLocation) .addInjectable(ShiroRequestHelper.class, shiroRequestHelper) .addEndpoint(UpenaClusterRestEndpoints.class).addEndpoint(UpenaHostRestEndpoints.class) .addEndpoint(UpenaServiceRestEndpoints.class).addEndpoint(UpenaReleaseRestEndpoints.class) .addEndpoint(UpenaInstanceRestEndpoints.class).addEndpoint(UpenaTenantRestEndpoints.class) .addInjectable(upenaHealth).addInjectable(upenaService).addInjectable(upenaStore) .addInjectable(upenaConfigStore).addInjectable(ubaService) //.addEndpoint(AmzaReplicationRestEndpoints.class) //.addInjectable(UpenaAmzaInstance.class, upenaAmzaService) .addEndpoint(UpenaEndpoints.class).addEndpoint(UpenaConnectivityEndpoints.class) .addEndpoint(UpenaManagedDeployableEndpoints.class).addEndpoint(UpenaHealthEndpoints.class) .addEndpoint(UpenaRepoEndpoints.class).addInjectable(DiscoveredRoutes.class, discoveredRoutes) .addInjectable(UpenaRingHost.class, ringHost).addInjectable(HostKey.class, hostKey) .addInjectable(UpenaAutoRelease.class, new UpenaAutoRelease(repositoryProvider, upenaStore)) .addInjectable(PathToRepo.class, localPathToRepo); PercentileHealthCheckConfig phcc = bindDefault(PercentileHealthCheckConfig.class); PercentileHealthChecker authFilterHealthCheck = new PercentileHealthChecker(phcc); AuthValidationFilter authValidationFilter = new AuthValidationFilter(authFilterHealthCheck); authValidationFilter.addEvaluator(new NoAuthEvaluator(), "/", "/swagger.json", "/ui/*", // Handled by Shiro "/repo/*" // Cough ); OAuth1Signature verifier = new OAuth1Signature(new OAuthServiceLocatorShim()); OAuthSecretManager oAuthSecretManager = new OAuthSecretManager() { @Override public void clearCache() { } @Override public String getSecret(String id) throws AuthValidationException { return id.equals(finalConsumerKey) ? finalSecret : null; } @Override public void verifyLastSecretRemovalTime() throws Exception { } }; AuthValidator<OAuth1Signature, OAuth1Request> oAuthValidator = new DefaultOAuthValidator( Executors.newScheduledThreadPool(1), Long.MAX_VALUE, oAuthSecretManager, 60_000, false, false); oAuthValidator.start(); authValidationFilter.addEvaluator(new NoAuthEvaluator(), "/repo/*", "/amza/rows/stream/*", "/amza/rows/taken/*", "/amza/pong/*", "/amza/invalidate/*"); authValidationFilter.addEvaluator(new OAuthEvaluator(oAuthValidator, verifier), "/upena/*", "/amza/*"); // TODO something better someday String upenaApiUsername = System.getProperty("upena.api.username", null); String upenaApiPassword = System.getProperty("upena.api.password", null); if (upenaApiUsername != null && upenaApiPassword != null) { authValidationFilter.addEvaluator(containerRequestContext -> { String authCredentials = containerRequestContext.getHeaderString("Authorization"); if (authCredentials == null) { return AuthStatus.not_handled; } final String encodedUserPassword = authCredentials.replaceFirst("Basic" + " ", ""); String usernameAndPassword = null; try { byte[] decodedBytes = Base64.getDecoder().decode(encodedUserPassword); usernameAndPassword = new String(decodedBytes, "UTF-8"); } catch (IOException e) { return AuthStatus.denied; } final StringTokenizer tokenizer = new StringTokenizer(usernameAndPassword, ":"); final String username = tokenizer.nextToken(); final String password = tokenizer.nextToken(); boolean authenticationStatus = upenaApiUsername.equals(username) && upenaApiPassword.equals(password); return authenticationStatus ? AuthStatus.authorized : AuthStatus.denied; }, "/api/*"); } jerseyEndpoints.addContainerRequestFilter(authValidationFilter); String region = System.getProperty("aws.region", null); String roleArn = System.getProperty("aws.roleArn", null); AWSClientFactory awsClientFactory = new AWSClientFactory(region, roleArn); String accountName = System.getProperty("account.name", clusterDiscoveryName == null ? "" : clusterDiscoveryName); String humanReadableUpenaClusterName = datacenter + " - " + accountName; injectUI(upenaVersion, awsClientFactory, storeMapper, mapper, jvmapi, amzaService, localPathToRepo, repositoryProvider, hostKey, ringHost, upenaSSLConfig, port, sessionStore, ubaService, upenaHealth, upenaStore, upenaConfigStore, jerseyEndpoints, humanReadableUpenaClusterName, discoveredRoutes); injectAmza(baInterner, amzaStats, jerseyEndpoints, amzaService, ringClient); InitializeRestfulServer initializeRestfulServer = new InitializeRestfulServer(false, port, "UpenaNode", sslEnable, sslKeyStoreAlias, sslKeystorePassword, sslKeystorePath, 128, 10_000); buildSwagger(); initializeRestfulServer.addClasspathResource("/resources"); initializeRestfulServer.addContextHandler("/", jerseyEndpoints); RestfulServer restfulServer = initializeRestfulServer.build(); restfulServer.start(); LOG.info("-----------------------------------------------------------------------"); LOG.info("| Jetty Service Online"); LOG.info("-----------------------------------------------------------------------"); UpenaJerseyEndpoints loopbackJerseyEndpoints = new UpenaJerseyEndpoints(null) .addEndpoint(UpenaLoopbackEndpoints.class).addEndpoint(UpenaConfigRestEndpoints.class) .addInjectable(SessionStore.class, sessionStore) .addInjectable(DiscoveredRoutes.class, discoveredRoutes).addInjectable(upenaConfigStore) .addInjectable(upenaStore).addInjectable(upenaHealth) .addInjectable(UpenaService.class, upenaService); InitializeRestfulServer initializeLoopbackRestfulServer = new InitializeRestfulServer( Boolean.parseBoolean(System.getProperty("amza.loopback.strict", "true")), loopbackPort, "UpenaNode", false, sslKeyStoreAlias, sslKeystorePassword, sslKeystorePath, 128, 10_000); initializeLoopbackRestfulServer.addClasspathResource("/resources"); initializeLoopbackRestfulServer.addContextHandler("/", loopbackJerseyEndpoints); RestfulServer loopbackRestfulServer = initializeLoopbackRestfulServer.build(); loopbackRestfulServer.start(); LOG.info("-----------------------------------------------------------------------"); LOG.info("| Jetty Service Online"); LOG.info("-----------------------------------------------------------------------"); if (ubaService != null) { Executors.newScheduledThreadPool(1).scheduleWithFixedDelay(() -> { try { ubaService.nanny(); } catch (Exception ex) { LOG.error("Nanny failure", ex); } }, 15, 15, TimeUnit.SECONDS); LOG.info("-----------------------------------------------------------------------"); LOG.info("| Uba Service Online"); LOG.info("-----------------------------------------------------------------------"); } ubaServiceReference.set(ubaService); /*String peers = System.getProperty("manual.peers"); if (peers != null) { String[] hostPortTuples = peers.split(","); for (String hostPortTuple : hostPortTuples) { String hostPort = hostPortTuple.trim(); if (hostPort.length() > 0 && hostPort.contains(":")) { String[] host_port = hostPort.split(":"); try { UpenaRingHost anotherRingHost = new UpenaRingHost(host_port[0].trim(), Integer.parseInt(host_port[1].trim())); List<UpenaRingHost> ring = upenaAmzaService.getRing("master"); if (!ring.contains(anotherRingHost)) { LOG.info("Adding host to the cluster: " + anotherRingHost); upenaAmzaService.addRingHost("master", anotherRingHost); } } catch (Exception x) { LOG.warn("Malformed hostPortTuple {}", hostPort); } } else { LOG.warn("Malformed hostPortTuple {}", hostPort); } } }*/ String vpc = System.getProperty("aws.vpc", null); UpenaAWSLoadBalancerNanny upenaAWSLoadBalancerNanny = new UpenaAWSLoadBalancerNanny(vpc, upenaStore, hostKey, awsClientFactory); Executors.newScheduledThreadPool(1).scheduleWithFixedDelay(() -> { try { upenaAWSLoadBalancerNanny.ensureSelf(); } catch (Exception x) { LOG.warn("Failures while nannying load loadbalancer.", x); } }, 1, 1, TimeUnit.MINUTES); // TODO better LOG.info("-----------------------------------------------------------------------"); LOG.info("| Waiting for amza to be ready...."); LOG.info("-----------------------------------------------------------------------"); while (!amzaService.isReady()) { Thread.sleep(1000); } LOG.info("-----------------------------------------------------------------------"); LOG.info("| Begin Migration"); LOG.info("-----------------------------------------------------------------------"); upenaStore.init(orderIdProvider, Integer.parseInt(System.getProperty("min.service.port", "10000")), Integer.parseInt(System.getProperty("max.service.port", String.valueOf(Short.MAX_VALUE))), false); upenaConfigStore.init(); LOG.info("-----------------------------------------------------------------------"); LOG.info("| End Migration"); LOG.info("-----------------------------------------------------------------------"); addManualPeers(amzaService); Host gotHost = upenaStore.hosts.get(hostKey); if (gotHost == null || !gotHost.equals(host)) { upenaStore.hosts.update(hostKey, host); } }
From source file:org.opendaylight.controller.cluster.raft.RaftActorServerConfigurationSupportTest.java
@Test public void testOnApplyState() { LOG.info("testOnApplyState starting"); DefaultConfigParamsImpl configParams = new DefaultConfigParamsImpl(); configParams.setHeartBeatInterval(new FiniteDuration(1, TimeUnit.DAYS)); TestActorRef<MockRaftActor> noLeaderActor = actorFactory.createTestActor(MockRaftActor.builder() .id(LEADER_ID).peerAddresses(ImmutableMap.of(FOLLOWER_ID, followerActor.path().toString())) .config(configParams).persistent(Optional.of(false)).props() .withDispatcher(Dispatchers.DefaultDispatcherId()), actorFactory.generateActorId(LEADER_ID)); RaftActorServerConfigurationSupport support = new RaftActorServerConfigurationSupport( noLeaderActor.underlyingActor()); ReplicatedLogEntry serverConfigEntry = new SimpleReplicatedLogEntry(1, 1, new ServerConfigurationPayload(Collections.<ServerInfo>emptyList())); boolean handled = support.handleMessage(new ApplyState(null, null, serverConfigEntry), ActorRef.noSender()); assertEquals("Message handled", true, handled); ReplicatedLogEntry nonServerConfigEntry = new SimpleReplicatedLogEntry(1, 1, new MockRaftActorContext.MockPayload("1")); handled = support.handleMessage(new ApplyState(null, null, nonServerConfigEntry), ActorRef.noSender()); assertEquals("Message handled", false, handled); LOG.info("testOnApplyState ending"); }
From source file:org.opendaylight.controller.cluster.raft.RaftActorServerConfigurationSupportTest.java
@Test public void testRemoveServerWithNoLeader() { LOG.info("testRemoveServerWithNoLeader starting"); DefaultConfigParamsImpl configParams = new DefaultConfigParamsImpl(); configParams.setHeartBeatInterval(new FiniteDuration(1, TimeUnit.DAYS)); TestActorRef<MockRaftActor> leaderActor = actorFactory.createTestActor(MockRaftActor.builder().id(LEADER_ID) .peerAddresses(ImmutableMap.of(FOLLOWER_ID, followerActor.path().toString())).config(configParams) .persistent(Optional.of(false)).props().withDispatcher(Dispatchers.DefaultDispatcherId()), actorFactory.generateActorId(LEADER_ID)); leaderActor.underlyingActor().waitForInitializeBehaviorComplete(); leaderActor.tell(new RemoveServer(FOLLOWER_ID), testKit.getRef()); RemoveServerReply removeServerReply = testKit.expectMsgClass(testKit.duration("5 seconds"), RemoveServerReply.class); assertEquals("getStatus", ServerChangeStatus.NO_LEADER, removeServerReply.getStatus()); LOG.info("testRemoveServerWithNoLeader ending"); }
From source file:com.linkedin.pinot.integration.tests.BaseClusterIntegrationTest.java
public static Future<Map<File, File>> buildSegmentsFromAvro(final List<File> avroFiles, Executor executor, int baseSegmentIndex, final File baseDirectory, final File segmentTarDir, final String tableName, final boolean createStarTreeIndex, final com.linkedin.pinot.common.data.Schema inputPinotSchema) { int segmentCount = avroFiles.size(); LOGGER.info("Building " + segmentCount + " segments in parallel"); List<ListenableFutureTask<Pair<File, File>>> futureTasks = new ArrayList<ListenableFutureTask<Pair<File, File>>>(); for (int i = 1; i <= segmentCount; ++i) { final int segmentIndex = i - 1; final int segmentNumber = i + baseSegmentIndex; final ListenableFutureTask<Pair<File, File>> buildSegmentFutureTask = ListenableFutureTask .<Pair<File, File>>create(new Callable<Pair<File, File>>() { @Override/*w ww . j av a 2 s .c o m*/ public Pair<File, File> call() throws Exception { try { // Build segment LOGGER.info("Starting to build segment " + segmentNumber); File outputDir = new File(baseDirectory, "segment-" + segmentNumber); final File inputAvroFile = avroFiles.get(segmentIndex); final SegmentGeneratorConfig genConfig = SegmentTestUtils .getSegmentGenSpecWithSchemAndProjectedColumns(inputAvroFile, outputDir, TimeUnit.DAYS, tableName, inputPinotSchema); if (inputPinotSchema != null) { genConfig.setSchema(inputPinotSchema); } // jfim: We add a space and a special character to do a regression test for PINOT-3296 Segments with spaces // in their filename don't work properly genConfig.setSegmentNamePostfix(Integer.toString(segmentNumber) + " %"); genConfig.setEnableStarTreeIndex(createStarTreeIndex); // Enable off heap star tree format in the integration test. StarTreeIndexSpec starTreeIndexSpec = null; if (createStarTreeIndex) { starTreeIndexSpec = new StarTreeIndexSpec(); starTreeIndexSpec.setEnableOffHeapFormat(true); } genConfig.setStarTreeIndexSpec(starTreeIndexSpec); final SegmentIndexCreationDriver driver = SegmentCreationDriverFactory.get(null); driver.init(genConfig); driver.build(); // Tar segment String segmentName = outputDir.list()[0]; final String tarGzPath = TarGzCompressionUtils.createTarGzOfDirectory( outputDir.getAbsolutePath() + "/" + segmentName, new File(segmentTarDir, segmentName).getAbsolutePath()); LOGGER.info("Completed segment " + segmentNumber + " : " + segmentName + " from file " + inputAvroFile.getName()); return new ImmutablePair<File, File>(inputAvroFile, new File(tarGzPath)); } catch (Exception e) { LOGGER.error("Exception while building segment input: {} output {} ", avroFiles.get(segmentIndex), "segment-" + segmentNumber); throw new RuntimeException(e); } } }); futureTasks.add(buildSegmentFutureTask); executor.execute(buildSegmentFutureTask); } ListenableFuture<List<Pair<File, File>>> pairListFuture = Futures.allAsList(futureTasks); return Futures.transform(pairListFuture, new AsyncFunction<List<Pair<File, File>>, Map<File, File>>() { @Override public ListenableFuture<Map<File, File>> apply(List<Pair<File, File>> input) throws Exception { Map<File, File> avroToSegmentMap = new HashMap<File, File>(); for (Pair<File, File> avroToSegmentPair : input) { avroToSegmentMap.put(avroToSegmentPair.getLeft(), avroToSegmentPair.getRight()); } return Futures.immediateFuture(avroToSegmentMap); } }); }
From source file:org.opendaylight.controller.cluster.raft.RaftActorServerConfigurationSupportTest.java
@Test public void testRemoveServerForwardToLeader() { LOG.info("testRemoveServerForwardToLeader starting"); DefaultConfigParamsImpl configParams = new DefaultConfigParamsImpl(); configParams.setHeartBeatInterval(new FiniteDuration(1, TimeUnit.DAYS)); ActorRef leaderActor = actorFactory.createTestActor(MessageCollectorActor.props(), actorFactory.generateActorId(LEADER_ID)); TestActorRef<MockRaftActor> followerRaftActor = actorFactory.createTestActor(MockRaftActor.builder() .id(FOLLOWER_ID).peerAddresses(ImmutableMap.of(LEADER_ID, leaderActor.path().toString())) .config(configParams).persistent(Optional.of(false)).props() .withDispatcher(Dispatchers.DefaultDispatcherId()), actorFactory.generateActorId(FOLLOWER_ID)); followerRaftActor.underlyingActor().waitForInitializeBehaviorComplete(); followerRaftActor.tell(new AppendEntries(1, LEADER_ID, 0, 1, Collections.<ReplicatedLogEntry>emptyList(), -1, -1, (short) 0), leaderActor); followerRaftActor.tell(new RemoveServer(FOLLOWER_ID), testKit.getRef()); expectFirstMatching(leaderActor, RemoveServer.class); LOG.info("testRemoveServerForwardToLeader ending"); }