Java tutorial
/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.twitter.distributedlog.service; import com.twitter.distributedlog.DistributedLogConfiguration; import com.twitter.distributedlog.LocalDLMEmulator; import com.twitter.distributedlog.metadata.BKDLConfig; import com.twitter.distributedlog.metadata.DLMetadata; import com.twitter.distributedlog.service.streamset.IdentityStreamPartitionConverter; import com.twitter.finagle.builder.Server; import org.apache.bookkeeper.conf.ServerConfiguration; import org.apache.bookkeeper.shims.zk.ZooKeeperServerShim; import org.apache.bookkeeper.stats.NullStatsProvider; import org.apache.bookkeeper.util.IOUtils; import org.apache.bookkeeper.util.LocalBookKeeper; import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.tuple.Pair; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.File; import java.net.BindException; import java.net.InetSocketAddress; import java.net.URI; import java.util.ArrayList; import java.util.List; import java.util.concurrent.TimeUnit; /** * DistributedLog Cluster is an emulator to run distributedlog components. */ public class DistributedLogCluster { private static final Logger LOG = LoggerFactory.getLogger(DistributedLogCluster.class); public static Builder newBuilder() { return new Builder(); } /** * Builder to build distributedlog cluster. */ public static class Builder { int _numBookies = 3; boolean _shouldStartZK = true; String _zkHost = "127.0.0.1"; int _zkPort = 0; boolean _shouldStartProxy = true; int _proxyPort = 7000; DistributedLogConfiguration _dlConf = new DistributedLogConfiguration().setLockTimeout(10) .setOutputBufferSize(0).setImmediateFlushEnabled(true); ServerConfiguration _bkConf = new ServerConfiguration(); private Builder() { } /** * How many bookies to run. By default is 3. * * @return builder */ public Builder numBookies(int numBookies) { this._numBookies = numBookies; return this; } /** * Whether to start zookeeper? By default is true. * * @param startZK * flag to start zookeeper? * @return builder */ public Builder shouldStartZK(boolean startZK) { this._shouldStartZK = startZK; return this; } /** * ZooKeeper server to run. By default it runs locally on '127.0.0.1'. * * @param zkServers * zk servers * @return builder */ public Builder zkServers(String zkServers) { this._zkHost = zkServers; return this; } /** * ZooKeeper server port to listen on. By default it listens on 2181. * * @param zkPort * zookeeper server port. * @return builder. */ public Builder zkPort(int zkPort) { this._zkPort = zkPort; return this; } /** * Whether to start proxy or not. By default is true. * * @param startProxy * whether to start proxy or not. * @return builder */ public Builder shouldStartProxy(boolean startProxy) { this._shouldStartProxy = startProxy; return this; } /** * Port that proxy server to listen on. By default is 7000. * * @param proxyPort * port that proxy server to listen on. * @return builder */ public Builder proxyPort(int proxyPort) { this._proxyPort = proxyPort; return this; } /** * DistributedLog Configuration * * @param dlConf * distributedlog configuration * @return builder */ public Builder dlConf(DistributedLogConfiguration dlConf) { this._dlConf = dlConf; return this; } /** * Bookkeeper server configuration * * @param bkConf * bookkeeper server configuration * @return builder */ public Builder bkConf(ServerConfiguration bkConf) { this._bkConf = bkConf; return this; } public DistributedLogCluster build() throws Exception { // build the cluster return new DistributedLogCluster(_dlConf, _bkConf, _numBookies, _shouldStartZK, _zkHost, _zkPort, _shouldStartProxy, _proxyPort); } } public static class DLServer { static final int MAX_RETRIES = 20; static final int MIN_PORT = 1025; static final int MAX_PORT = 65535; int proxyPort; public final InetSocketAddress address; public final Pair<DistributedLogServiceImpl, Server> dlServer; protected DLServer(DistributedLogConfiguration dlConf, URI uri, int basePort) throws Exception { proxyPort = basePort; boolean success = false; int retries = 0; Pair<DistributedLogServiceImpl, Server> serverPair = null; while (!success) { try { com.twitter.distributedlog.service.config.ServerConfiguration serverConf = new com.twitter.distributedlog.service.config.ServerConfiguration(); serverConf.loadConf(dlConf); serverConf.setServerShardId(proxyPort); serverPair = DistributedLogServer.runServer(serverConf, dlConf, uri, new IdentityStreamPartitionConverter(), new NullStatsProvider(), proxyPort); success = true; } catch (BindException be) { retries++; if (retries > MAX_RETRIES) { throw be; } proxyPort++; if (proxyPort > MAX_PORT) { proxyPort = MIN_PORT; } } } LOG.info("Runnning DL on port {}", proxyPort); dlServer = serverPair; address = DLSocketAddress.getSocketAddress(proxyPort); } public InetSocketAddress getAddress() { return address; } public void shutdown() { DistributedLogServer.closeServer(dlServer, 0, TimeUnit.MILLISECONDS); } } private final DistributedLogConfiguration dlConf; private final ZooKeeperServerShim zks; private final LocalDLMEmulator dlmEmulator; private DLServer dlServer; private final boolean shouldStartProxy; private final int proxyPort; private final List<File> tmpDirs = new ArrayList<File>(); private DistributedLogCluster(DistributedLogConfiguration dlConf, ServerConfiguration bkConf, int numBookies, boolean shouldStartZK, String zkServers, int zkPort, boolean shouldStartProxy, int proxyPort) throws Exception { this.dlConf = dlConf; if (shouldStartZK) { File zkTmpDir = IOUtils.createTempDir("zookeeper", "distrlog"); tmpDirs.add(zkTmpDir); if (0 == zkPort) { Pair<ZooKeeperServerShim, Integer> serverAndPort = LocalDLMEmulator.runZookeeperOnAnyPort(zkTmpDir); this.zks = serverAndPort.getLeft(); zkPort = serverAndPort.getRight(); } else { this.zks = LocalBookKeeper.runZookeeper(1000, zkPort, zkTmpDir); } } else { this.zks = null; } this.dlmEmulator = LocalDLMEmulator.newBuilder().numBookies(numBookies).zkHost(zkServers).zkPort(zkPort) .serverConf(bkConf).shouldStartZK(false).build(); this.shouldStartProxy = shouldStartProxy; this.proxyPort = proxyPort; } public void start() throws Exception { this.dlmEmulator.start(); BKDLConfig bkdlConfig = new BKDLConfig(this.dlmEmulator.getZkServers(), "/ledgers").setACLRootPath(".acl"); DLMetadata.create(bkdlConfig).update(this.dlmEmulator.getUri()); if (shouldStartProxy) { this.dlServer = new DLServer(dlConf, this.dlmEmulator.getUri(), proxyPort); } else { this.dlServer = null; } } public void stop() throws Exception { if (null != dlServer) { this.dlServer.shutdown(); } this.dlmEmulator.teardown(); if (null != this.zks) { this.zks.stop(); } for (File dir : tmpDirs) { FileUtils.deleteDirectory(dir); } } public URI getUri() { return this.dlmEmulator.getUri(); } public String getZkServers() { return this.dlmEmulator.getZkServers(); } public String getProxyFinagleStr() { return "inet!" + (dlServer == null ? "127.0.0.1:" + proxyPort : dlServer.getAddress().toString()); } }