Java tutorial
/** * Licensed to Cloudera, Inc. under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. Cloudera, Inc. licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.cloudera.whirr.cm.server.impl; import java.io.BufferedWriter; import java.io.File; import java.io.FileWriter; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; import java.util.TreeSet; import java.util.concurrent.atomic.AtomicBoolean; import java.util.zip.ZipEntry; import java.util.zip.ZipInputStream; import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.WordUtils; import org.apache.cxf.jaxrs.ext.multipart.InputStreamDataSource; import org.apache.maven.artifact.versioning.ArtifactVersion; import org.apache.maven.artifact.versioning.DefaultArtifactVersion; import com.cloudera.api.ApiRootResource; import com.cloudera.api.ClouderaManagerClientBuilder; import com.cloudera.api.DataView; import com.cloudera.api.model.ApiBulkCommandList; import com.cloudera.api.model.ApiCluster; import com.cloudera.api.model.ApiClusterList; import com.cloudera.api.model.ApiClusterVersion; import com.cloudera.api.model.ApiCommand; import com.cloudera.api.model.ApiConfig; import com.cloudera.api.model.ApiConfigList; import com.cloudera.api.model.ApiHost; import com.cloudera.api.model.ApiHostRef; import com.cloudera.api.model.ApiHostRefList; import com.cloudera.api.model.ApiParcel; import com.cloudera.api.model.ApiRole; import com.cloudera.api.model.ApiRoleConfigGroup; import com.cloudera.api.model.ApiRoleNameList; import com.cloudera.api.model.ApiRoleState; import com.cloudera.api.model.ApiService; import com.cloudera.api.model.ApiServiceConfig; import com.cloudera.api.model.ApiServiceList; import com.cloudera.api.model.ApiServiceState; import com.cloudera.api.v3.ParcelResource; import com.cloudera.api.v3.RootResourceV3; import com.cloudera.api.v4.RootResourceV4; import com.cloudera.api.v5.RootResourceV5; import com.cloudera.api.v6.RootResourceV6; import com.cloudera.whirr.cm.server.CmServer; import com.cloudera.whirr.cm.server.CmServerBuilder.CmServerCommandMethod; import com.cloudera.whirr.cm.server.CmServerCluster; import com.cloudera.whirr.cm.server.CmServerException; import com.cloudera.whirr.cm.server.CmServerService; import com.cloudera.whirr.cm.server.CmServerService.CmServerServiceStatus; import com.cloudera.whirr.cm.server.CmServerServiceBuilder; import com.cloudera.whirr.cm.server.CmServerServiceType; import com.cloudera.whirr.cm.server.CmServerServiceTypeCms; import com.cloudera.whirr.cm.server.impl.CmServerLog.CmServerLogSyncCommand; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; public class CmServerImpl implements CmServer { // CM Version Matrix, of form {CM_VERSION=CM_API_VERSION} // Entry for the latest CM minor version for each API upgrade, from baseline 4.5.0 public static Map<String, Integer> VERSION_CM_API_MATRIX = ImmutableMap.of("4.5.0", 3, "4.5.3", 3, "4.6.3", 4, "4.8.0", 5, "5.0.0", 6); public static String VERSION_CM_API_MATRIX_CM_MIN = VERSION_CM_API_MATRIX.keySet() .toArray(new String[VERSION_CM_API_MATRIX.size()])[0]; public static String VERSION_CM_API_MATRIX_CM_MAX = VERSION_CM_API_MATRIX.keySet() .toArray(new String[VERSION_CM_API_MATRIX.size()])[VERSION_CM_API_MATRIX.size() - 1]; public static int VERSION_CM_API_MATRIX_CM_MAX_MAJOR = new DefaultArtifactVersion(VERSION_CM_API_MATRIX_CM_MAX) .getMajorVersion(); public static int CM_VERSION_API_EARLIEST = VERSION_CM_API_MATRIX.values() .toArray(new Integer[VERSION_CM_API_MATRIX.size()])[0]; public static int VERSION_CM_API_MATRIX_API_MAX = VERSION_CM_API_MATRIX.values() .toArray(new Integer[VERSION_CM_API_MATRIX.size()])[VERSION_CM_API_MATRIX.size() - 1]; public static int VERSION_CM_MIN = 5; public static int VERSION_CDH_MIN = 4; public static int VERSION_CDH_MAX = 5; private static final String CDH_REPO_PREFIX = "CDH"; private static final String CM_PARCEL_STAGE_DOWNLOADED = "DOWNLOADED"; private static final String CM_PARCEL_STAGE_DISTRIBUTED = "DISTRIBUTED"; private static final String CM_PARCEL_STAGE_ACTIVATED = "ACTIVATED"; private static final String CM_CONFIG_UPDATE_MESSAGE = "Update base config group with defaults"; private static int API_POLL_PERIOD_MS = 500; private static int API_POLL_PERIOD_BACKOFF_NUMBER = 3; private static int API_POLL_PERIOD_BACKOFF_INCRAMENT = 2; private CmServerLog logger; private String version; private int versionApi; private int versionCdh; private CmServerService host; final private RootResourceV3 apiResourceRootV3; final private RootResourceV4 apiResourceRootV4; @SuppressWarnings("unused") final private RootResourceV5 apiResourceRootV5; final private RootResourceV6 apiResourceRootV6; private boolean isFirstStartRequired = true; protected CmServerImpl(String version, String vesionApi, String versionCdh, String ip, String ipInternal, int port, String user, String password, CmServerLog logger) throws CmServerException { this.version = getVersion(version); this.versionApi = getVersionApi(this.version, vesionApi); this.versionCdh = getVersionCdh(versionCdh); this.host = new CmServerServiceBuilder().ip(ip).ipInternal(ipInternal).build(); this.logger = logger; ApiRootResource apiResource = new ClouderaManagerClientBuilder().withHost(ip).withPort(port) .withUsernamePassword(user, password).build(); this.apiResourceRootV3 = apiResource.getRootV3(); this.apiResourceRootV4 = this.versionApi >= 4 ? apiResource.getRootV4() : null; this.apiResourceRootV5 = this.versionApi >= 5 ? apiResource.getRootV5() : null; this.apiResourceRootV6 = this.versionApi >= 6 ? apiResource.getRootV6() : null; } private static String getVersion(String version) throws CmServerException { String versionValidated = null; if (version != null && !version.equals("")) { String versionFullyQualified = version.contains(".") ? version : version + "." + Integer.MAX_VALUE + "." + Integer.MAX_VALUE; if (new DefaultArtifactVersion(versionFullyQualified) .compareTo(new DefaultArtifactVersion(VERSION_CM_API_MATRIX_CM_MIN)) < 0 || new DefaultArtifactVersion(versionFullyQualified).compareTo(new DefaultArtifactVersion( new DefaultArtifactVersion(VERSION_CM_API_MATRIX_CM_MAX).getMajorVersion() + "." + Integer.MAX_VALUE + "." + Integer.MAX_VALUE)) > 0) { throw new CmServerException("Requested CM version [" + version + "] is invalid and cannot be reconciled with CM versions " + VERSION_CM_API_MATRIX.keySet()); } else { versionValidated = version; } } versionValidated = versionValidated == null ? "" + VERSION_CM_API_MATRIX_CM_MAX_MAJOR : versionValidated; if (new DefaultArtifactVersion(versionValidated).getMajorVersion() < VERSION_CM_MIN) { throw new CmServerException( "Requested CM version [" + version + "] is below required mininum [" + VERSION_CM_MIN + "]"); } return versionValidated; } private static int getVersionApi(String version, String versionApi) throws CmServerException { Integer versionApiValidated = null; if (version == null || version.equals("")) { version = VERSION_CM_API_MATRIX_CM_MAX; } if (!version.contains(".")) { String versionLatest = null; for (String versionIterator : VERSION_CM_API_MATRIX.keySet()) { if (versionIterator.startsWith(version)) { versionLatest = versionIterator; } } version = versionLatest; } if (version != null) { ArtifactVersion versionArtifact = new DefaultArtifactVersion(version); for (String versionUpperBound : VERSION_CM_API_MATRIX.keySet()) { if (versionArtifact.compareTo(new DefaultArtifactVersion(versionUpperBound)) <= 0) { versionApiValidated = VERSION_CM_API_MATRIX.get(versionUpperBound); break; } if (versionApiValidated == null) { versionApiValidated = VERSION_CM_API_MATRIX_API_MAX; } } } if (version == null || versionApi != null && !versionApi.equals("") && (new DefaultArtifactVersion(versionApi) .compareTo(new DefaultArtifactVersion(versionApiValidated.toString())) > 0 || new DefaultArtifactVersion(versionApi) .compareTo(new DefaultArtifactVersion("" + CM_VERSION_API_EARLIEST)) < 0)) { throw new CmServerException("Requested CM API version [" + versionApi + "] of CM version [" + version + "] could not be reconciled with CM API version matrix " + VERSION_CM_API_MATRIX); } if (versionApi != null && !StringUtils.isNumeric(versionApi)) { throw new CmServerException("CM API version requested is non-numeric [" + versionApi + "]"); } return versionApi == null ? versionApiValidated : Integer.parseInt(versionApi); } private static int getVersionCdh(String versionCdh) throws CmServerException { int versionCdhValidated; if (versionCdh == null || versionCdh.equals("")) { versionCdhValidated = VERSION_CDH_MAX; } else { try { versionCdhValidated = new DefaultArtifactVersion(versionCdh).getMajorVersion(); if (versionCdhValidated < 4 || versionCdhValidated > VERSION_CDH_MAX) { throw new CmServerException("CDH version requested [" + versionCdh + "] is not within the supported major version range [" + VERSION_CDH_MIN + "-" + VERSION_CDH_MAX + "]"); } } catch (Exception e) { throw new CmServerException("CDH version requested [" + versionCdh + "] cannot be corelated with CDH versions " + Arrays.asList(ApiClusterVersion.values())); } } return versionCdhValidated; } @Override public String getVersion() { return version; } @Override public int getVersionApi() { return versionApi; } @Override public int getVersionCdh() { return versionCdh; } @Override @CmServerCommandMethod(name = "client") public boolean getServiceConfigs(final CmServerCluster cluster, final File directory) throws CmServerException { final AtomicBoolean executed = new AtomicBoolean(false); try { if (isProvisioned(cluster)) { logger.logOperation("GetConfig", new CmServerLogSyncCommand() { @Override public void execute() throws IOException { for (ApiService apiService : apiResourceRootV3.getClustersResource() .getServicesResource(getName(cluster)).readServices(DataView.SUMMARY)) { CmServerServiceType type = CmServerServiceType.valueOfId(apiService.getType()); if (type.equals(CmServerServiceType.HDFS) || type.equals(CmServerServiceType.MAPREDUCE) || type.equals(CmServerServiceType.YARN) || type.equals(CmServerServiceType.HBASE) || versionApi >= 4 && type.equals(CmServerServiceType.HIVE) || versionApi >= 5 && type.equals(CmServerServiceType.SOLR)) { ZipInputStream configInputZip = null; try { InputStreamDataSource configInput = apiResourceRootV3.getClustersResource() .getServicesResource(getName(cluster)) .getClientConfig(apiService.getName()); if (configInput != null) { configInputZip = new ZipInputStream(configInput.getInputStream()); ZipEntry configInputZipEntry = null; while ((configInputZipEntry = configInputZip.getNextEntry()) != null) { String configFile = configInputZipEntry.getName(); if (configFile.contains(File.separator)) { configFile = configFile.substring( configFile.lastIndexOf(File.separator), configFile.length()); } directory.mkdirs(); BufferedWriter configOutput = null; try { int read; configOutput = new BufferedWriter( new FileWriter(new File(directory, configFile))); while (configInputZip.available() > 0) { if ((read = configInputZip.read()) != -1) { configOutput.write(read); } } } finally { configOutput.close(); } } } } finally { if (configInputZip != null) { configInputZip.close(); } } executed.set(true); } } } }); } } catch (Exception e) { throw new CmServerException("Failed to get cluster config", e); } return executed.get(); } @Override public List<CmServerService> getServiceHosts() throws CmServerException { final List<CmServerService> services = new ArrayList<CmServerService>(); try { logger.logOperation("GetHosts", new CmServerLogSyncCommand() { @Override public void execute() { for (ApiHost host : apiResourceRootV3.getHostsResource().readHosts(DataView.SUMMARY) .getHosts()) { services.add(new CmServerServiceBuilder().host(host.getHostId()).ip(host.getIpAddress()) .ipInternal(host.getIpAddress()).status(CmServerServiceStatus.STARTED).build()); } } }); } catch (Exception e) { throw new CmServerException("Failed to list cluster hosts", e); } return services; } @Override public CmServerService getServiceHost(CmServerService service) throws CmServerException { return getServiceHost(service, getServiceHosts()); } @Override public CmServerService getServiceHost(CmServerService service, List<CmServerService> services) throws CmServerException { CmServerService serviceFound = null; try { for (CmServerService serviceTmp : services) { if (service.getHost() != null && service.getHost().equals(serviceTmp.getHost()) || service.getHost() != null && service.getHost().equals(serviceTmp.getIp()) || service.getHost() != null && service.getHost().equals(serviceTmp.getIpInternal()) || service.getIp() != null && service.getIp().equals(serviceTmp.getIp()) || service.getIp() != null && service.getIp().equals(serviceTmp.getIpInternal()) || service.getIpInternal() != null && service.getIpInternal().equals(serviceTmp.getIp()) || service.getIpInternal() != null && service.getIpInternal().equals(serviceTmp.getIpInternal())) { serviceFound = serviceTmp; break; } } } catch (Exception e) { throw new CmServerException("Failed to find service", e); } return serviceFound; } @Override @CmServerCommandMethod(name = "services") public CmServerCluster getServices(final CmServerCluster cluster) throws CmServerException { final CmServerCluster clusterView = new CmServerCluster(); try { clusterView.setServer(cluster.getServer()); List<CmServerService> services = getServiceHosts(); for (CmServerService server : cluster.getAgents()) { if (getServiceHost(server, services) != null) { clusterView.addAgent(getServiceHost(server, services)); } } if (!cluster.isEmpty() && isProvisioned(cluster)) { logger.logOperation("GetServices", new CmServerLogSyncCommand() { @Override public void execute() throws IOException, CmServerException { Map<String, String> ips = new HashMap<String, String>(); for (ApiService apiService : apiResourceRootV3.getClustersResource() .getServicesResource(getName(cluster)).readServices(DataView.SUMMARY)) { for (ApiRole apiRole : apiResourceRootV3.getClustersResource() .getServicesResource(getName(cluster)).getRolesResource(apiService.getName()) .readRoles()) { if (!ips.containsKey(apiRole.getHostRef().getHostId())) { ips.put(apiRole.getHostRef().getHostId(), apiResourceRootV3.getHostsResource() .readHost(apiRole.getHostRef().getHostId()).getIpAddress()); } CmServerServiceStatus status = null; try { status = CmServerServiceStatus.valueOf(apiRole.getRoleState().toString()); } catch (IllegalArgumentException exception) { status = CmServerServiceStatus.UNKNOWN; } try { CmServerService service = new CmServerServiceBuilder().name(apiRole.getName()) .host(apiRole.getHostRef().getHostId()) .ip(ips.get(apiRole.getHostRef().getHostId())) .ipInternal(ips.get(apiRole.getHostRef().getHostId())).status(status) .build(); if (service.getType().isConcrete()) { clusterView.addService(service); } } catch (IllegalArgumentException exception) { // ignore } } } } }); } } catch (Exception e) { throw new CmServerException("Failed to find services", e); } return clusterView; } @Override public CmServerService getService(final CmServerCluster cluster, final CmServerServiceType type) throws CmServerException { return getServices(cluster, type).getService(type, versionApi, versionCdh); } @Override public CmServerCluster getServices(final CmServerCluster cluster, final CmServerServiceType type) throws CmServerException { final CmServerCluster clusterView = new CmServerCluster(); try { for (CmServerService service : getServices(cluster).getServices(CmServerServiceType.CLUSTER, versionApi, versionCdh)) { if (type.equals(CmServerServiceType.CLUSTER) || type.equals(service.getType().getParent()) || type.equals(service.getType())) { clusterView.addService(service); } } } catch (Exception e) { throw new CmServerException("Failed to find services", e); } return clusterView; } @Override public boolean isProvisioned(final CmServerCluster cluster) throws CmServerException { boolean executed = false; try { for (ApiCluster apiCluster : apiResourceRootV3.getClustersResource().readClusters(DataView.SUMMARY)) { if (apiCluster.getName().equals(getName(cluster))) { executed = true; break; } } } catch (Exception e) { throw new CmServerException("Failed to detrermine if cluster is provisioned", e); } return executed; } @Override public boolean isConfigured(final CmServerCluster cluster) throws CmServerException { boolean executed = false; final Set<String> servicesNotConfigured = new HashSet<String>(); try { if (isProvisioned(cluster)) { for (CmServerService service : cluster.getServices(CmServerServiceType.CLUSTER, versionApi, versionCdh)) { servicesNotConfigured.add(service.getName()); } for (ApiService apiService : apiResourceRootV3.getClustersResource() .getServicesResource(getName(cluster)).readServices(DataView.SUMMARY)) { for (ApiRole apiRole : apiResourceRootV3.getClustersResource() .getServicesResource(getName(cluster)).getRolesResource(apiService.getName()) .readRoles()) { servicesNotConfigured.remove(apiRole.getName()); } } executed = true; } } catch (Exception e) { throw new CmServerException("Failed to detrermine if cluster is configured", e); } return executed && servicesNotConfigured.size() == 0; } @Override public boolean isStarted(final CmServerCluster cluster) throws CmServerException { boolean executed = true; final Set<String> servicesNotStarted = new HashSet<String>(); try { if (isConfigured(cluster)) { for (CmServerService service : cluster.getServices(CmServerServiceType.CLUSTER, versionApi, versionCdh)) { servicesNotStarted.add(service.getName()); } for (ApiService apiService : apiResourceRootV3.getClustersResource() .getServicesResource(getName(cluster)).readServices(DataView.SUMMARY)) { for (ApiRole apiRole : apiResourceRootV3.getClustersResource() .getServicesResource(getName(cluster)).getRolesResource(apiService.getName()) .readRoles()) { if (apiRole.getRoleState().equals(ApiRoleState.STARTED)) { servicesNotStarted.remove(apiRole.getName()); } } } } else { executed = false; } } catch (Exception e) { throw new CmServerException("Failed to detrermine if cluster is started", e); } return executed ? servicesNotStarted.size() == 0 : false; } @Override public boolean isStopped(final CmServerCluster cluster) throws CmServerException { final Set<String> servicesNotStopped = new HashSet<String>(); try { if (isConfigured(cluster)) { for (CmServerService service : cluster.getServices(CmServerServiceType.CLUSTER, versionApi, versionCdh)) { servicesNotStopped.add(service.getName()); } for (ApiService apiService : apiResourceRootV3.getClustersResource() .getServicesResource(getName(cluster)).readServices(DataView.SUMMARY)) { for (ApiRole apiRole : apiResourceRootV3.getClustersResource() .getServicesResource(getName(cluster)).getRolesResource(apiService.getName()) .readRoles()) { if (apiRole.getRoleState().equals(ApiRoleState.STOPPED)) { servicesNotStopped.remove(apiRole.getName()); } } } } } catch (Exception e) { throw new CmServerException("Failed to detrermine if cluster is stopped", e); } return servicesNotStopped.size() == 0; } @Override @CmServerCommandMethod(name = "initialise") public boolean initialise(final CmServerCluster cluster) throws CmServerException { boolean executed = false; try { logger.logOperationStartedSync("ClusterInitialise"); Map<String, String> configuration = cluster.getServiceConfiguration(versionApi) .get(CmServerServiceTypeCms.CM.getId()); configuration.remove("cm_database_name"); configuration.remove("cm_database_type"); executed = CmServerServiceTypeCms.CM.getId() != null && provisionCmSettings(configuration).size() >= configuration.size(); logger.logOperationFinishedSync("ClusterInitialise"); } catch (Exception e) { logger.logOperationFailedSync("ClusterInitialise"); throw new CmServerException("Failed to initialise cluster", e); } return executed; } @Override public boolean provision(CmServerCluster cluster) throws CmServerException { boolean executed = false; try { logger.logOperationStartedSync("ClusterProvision"); provisionManagement(cluster); if (!cluster.isEmpty() && !isProvisioned(cluster)) { provsionCluster(cluster); if (cluster.getIsParcel()) { provisionParcels(cluster); } executed = true; } logger.logOperationFinishedSync("ClusterProvision"); } catch (Exception e) { logger.logOperationFailedSync("ClusterProvision"); throw new CmServerException("Failed to provision cluster", e); } return executed; } @Override @CmServerCommandMethod(name = "configure") public boolean configure(CmServerCluster cluster) throws CmServerException { boolean executed = false; try { logger.logOperationStartedSync("ClusterConfigure"); if (!cluster.isEmpty()) { if (!isProvisioned(cluster)) { provision(cluster); } if (!isConfigured(cluster)) { configureServices(cluster); isFirstStartRequired = true; executed = true; } } logger.logOperationFinishedSync("ClusterConfigure"); } catch (Exception e) { logger.logOperationFailedSync("ClusterConfigure"); throw new CmServerException("Failed to configure cluster", e); } return executed; } @Override public boolean start(final CmServerCluster cluster) throws CmServerException { boolean executed = true; try { logger.logOperationStartedSync("ClusterStart"); if (!cluster.isEmpty()) { if (!isConfigured(cluster)) { configure(cluster); } if (!isStarted(cluster)) { for (CmServerServiceType type : cluster.getServiceTypes(versionApi, versionCdh)) { if (isFirstStartRequired) { for (CmServerService service : cluster.getServices(type, versionApi, versionCdh)) { initPreStartServices(cluster, service); } } startService(cluster, type); if (isFirstStartRequired) { for (CmServerService service : cluster.getServices(type, versionApi, versionCdh)) { initPostStartServices(cluster, service); } } } isFirstStartRequired = false; } else { executed = false; } // push into provision phase once OPSAPS-13194/OPSAPS-12870 is addressed startManagement(cluster); } logger.logOperationFinishedSync("ClusterStart"); } catch (Exception e) { logger.logOperationFailedSync("ClusterStart"); throw new CmServerException("Failed to start cluster", e); } return executed; } @Override public boolean stop(final CmServerCluster cluster) throws CmServerException { boolean executed = true; try { logger.logOperationStartedSync("ClusterStop"); if (!cluster.isEmpty()) { if (isConfigured(cluster) && !isStopped(cluster)) { final Set<CmServerServiceType> types = new TreeSet<CmServerServiceType>( Collections.reverseOrder()); types.addAll(cluster.getServiceTypes(versionApi, versionCdh)); for (CmServerServiceType type : types) { stopService(cluster, type); } } else { executed = false; } } logger.logOperationFinishedSync("ClusterStop"); } catch (Exception e) { logger.logOperationFailedSync("ClusterStop"); throw new CmServerException("Failed to stop cluster", e); } return executed; } @Override @CmServerCommandMethod(name = "unconfigure") public boolean unconfigure(final CmServerCluster cluster) throws CmServerException { boolean executed = false; try { logger.logOperationStartedSync("ClusterUnConfigure"); if (!cluster.isEmpty()) { if (isConfigured(cluster)) { if (!isStopped(cluster)) { stop(cluster); } unconfigureServices(cluster); executed = true; } } logger.logOperationFinishedSync("ClusterUnConfigure"); } catch (Exception e) { logger.logOperationFailedSync("ClusterUnConfigure"); throw new CmServerException("Failed to unconfigure cluster", e); } return executed; } @Override @CmServerCommandMethod(name = "unprovision") public boolean unprovision(final CmServerCluster cluster) throws CmServerException { boolean executed = false; try { logger.logOperationStartedSync("ClusterUnProvision"); if (!cluster.isEmpty()) { if (isProvisioned(cluster)) { logger.logOperation("UnProvisionCluster", new CmServerLogSyncCommand() { @Override public void execute() throws IOException { apiResourceRootV3.getClustersResource().deleteCluster(getName(cluster)); } }); executed = true; } } logger.logOperationFinishedSync("ClusterUnProvision"); } catch (Exception e) { logger.logOperationFailedSync("ClusterUnProvision"); throw new CmServerException("Failed to unprovision cluster", e); } return executed; } private String getName(CmServerCluster cluster) { try { return cluster.getServiceName(CmServerServiceType.CLUSTER); } catch (IOException e) { throw new RuntimeException("Could not resolve cluster name", e); } } private Map<String, String> provisionCmSettings(Map<String, String> config) throws InterruptedException { Map<String, String> configPostUpdate = new HashMap<String, String>(); ApiConfigList apiConfigList = new ApiConfigList(); if (config != null && !config.isEmpty()) { for (String key : config.keySet()) { apiConfigList.add(new ApiConfig(key, config.get(key))); } apiResourceRootV3.getClouderaManagerResource().updateConfig(apiConfigList); } apiConfigList = apiResourceRootV3.getClouderaManagerResource().getConfig(DataView.SUMMARY); for (ApiConfig apiConfig : apiConfigList) { configPostUpdate.put(apiConfig.getName(), apiConfig.getValue()); } return configPostUpdate; } private void provisionManagement(final CmServerCluster cluster) throws Exception { boolean cmsProvisionRequired = false; try { try { cmsProvisionRequired = apiResourceRootV3.getClouderaManagerResource().getMgmtServiceResource() .readService(DataView.SUMMARY) == null; } catch (RuntimeException exception) { cmsProvisionRequired = true; } } catch (RuntimeException exception) { // ignore } if (cmsProvisionRequired) { final ApiHostRef cmServerHostRefApi = new ApiHostRef(getServiceHost(host).getHost()); boolean licenseDeployed = false; try { licenseDeployed = apiResourceRootV3.getClouderaManagerResource().readLicense() != null; } catch (Exception e) { // ignore } if (versionApi >= 7 && !licenseDeployed) { apiResourceRootV6.getClouderaManagerResource().beginTrial(); licenseDeployed = true; } final boolean enterpriseDeployed = licenseDeployed; if (versionApi >= 4 || licenseDeployed) { logger.logOperation("CreateManagementServices", new CmServerLogSyncCommand() { @Override public void execute() throws IOException, CmServerException, InterruptedException { ApiService cmsServiceApi = new ApiService(); List<ApiRole> cmsRoleApis = new ArrayList<ApiRole>(); cmsServiceApi.setName(CmServerServiceTypeCms.MANAGEMENT.getName()); cmsServiceApi.setType(CmServerServiceTypeCms.MANAGEMENT.getId()); for (CmServerServiceTypeCms type : CmServerServiceTypeCms.values()) { if (type.getParent() != null && (!type.getEnterprise() || enterpriseDeployed)) { ApiRole cmsRoleApi = new ApiRole(); cmsRoleApi.setName(type.getName()); cmsRoleApi.setType(type.getId()); cmsRoleApi.setHostRef(cmServerHostRefApi); cmsRoleApis.add(cmsRoleApi); } } cmsServiceApi.setRoles(cmsRoleApis); apiResourceRootV3.getClouderaManagerResource().getMgmtServiceResource() .setupCMS(cmsServiceApi); for (ApiRoleConfigGroup cmsRoleConfigGroupApi : apiResourceRootV3 .getClouderaManagerResource().getMgmtServiceResource().getRoleConfigGroupsResource() .readRoleConfigGroups()) { try { CmServerServiceTypeCms type = CmServerServiceTypeCms .valueOf(cmsRoleConfigGroupApi.getRoleType()); if (!type.getEnterprise() || enterpriseDeployed) { ApiRoleConfigGroup cmsRoleConfigGroupApiNew = new ApiRoleConfigGroup(); ApiServiceConfig cmsServiceConfigApi = new ApiServiceConfig(); if (cluster.getServiceConfiguration(versionApi).get(type.getId()) != null) { for (String setting : cluster.getServiceConfiguration(versionApi) .get(type.getId()).keySet()) { cmsServiceConfigApi.add(new ApiConfig(setting, cluster.getServiceConfiguration(versionApi).get(type.getId()) .get(setting))); } } cmsRoleConfigGroupApiNew.setConfig(cmsServiceConfigApi); apiResourceRootV3.getClouderaManagerResource().getMgmtServiceResource() .getRoleConfigGroupsResource() .updateRoleConfigGroup(cmsRoleConfigGroupApi.getName(), cmsRoleConfigGroupApiNew, CM_CONFIG_UPDATE_MESSAGE); } } catch (IllegalArgumentException e) { // ignore } } } }); } } } private void provsionCluster(final CmServerCluster cluster) throws Exception { execute(apiResourceRootV3.getClouderaManagerResource().inspectHostsCommand()); final ApiClusterList clusterList = new ApiClusterList(); ApiCluster apiCluster = new ApiCluster(); apiCluster.setName(getName(cluster)); apiCluster.setVersion(ApiClusterVersion.valueOf(CDH_REPO_PREFIX + versionCdh)); clusterList.add(apiCluster); logger.logOperation("CreateCluster", new CmServerLogSyncCommand() { @Override public void execute() throws IOException { apiResourceRootV3.getClustersResource().createClusters(clusterList); } }); List<ApiHostRef> apiHostRefs = Lists.newArrayList(); for (CmServerService service : getServiceHosts()) { apiHostRefs.add(new ApiHostRef(service.getHost())); } apiResourceRootV3.getClustersResource().addHosts(getName(cluster), new ApiHostRefList(apiHostRefs)); } private void provisionParcels(final CmServerCluster cluster) throws InterruptedException, IOException { apiResourceRootV3.getClouderaManagerResource().updateConfig( new ApiConfigList(Arrays.asList(new ApiConfig[] { new ApiConfig("PARCEL_UPDATE_FREQ", "1") }))); final Set<String> repositoriesRequired = new HashSet<String>(); for (CmServerServiceType type : cluster.getServiceTypes(versionApi, versionCdh)) { repositoriesRequired.add(type.getRepository().toString(CDH_REPO_PREFIX + versionCdh)); } final List<String> repositoriesRequiredOrdered = new ArrayList<String>(); for (String repository : repositoriesRequired) { if (repository.equals(CDH_REPO_PREFIX)) { repositoriesRequiredOrdered.add(0, repository); } else { repositoriesRequiredOrdered.add(repository); } } execute("WaitForParcelsAvailability", new Callback() { @Override public boolean poll() { for (ApiParcel parcel : apiResourceRootV3.getClustersResource().getParcelsResource(getName(cluster)) .readParcels(DataView.FULL).getParcels()) { try { repositoriesRequired.remove(parcel.getProduct()); } catch (IllegalArgumentException e) { // ignore } } return repositoriesRequired.isEmpty(); } }); apiResourceRootV3.getClouderaManagerResource().updateConfig( new ApiConfigList(Arrays.asList(new ApiConfig[] { new ApiConfig("PARCEL_UPDATE_FREQ", "60") }))); for (String repository : repositoriesRequiredOrdered) { DefaultArtifactVersion parcelVersion = null; for (ApiParcel apiParcel : apiResourceRootV3.getClustersResource().getParcelsResource(getName(cluster)) .readParcels(DataView.FULL).getParcels()) { DefaultArtifactVersion parcelVersionTmp = new DefaultArtifactVersion(apiParcel.getVersion()); if (apiParcel.getProduct().equals(repository)) { if (!apiParcel.getProduct().equals(CDH_REPO_PREFIX) || versionCdh == parcelVersionTmp.getMajorVersion()) { if (parcelVersion == null || parcelVersion.compareTo(parcelVersionTmp) < 0) { parcelVersion = new DefaultArtifactVersion(apiParcel.getVersion()); } } } } final ParcelResource apiParcelResource = apiResourceRootV3.getClustersResource() .getParcelsResource(getName(cluster)).getParcelResource(repository, parcelVersion.toString()); execute(apiParcelResource.startDownloadCommand(), new Callback() { @Override public boolean poll() { return apiParcelResource.readParcel().getStage().equals(CM_PARCEL_STAGE_DOWNLOADED); } }, false); execute(apiParcelResource.startDistributionCommand(), new Callback() { @Override public boolean poll() { return apiParcelResource.readParcel().getStage().equals(CM_PARCEL_STAGE_DISTRIBUTED); } }, false); execute(apiParcelResource.activateCommand(), new Callback() { @Override public boolean poll() { return apiParcelResource.readParcel().getStage().equals(CM_PARCEL_STAGE_ACTIVATED); } }, false); } } private void configureServices(final CmServerCluster cluster) throws Exception { final List<CmServerService> services = getServiceHosts(); logger.logOperation("CreateClusterServices", new CmServerLogSyncCommand() { @Override public void execute() throws IOException, InterruptedException, CmServerException { ApiServiceList serviceList = new ApiServiceList(); for (CmServerServiceType type : cluster.getServiceTypes(versionApi, versionCdh)) { ApiService apiService = new ApiService(); List<ApiRole> apiRoles = new ArrayList<ApiRole>(); apiService.setType(type.getId()); apiService.setName(cluster.getServiceName(type)); ApiServiceConfig apiServiceConfig = new ApiServiceConfig(); if (cluster.getServiceConfiguration(versionApi).get(type.getId()) != null) { for (String setting : cluster.getServiceConfiguration(versionApi).get(type.getId()) .keySet()) { apiServiceConfig.add(new ApiConfig(setting, cluster.getServiceConfiguration(versionApi).get(type.getId()).get(setting))); } } Set<CmServerServiceType> serviceTypes = cluster.getServiceTypes(versionApi, versionCdh); switch (type) { case YARN: apiServiceConfig.add( new ApiConfig("hdfs_service", cluster.getServiceName(CmServerServiceType.HDFS))); break; case MAPREDUCE: apiServiceConfig.add( new ApiConfig("hdfs_service", cluster.getServiceName(CmServerServiceType.HDFS))); break; case HBASE: apiServiceConfig.add( new ApiConfig("hdfs_service", cluster.getServiceName(CmServerServiceType.HDFS))); apiServiceConfig.add(new ApiConfig("zookeeper_service", cluster.getServiceName(CmServerServiceType.ZOOKEEPER))); break; case SOLR: apiServiceConfig.add( new ApiConfig("hdfs_service", cluster.getServiceName(CmServerServiceType.HDFS))); apiServiceConfig.add(new ApiConfig("zookeeper_service", cluster.getServiceName(CmServerServiceType.ZOOKEEPER))); break; case SOLR_INDEXER: apiServiceConfig.add( new ApiConfig("hbase_service", cluster.getServiceName(CmServerServiceType.HBASE))); apiServiceConfig.add( new ApiConfig("solr_service", cluster.getServiceName(CmServerServiceType.SOLR))); break; case HUE: apiServiceConfig.add(new ApiConfig("hue_webhdfs", cluster.getServiceName(CmServerServiceType.HDFS_HTTP_FS))); apiServiceConfig.add( new ApiConfig("oozie_service", cluster.getServiceName(CmServerServiceType.OOZIE))); apiServiceConfig.add( new ApiConfig("hive_service", cluster.getServiceName(CmServerServiceType.HIVE))); if (serviceTypes.contains(CmServerServiceType.HBASE)) { apiServiceConfig.add(new ApiConfig("hbase_service", cluster.getServiceName(CmServerServiceType.HBASE))); } if (serviceTypes.contains(CmServerServiceType.IMPALA)) { apiServiceConfig.add(new ApiConfig("impala_service", cluster.getServiceName(CmServerServiceType.IMPALA))); } if (serviceTypes.contains(CmServerServiceType.SOLR)) { apiServiceConfig.add(new ApiConfig("solr_service", cluster.getServiceName(CmServerServiceType.SOLR))); } if (serviceTypes.contains(CmServerServiceType.SQOOP)) { apiServiceConfig.add(new ApiConfig("sqoop_service", cluster.getServiceName(CmServerServiceType.SQOOP))); } if (cluster.getService(CmServerServiceType.HBASE_THRIFT_SERVER) != null) { apiServiceConfig.add(new ApiConfig("hue_hbase_thrift", cluster.getServiceName(CmServerServiceType.HBASE_THRIFT_SERVER))); } break; case SQOOP: apiServiceConfig.add(new ApiConfig("mapreduce_yarn_service", serviceTypes.contains(CmServerServiceType.YARN) ? cluster.getServiceName(CmServerServiceType.YARN) : cluster.getServiceName(CmServerServiceType.MAPREDUCE))); break; case OOZIE: apiServiceConfig.add(new ApiConfig("mapreduce_yarn_service", serviceTypes.contains(CmServerServiceType.YARN) ? cluster.getServiceName(CmServerServiceType.YARN) : cluster.getServiceName(CmServerServiceType.MAPREDUCE))); break; case HIVE: apiServiceConfig.add(new ApiConfig("mapreduce_yarn_service", serviceTypes.contains(CmServerServiceType.YARN) ? cluster.getServiceName(CmServerServiceType.YARN) : cluster.getServiceName(CmServerServiceType.MAPREDUCE))); if (versionApi >= 4) { apiServiceConfig.add(new ApiConfig("zookeeper_service", cluster.getServiceName(CmServerServiceType.ZOOKEEPER))); } break; case IMPALA: apiServiceConfig.add( new ApiConfig("hdfs_service", cluster.getServiceName(CmServerServiceType.HDFS))); apiServiceConfig.add( new ApiConfig("hbase_service", cluster.getServiceName(CmServerServiceType.HBASE))); apiServiceConfig.add( new ApiConfig("hive_service", cluster.getServiceName(CmServerServiceType.HIVE))); break; case FLUME: apiServiceConfig.add( new ApiConfig("hdfs_service", cluster.getServiceName(CmServerServiceType.HDFS))); apiServiceConfig.add( new ApiConfig("hbase_service", cluster.getServiceName(CmServerServiceType.HBASE))); default: break; } apiService.setConfig(apiServiceConfig); for (CmServerService subService : cluster.getServices(type, versionApi, versionCdh)) { if (subService.getType().isValid(versionApi, versionCdh)) { CmServerService subServiceHost = getServiceHost(subService, services); if (subServiceHost == null || subServiceHost.getHost() == null) { throw new CmServerException( "Could not find CM agent host to match [" + subService + "]"); } ApiRole apiRole = new ApiRole(); apiRole.setName(subService.getName()); apiRole.setType(subService.getType().getId()); apiRole.setHostRef(new ApiHostRef(subServiceHost.getHost())); apiRoles.add(apiRole); } } apiService.setRoles(apiRoles); serviceList.add(apiService); } apiResourceRootV3.getClustersResource().getServicesResource(getName(cluster)) .createServices(serviceList); for (CmServerServiceType type : cluster.getServiceTypes(versionApi, versionCdh)) { for (ApiRoleConfigGroup roleConfigGroup : apiResourceRootV3.getClustersResource() .getServicesResource(getName(cluster)) .getRoleConfigGroupsResource(cluster.getServiceName(type)).readRoleConfigGroups()) { ApiConfigList apiConfigList = new ApiConfigList(); CmServerServiceType roleConfigGroupType = null; try { roleConfigGroupType = CmServerServiceType.valueOfId(roleConfigGroup.getRoleType()); } catch (IllegalArgumentException e) { // ignore } if (roleConfigGroupType != null && roleConfigGroupType.equals(CmServerServiceType.GATEWAY)) { try { roleConfigGroupType = CmServerServiceType.valueOfId(new CmServerServiceBuilder() .name(roleConfigGroup.getServiceRef().getServiceName()).build().getType() + "_" + roleConfigGroup.getRoleType()); } catch (IllegalArgumentException e) { // ignore } } if (roleConfigGroupType != null) { Map<String, String> config = cluster.getServiceConfiguration(versionApi) .get(roleConfigGroupType.getId()); if (config != null) { for (String setting : config.keySet()) { apiConfigList.add(new ApiConfig(setting, config.get(setting))); } } ApiRoleConfigGroup apiRoleConfigGroup = new ApiRoleConfigGroup(); apiRoleConfigGroup.setConfig(apiConfigList); apiResourceRootV3.getClustersResource().getServicesResource(getName(cluster)) .getRoleConfigGroupsResource(cluster.getServiceName(type)) .updateRoleConfigGroup(roleConfigGroup.getName(), apiRoleConfigGroup, CM_CONFIG_UPDATE_MESSAGE); } } } } }); // Necessary, since createServices a habit of kicking off async commands (eg ZkAutoInit ) for (CmServerServiceType type : cluster.getServiceTypes(versionApi, versionCdh)) { for (ApiCommand command : apiResourceRootV3.getClustersResource().getServicesResource(getName(cluster)) .listActiveCommands(cluster.getServiceName(type), DataView.SUMMARY)) { CmServerImpl.this.execute(command, false); } } execute(apiResourceRootV3.getClustersResource().deployClientConfig(getName(cluster))); } private void unconfigureServices(final CmServerCluster cluster) throws Exception { final Set<CmServerServiceType> types = new TreeSet<CmServerServiceType>(Collections.reverseOrder()); types.addAll(cluster.getServiceTypes(versionApi, versionCdh)); logger.logOperation("DestroyClusterServices", new CmServerLogSyncCommand() { @Override public void execute() throws IOException { for (final CmServerServiceType type : types) { apiResourceRootV3.getClustersResource().getServicesResource(getName(cluster)) .deleteService(cluster.getServiceName(type)); } } }); } private void initPreStartServices(final CmServerCluster cluster, CmServerService service) throws IOException, InterruptedException { switch (service.getType().getParent()) { case HIVE: execute(apiResourceRootV3.getClustersResource().getServicesResource(getName(cluster)) .createHiveWarehouseCommand(cluster.getServiceName(CmServerServiceType.HIVE))); execute(apiResourceRootV3.getClustersResource().getServicesResource(getName(cluster)) .hiveCreateMetastoreDatabaseTablesCommand(cluster.getServiceName(CmServerServiceType.HIVE)), false); break; case OOZIE: execute(apiResourceRootV3.getClustersResource().getServicesResource(getName(cluster)) .installOozieShareLib(cluster.getServiceName(CmServerServiceType.OOZIE)), false); execute(apiResourceRootV3.getClustersResource().getServicesResource(getName(cluster)) .createOozieDb(cluster.getServiceName(CmServerServiceType.OOZIE)), false); break; case HBASE: execute(apiResourceRootV3.getClustersResource().getServicesResource(getName(cluster)) .createHBaseRootCommand(cluster.getServiceName(CmServerServiceType.HBASE))); case ZOOKEEPER: execute(apiResourceRootV3.getClustersResource().getServicesResource(getName(cluster)) .zooKeeperInitCommand(cluster.getServiceName(CmServerServiceType.ZOOKEEPER)), false); break; case SOLR: if (versionApi >= 4) { execute(apiResourceRootV4.getClustersResource().getServicesResource(getName(cluster)) .initSolrCommand(cluster.getServiceName(CmServerServiceType.SOLR)), false); execute(apiResourceRootV4.getClustersResource().getServicesResource(getName(cluster)) .createSolrHdfsHomeDirCommand(cluster.getServiceName(CmServerServiceType.SOLR))); } break; case SQOOP: if (versionApi >= 4) { execute(apiResourceRootV4.getClustersResource().getServicesResource(getName(cluster)) .createSqoopUserDirCommand(cluster.getServiceName(CmServerServiceType.SQOOP))); } break; default: break; } switch (service.getType()) { case HDFS_NAMENODE: execute(apiResourceRootV3.getClustersResource().getServicesResource(getName(cluster)) .getRoleCommandsResource(cluster.getServiceName(CmServerServiceType.HDFS)).formatCommand( new ApiRoleNameList(ImmutableList.<String>builder().add(service.getName()).build())), false); break; case YARN_RESOURCE_MANAGER: if (versionApi >= 6) { execute(apiResourceRootV6.getClustersResource().getServicesResource(getName(cluster)) .createYarnNodeManagerRemoteAppLogDirCommand( cluster.getServiceName(CmServerServiceType.YARN))); } break; case YARN_JOB_HISTORY: if (versionApi >= 6) { execute(apiResourceRootV6.getClustersResource().getServicesResource(getName(cluster)) .createYarnJobHistoryDirCommand(cluster.getServiceName(CmServerServiceType.YARN))); } break; case HUE_SERVER: execute(apiResourceRootV3.getClustersResource().getServicesResource(getName(cluster)) .getRoleCommandsResource(cluster.getServiceName(CmServerServiceType.HUE)).syncHueDbCommand( new ApiRoleNameList(ImmutableList.<String>builder().add(service.getName()).build())), false); break; default: break; } } private void initPostStartServices(final CmServerCluster cluster, CmServerService service) throws IOException, InterruptedException { switch (service.getType().getParent()) { default: break; } switch (service.getType()) { case HDFS_NAMENODE: ApiRoleNameList formatList = new ApiRoleNameList(); formatList.add(service.getName()); execute(apiResourceRootV3.getClustersResource().getServicesResource(getName(cluster)) .hdfsCreateTmpDir(cluster.getServiceName(CmServerServiceType.HDFS))); break; default: break; } } private void startManagement(final CmServerCluster cluster) throws InterruptedException { try { if (apiResourceRootV3.getClouderaManagerResource().getMgmtServiceResource() .readService(DataView.SUMMARY).getServiceState().equals(ApiServiceState.STOPPED)) { CmServerImpl.this.execute("Start " + CmServerServiceTypeCms.MANAGEMENT.getId().toLowerCase(), apiResourceRootV3.getClouderaManagerResource().getMgmtServiceResource().startCommand()); } } catch (RuntimeException exception) { // ignore } } private void startService(CmServerCluster cluster, CmServerServiceType type) throws InterruptedException, IOException { execute("Start " + type.getId().toLowerCase(), apiResourceRootV3.getClustersResource() .getServicesResource(getName(cluster)).startCommand(cluster.getServiceName(type))); } private void stopService(CmServerCluster cluster, CmServerServiceType type) throws InterruptedException, IOException { execute("Stop " + type.getId().toLowerCase(), apiResourceRootV3.getClustersResource() .getServicesResource(getName(cluster)).stopCommand(cluster.getServiceName(type)), false); } private ApiCommand execute(final ApiBulkCommandList bulkCommand, boolean checkReturn) throws InterruptedException { ApiCommand lastCommand = null; for (ApiCommand command : bulkCommand) { lastCommand = execute(command, checkReturn); } return lastCommand; } private ApiCommand execute(final ApiCommand command) throws InterruptedException { return execute(command, true); } private ApiCommand execute(String label, final ApiCommand command) throws InterruptedException { return execute(label, command, true); } private ApiCommand execute(final ApiCommand command, boolean checkReturn) throws InterruptedException { return execute(command.getName(), command, checkReturn); } private ApiCommand execute(String label, final ApiCommand command, boolean checkReturn) throws InterruptedException { return execute(label, command, new Callback() { @Override public boolean poll() { return apiResourceRootV3.getCommandsResource().readCommand(command.getId()).getEndTime() != null; } }, checkReturn); } private ApiCommand execute(ApiCommand command, Callback callback, boolean checkReturn) throws InterruptedException { return execute(command.getName(), command, callback, checkReturn); } private ApiCommand execute(String label, Callback callback) throws InterruptedException { return execute(label, null, callback, false); } private ApiCommand execute(String label, ApiCommand command, Callback callback, boolean checkReturn) throws InterruptedException { label = WordUtils.capitalize(label.replace("-", " ").replace("_", " ")).replace(" ", ""); logger.logOperationStartedAsync(label); ApiCommand commandReturn = null; int apiPollPeriods = 1; int apiPollPeriodLog = 1; int apiPollPeriodBackoffNumber = API_POLL_PERIOD_BACKOFF_NUMBER; while (true) { if (apiPollPeriods++ % apiPollPeriodLog == 0) { logger.logOperationInProgressAsync(label); if (apiPollPeriodBackoffNumber-- == 0) { apiPollPeriodLog += API_POLL_PERIOD_BACKOFF_INCRAMENT; apiPollPeriodBackoffNumber = API_POLL_PERIOD_BACKOFF_NUMBER; } } if (callback.poll()) { if (checkReturn && command != null && !(commandReturn = apiResourceRootV3.getCommandsResource().readCommand(command.getId())) .getSuccess()) { logger.logOperationFailedAsync(label); throw new RuntimeException("Command [" + command + "] failed [" + commandReturn + "]"); } logger.logOperationFinishedAsync(label); return commandReturn; } Thread.sleep(API_POLL_PERIOD_MS); } } private static abstract class Callback { public abstract boolean poll(); } }