Java tutorial
/* * Copyright 2014 Ricardo Lorenzo<unshakablespirit@gmail.com> * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package services; import akka.actor.ActorRef; import akka.actor.Inbox; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.JsonNodeFactory; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.api.services.compute.model.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Configurable; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Service; import play.libs.Akka; import play.libs.Json; import utils.gce.GoogleComputeEngineClient; import utils.gce.GoogleComputeEngineException; import utils.gce.OperationsCache; import utils.gce.storage.GoogleCloudStorageException; import utils.security.SSHKey; import utils.security.SSHKeyFactory; import utils.security.SSHKeyStore; import javax.inject.Inject; import java.io.File; import java.io.IOException; import java.text.DateFormat; import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.*; /** * Created by ricardolorenzo on 18/07/2014. */ @Service(value = "gce-service") @Configurable public class GoogleComputeEngineService { private static Logger log = LoggerFactory.getLogger(GoogleComputeEngineService.class); private static GoogleAuthenticationService authService; protected static GoogleComputeEngineClient client; private static ConfigurationService configurationService; @Inject void setGoogleService(@Qualifier("gauth-service") GoogleAuthenticationService googleAuthService) { GoogleComputeEngineService.authService = googleAuthService; } @Inject void setConfigurationService(@Qualifier("conf-service") ConfigurationService confService) { GoogleComputeEngineService.configurationService = confService; } private static void checkAuthentication() throws GoogleComputeEngineException { if (client == null) { if (GoogleAuthenticationService.getCredential() == null) { throw new GoogleComputeEngineException("not authenticated on Google"); } client = new GoogleComputeEngineClient(GoogleAuthenticationService.getAuthentication()); } } public static GoogleComputeEngineClient getClient() { return client; } public static JsonNode listDisks() throws GoogleComputeEngineException { checkAuthentication(); ObjectNode disks = Json.newObject(); for (Disk d : client.getDisks()) { ObjectNode disk = Json.newObject().put("name", d.getName()).put("description", d.getDescription()) .put("sizeGb", d.getSizeGb()).put("status", d.getStatus()).put("type", d.getType()); disks.set(String.valueOf(d.getId()), disk); } return Json.newObject().set("disks", disks); } public static JsonNode listDiskTypes() throws GoogleComputeEngineException { checkAuthentication(); ObjectNode diskTypes = Json.newObject(); for (DiskType d : client.getDiskTypes()) { ObjectNode type = Json.newObject().put("name", d.getName()).put("description", d.getDescription()) .put("validDiskSize", d.getValidDiskSize()); diskTypes.set(d.getName(), type); } return Json.newObject().set("diskTypes", diskTypes); } public static JsonNode listInstances(List<String> tags) throws GoogleComputeEngineException { checkAuthentication(); ObjectNode instances = Json.newObject(); for (Instance i : client.getInstances(tags)) { ObjectNode instance = Json.newObject().put("name", i.getName()).put("description", i.getDescription()) .put("type", i.getMachineType()).put("status", i.getStatus()); ObjectNode disks = Json.newObject(); for (AttachedDisk d : i.getDisks()) { disks.set(String.valueOf(d.getIndex()), Json.newObject().put("name", d.getDeviceName()).put("type", d.getType()) .put("mode", d.getMode()).put("boot", d.getBoot()) .put("autodelete", d.getAutoDelete())); } instance.set("disks", disks); ObjectNode interfaces = Json.newObject(); for (NetworkInterface n : i.getNetworkInterfaces()) { interfaces.set(n.getName(), Json.newObject().put("network", n.getNetwork()).put("address", n.getNetworkIP())); } instance.set("network-interfaces", interfaces); instance.set("disks", disks); ArrayNode tagList = new ArrayNode(JsonNodeFactory.instance); if (i.getTags() != null && i.getTags().getItems() != null) { i.getTags().getItems().forEach(tagList::add); } instance.set("tags", tagList); instances.set(i.getId().toString(), instance); } return Json.newObject().set("instances", instances); } public static JsonNode listMachineTypes() throws GoogleComputeEngineException { checkAuthentication(); ObjectNode types = Json.newObject(); for (MachineType m : client.getMachineTypes()) { ObjectNode type = Json.newObject().put("name", m.getName()).put("description", m.getDescription()) .put("cpus", m.getGuestCpus()).put("memoryMb", m.getMemoryMb()); types.set(String.valueOf(m.getId()), type); } return Json.newObject().set("machinetypes", types); } public static JsonNode listImages() throws GoogleComputeEngineException { checkAuthentication(); ObjectNode images = Json.newObject(); for (Image i : client.getImages()) { ObjectNode image = Json.newObject().put("name", i.getName()).put("description", i.getDescription()) .put("status", i.getStatus()); images.set(String.valueOf(i.getId()), image); } return Json.newObject().set("images", images); } public static JsonNode listNetworks() throws GoogleComputeEngineException { checkAuthentication(); ObjectNode networks = Json.newObject(); for (Network n : client.getNetworks()) { ObjectNode network = Json.newObject().put("name", n.getName()).put("description", n.getDescription()) .put("ip4Range", n.getIPv4Range()).put("ip4Gateway", n.getGatewayIPv4()); networks.set(String.valueOf(n.getId()), network); } return Json.newObject().set("networks", networks); } public static void listOperations(ActorRef actor, Date lastOperationDate) throws GoogleComputeEngineException { checkAuthentication(); DateFormat format = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSX"); Inbox inbox = Inbox.create(Akka.system()); if (lastOperationDate != null) { lastOperationDate = new Date(lastOperationDate.getTime() + 1); } OperationsCache.cleanOldOperationsFromCache(); for (Operation o : client.getOperations()) { if (lastOperationDate != null) { if (!OperationsCache.statusHasChanged(o) && !"RUNNING".equalsIgnoreCase(o.getStatus())) { if (o.getStartTime() == null) { continue; } try { Date date = format.parse(o.getStartTime()); if (date.before(lastOperationDate)) { continue; } } catch (ParseException e) { continue; } } } OperationsCache.addOperation(o); inbox.send(actor, o); } } public static JsonNode listZones() throws GoogleComputeEngineException { checkAuthentication(); ObjectNode zones = Json.newObject(); for (Zone z : client.getZones()) { ObjectNode zone = Json.newObject().put("name", z.getName()).put("description", z.getDescription()) .put("region", z.getRegion()).put("status", z.getStatus()); zones.set(String.valueOf(z.getId()), zone); } return Json.newObject().set("zones", zones); } public static void setZoneName(String name) throws GoogleComputeEngineException { checkAuthentication(); client.setZone(name); } public boolean clusterExists() throws GoogleComputeEngineException { String clusterName = ConfigurationService.getClusterName(); StringBuilder instance_name = new StringBuilder(); instance_name.append(clusterName); instance_name.append("-"); instance_name.append(ConfigurationService.NODE_NAME_PUPPET); return client.instanceExists(instance_name.toString()); } public List<String> getInstancesNetworkAddresses(List<String> tags) throws GoogleComputeEngineException { checkAuthentication(); List<String> addresses = new ArrayList<>(); for (Instance instance : client.getInstances(tags)) { for (NetworkInterface i : instance.getNetworkInterfaces()) { addresses.add(i.getNetworkIP()); } } return addresses; } public List<String> getInstancesNames(List<String> tags) throws GoogleComputeEngineException { checkAuthentication(); List<String> names = new ArrayList<>(); for (Instance instance : client.getInstances(tags)) { names.add(instance.getName()); } return names; } public String getClusterPublicAddress() throws GoogleComputeEngineException { checkAuthentication(); String clusterName = ConfigurationService.getClusterName(); StringBuilder instance_name = new StringBuilder(); instance_name.append(clusterName); instance_name.append("-"); instance_name.append(ConfigurationService.NODE_NAME_PUPPET); Instance instance = client.getInstance(instance_name.toString()); if (instance != null) { for (NetworkInterface i : instance.getNetworkInterfaces()) { List<AccessConfig> accessConfigList = i.getAccessConfigs(); if (accessConfigList != null) { for (AccessConfig c : accessConfigList) { return c.getNatIP(); } } } } return null; } public String getClusterNetwork() throws GoogleComputeEngineException { checkAuthentication(); String clusterName = ConfigurationService.getClusterName(); StringBuilder instance_name = new StringBuilder(); instance_name.append(clusterName); instance_name.append("-"); instance_name.append(ConfigurationService.NODE_NAME_PUPPET); Instance instance = client.getInstance(instance_name.toString()); if (instance != null) { for (NetworkInterface i : instance.getNetworkInterfaces()) { return i.getNetwork(); } } return null; } public String getJumpServerPublicAddress() throws GoogleComputeEngineException { checkAuthentication(); String clusterName = ConfigurationService.getClusterName(); StringBuilder instance_name = new StringBuilder(); instance_name.append(clusterName); instance_name.append("-"); instance_name.append(ConfigurationService.NODE_NAME_TEST_JUMP); Instance instance = client.getInstance(instance_name.toString()); if (instance != null) { for (NetworkInterface i : instance.getNetworkInterfaces()) { List<AccessConfig> accessConfigList = i.getAccessConfigs(); if (accessConfigList != null) { for (AccessConfig c : accessConfigList) { return c.getNatIP(); } } } } return null; } public String getNetworkRange(String networkName) throws GoogleComputeEngineException { Network n = client.getNetwork(networkName); return n.getIPv4Range(); } public void createCluster(String clusterName, Integer shards, Integer processes, Integer disksPerShard, String machineType, List<String> network, String sourceImage, String diskType, String diskRaid, String dataFileSystem, Integer dataDiskSizeGb, Integer rootDiskSizeGb) throws GoogleComputeEngineException, GoogleCloudStorageException { List<String> tags; SSHKey sshKey = SSHKeyFactory.generateKey(); StringBuilder machinePrefix = new StringBuilder(); machinePrefix.append("https://www.googleapis.com/compute/v1/projects/"); machinePrefix.append(client.getProjectId()); machinePrefix.append("/zones/"); machinePrefix.append(client.getZone()); machinePrefix.append("/machineTypes/"); if (ConfigurationService.getClusterName() != null) { throw new GoogleComputeEngineException("cluster [" + ConfigurationService.getClusterName() + "] is already created, delete it first to create another"); } if (clusterName == null) { throw new GoogleComputeEngineException("cluster name not defined"); } if (network == null || network.isEmpty()) { throw new GoogleComputeEngineException("network not defined"); } File startupScript = ConfigurationService.getPuppetNodeStartupScriptFile(clusterName); /** * Verify the machine type to get the proper Link */ for (MachineType t : client.getMachineTypes()) { if (machineType != null && machineType.equals(t.getName())) { machineType = t.getSelfLink(); break; } } /** * Verify the disk type to get the proper Link */ for (DiskType t : client.getDiskTypes()) { if (diskType != null && diskType.equals(t.getName())) { diskType = t.getSelfLink(); break; } } /** * Verify the source image to get the proper Link */ for (Image i : client.getImages()) { if (sourceImage != null && sourceImage.equals(i.getName())) { sourceImage = i.getSelfLink(); break; } } /** * Verify the networks to get the proper Link */ if (network != null) { for (Network n : client.getNetworks()) { for (int i = 0; i < network.size(); i++) { if (network.get(i) != null && network.get(i).equals(n.getName())) { network.set(i, n.getSelfLink()); } } } } /** * Overwrites the SSH key */ try { SSHKeyStore store = new SSHKeyStore(); store.addKey(ConfigurationService.CLUSTER_USER, sshKey); } catch (ClassNotFoundException e) { log.info("cannot store cluster ssh key on disk: " + e.getMessage()); throw new GoogleComputeEngineException(e); } catch (IOException e) { log.info("cannot store cluster ssh key on disk: " + e.getMessage()); throw new GoogleComputeEngineException(e); } /** * Create the puppetmaster node */ String instanceName = ConfigurationService.getServerName(clusterName, ConfigurationService.NODE_NAME_PUPPET); tags = Arrays.asList(ConfigurationService.NODE_TAG_PUPPET, clusterName); if (!client.instanceExists(instanceName)) { String networkName = network.get(0); if (networkName.contains("/")) { networkName = networkName.substring(networkName.lastIndexOf("/") + 1); } try { client.createInstance(instanceName, machinePrefix.toString().concat("n1-standard-1"), network, rootDiskSizeGb, sourceImage, null, tags, Arrays.asList(sshKey.getSSHPublicKey(ConfigurationService.CLUSTER_USER)), ConfigurationService.generatePuppetMasterStartupScript(clusterName, networkName, processes, diskRaid, dataFileSystem), true); } catch (IOException e) { throw new GoogleComputeEngineException(e); } } else { log.info("instance [" + instanceName + "] already exists, not created"); } ConfigurationService.setClusterName(clusterName); ConfigurationService.setClusterNodeProcesses(processes); /** * Create the config nodes */ instanceName = ConfigurationService.getServerName(clusterName, ConfigurationService.NODE_NAME_CONF); tags = Arrays.asList(ConfigurationService.NODE_TAG_CONF, clusterName); for (Integer i = 1; i <= 3; i++) { StringBuilder instance_name = new StringBuilder(); instance_name.append(instanceName); instance_name.append("-node-"); instance_name.append(i); if (client.instanceExists(instance_name.toString())) { log.info("instance [" + instance_name.toString() + "] already exists, not created"); continue; } client.createInstance(instance_name.toString(), machinePrefix.toString().concat("n1-standard-1"), network, rootDiskSizeGb, sourceImage, null, tags, Arrays.asList(sshKey.getSSHPublicKey(ConfigurationService.CLUSTER_USER)), startupScript.getAbsolutePath(), false); } /** * Put the cluster node shard creation task in a background thread */ final String t_instanceName = ConfigurationService.getServerName(clusterName, ConfigurationService.NODE_NAME_SHARD); final List<String> t_tags = Arrays.asList(ConfigurationService.NODE_TAG_SHARD, clusterName); final String t_machineType = machineType; final String t_sourceImage = sourceImage; final String t_diskType = diskType; Runnable shardNodesCreation = () -> { /** * Create the shard nodes data disks */ String lastDiskCreated = null; Map<String, Map<String, String>> instancesDataDisks = new HashMap<>(); for (Integer i = 1; i <= shards; i++) { Map<String, String> dataDisks = new HashMap<>(); StringBuilder instance_name = new StringBuilder(); instance_name.append(t_instanceName); instance_name.append("-node-"); instance_name.append(i); try { for (Integer d = 1; d <= disksPerShard; d++) { StringBuilder disk_name = new StringBuilder(); disk_name.append(instance_name.toString()); disk_name.append("-disk"); disk_name.append(d); dataDisks.put(disk_name.toString(), client.createDisk(disk_name.toString(), t_diskType, dataDiskSizeGb)); lastDiskCreated = disk_name.toString(); } instancesDataDisks.put(instance_name.toString(), dataDisks); } catch (GoogleComputeEngineException e) { log.error("cannot create data disks for [" + instance_name.toString() + "], instance not created: " + e.getMessage()); continue; } } /** * Time to get all the disk resources ready to be attached */ try { Thread.sleep(2000); } catch (InterruptedException e) { } /** * Create the shard nodes instances */ try { try { for (Map.Entry<String, Map<String, String>> instance : instancesDataDisks.entrySet()) { if (client.instanceExists(instance.getKey())) { log.info("instance [" + instance.getKey() + "] already exists, not created"); continue; } client.createInstance(instance.getKey(), t_machineType, network, rootDiskSizeGb, t_sourceImage, instance.getValue(), t_tags, Arrays.asList(sshKey.getSSHPublicKey(ConfigurationService.CLUSTER_USER)), startupScript.getAbsolutePath(), false); } log.info("Shard nodes creation finished"); } catch (GoogleComputeEngineException e) { log.error("Shard nodes creation error: " + e.getMessage()); } } finally { startupScript.delete(); } }; Thread thread = new Thread(shardNodesCreation); thread.start(); } public void deleteCluster() throws GoogleComputeEngineException { checkAuthentication(); String clusterName = ConfigurationService.getClusterName(); if (clusterName == null) { throw new GoogleComputeEngineException("no cluster previously created"); } for (Instance i : client.getInstances(Arrays.asList(ConfigurationService.NODE_TAG_SHARD, clusterName))) { client.deleteInstance(i.getName()); } for (Instance i : client.getInstances(Arrays.asList(ConfigurationService.NODE_TAG_CONF, clusterName))) { client.deleteInstance(i.getName()); } for (Instance i : client.getInstances(Arrays.asList(ConfigurationService.NODE_TAG_PUPPET, clusterName))) { client.deleteInstance(i.getName()); } ConfigurationService.setClusterName(null); } public void createTestNodes(Integer testNodes, String machineType, String sourceImage, Integer rootDiskSizeGb) throws GoogleComputeEngineException, GoogleCloudStorageException { List<String> tags; SSHKey sshKey = SSHKeyFactory.generateKey(); StringBuilder machinePrefix = new StringBuilder(); machinePrefix.append("https://www.googleapis.com/compute/v1/projects/"); machinePrefix.append(client.getProjectId()); machinePrefix.append("/zones/"); machinePrefix.append(client.getZone()); machinePrefix.append("/machineTypes/"); String clusterName = ConfigurationService.getClusterName(); if (clusterName == null) { throw new GoogleComputeEngineException( "cluster is not created, you must create a cluster before to create the testing nodes"); } /** * Get the cluster network */ String clusterNetwork = getClusterNetwork(); /** * Verify the machine type to get the proper Link */ for (MachineType t : client.getMachineTypes()) { if (machineType != null && machineType.equals(t.getName())) { machineType = t.getSelfLink(); break; } } /** * Verify the source image to get the proper Link */ for (Image i : client.getImages()) { if (sourceImage != null && sourceImage.equals(i.getName())) { sourceImage = i.getSelfLink(); break; } } /** * Overwrites the SSH key */ try { SSHKeyStore store = new SSHKeyStore(); store.addKey(ConfigurationService.TEST_USER, sshKey); } catch (ClassNotFoundException e) { log.info("cannot store cluster ssh key on disk: " + e.getMessage()); throw new GoogleComputeEngineException(e); } catch (IOException e) { log.info("cannot store cluster ssh key on disk: " + e.getMessage()); throw new GoogleComputeEngineException(e); } /** * Creates the jump server */ String instanceName = ConfigurationService.getServerName(clusterName, ConfigurationService.NODE_NAME_TEST_JUMP); tags = Arrays.asList(ConfigurationService.NODE_TAG_TEST_JUMP, clusterName); if (!client.instanceExists(instanceName)) { String networkName = clusterNetwork; if (networkName.contains("/")) { networkName = networkName.substring(networkName.lastIndexOf("/") + 1); } File startupScript = ConfigurationService.getTestJumpNodeStartupScriptFile(clusterName, networkName); try { client.createInstance(instanceName, machinePrefix.toString().concat("n1-standard-1"), Arrays.asList(clusterNetwork), rootDiskSizeGb, sourceImage, null, tags, Arrays.asList(sshKey.getSSHPublicKey(ConfigurationService.TEST_USER)), startupScript.getAbsolutePath(), true); } finally { startupScript.delete(); } } else { log.info("instance [" + instanceName + "] already exists, not created"); } /** * Put the test node creation task in a background thread */ final File t_startupScript = ConfigurationService.getTestNodeStartupScriptFile(clusterName); final String t_instanceName = ConfigurationService.getServerName(clusterName, ConfigurationService.NODE_NAME_TEST); final List<String> t_tags = Arrays.asList(ConfigurationService.NODE_TAG_TEST, clusterName); final String t_machineType = machineType; final String t_sourceImage = sourceImage; Runnable testNodesCreation = () -> { /** * Time to start and configure the jump server */ try { Thread.sleep(10000); } catch (InterruptedException e) { } try { for (Integer i = 1; i <= testNodes; i++) { StringBuilder instance_name = new StringBuilder(); instance_name.append(t_instanceName); instance_name.append("-node-"); instance_name.append(i); if (client.instanceExists(instance_name.toString())) { log.info("instance [" + instance_name.toString() + "] already exists, not created"); continue; } client.createInstance(instance_name.toString(), t_machineType, Arrays.asList(clusterNetwork), rootDiskSizeGb, t_sourceImage, null, t_tags, Arrays.asList(sshKey.getSSHPublicKey(ConfigurationService.TEST_USER)), t_startupScript.getAbsolutePath(), false); } log.info("Test nodes creation finished"); } catch (GoogleComputeEngineException e) { log.error("Test nodes creation error: " + e.getMessage()); } }; Thread thread = new Thread(testNodesCreation); thread.start(); } public void deleteTestNodes() throws GoogleComputeEngineException { checkAuthentication(); String clusterName = ConfigurationService.getClusterName(); if (clusterName == null) { throw new GoogleComputeEngineException("no cluster previously created"); } for (Instance i : client.getInstances(Arrays.asList(ConfigurationService.NODE_TAG_TEST, clusterName))) { client.deleteInstance(i.getName()); } for (Instance i : client .getInstances(Arrays.asList(ConfigurationService.NODE_TAG_TEST_JUMP, clusterName))) { client.deleteInstance(i.getName()); } } }