Java tutorial
/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.zeppelin.interpreter.remote; import com.google.gson.Gson; import com.google.gson.reflect.TypeToken; import org.apache.commons.lang3.StringUtils; import org.apache.thrift.TException; import org.apache.thrift.protocol.TBinaryProtocol; import org.apache.thrift.protocol.TProtocol; import org.apache.thrift.server.TThreadPoolServer; import org.apache.thrift.transport.TServerSocket; import org.apache.thrift.transport.TSocket; import org.apache.thrift.transport.TTransport; import org.apache.thrift.transport.TTransportException; import org.apache.zeppelin.dep.DependencyResolver; import org.apache.zeppelin.display.AngularObject; import org.apache.zeppelin.display.AngularObjectRegistry; import org.apache.zeppelin.display.GUI; import org.apache.zeppelin.helium.Application; import org.apache.zeppelin.helium.ApplicationContext; import org.apache.zeppelin.helium.ApplicationException; import org.apache.zeppelin.helium.ApplicationLoader; import org.apache.zeppelin.helium.HeliumAppAngularObjectRegistry; import org.apache.zeppelin.helium.HeliumPackage; import org.apache.zeppelin.interpreter.Constants; import org.apache.zeppelin.interpreter.Interpreter; import org.apache.zeppelin.interpreter.InterpreterContext; import org.apache.zeppelin.interpreter.InterpreterException; import org.apache.zeppelin.interpreter.InterpreterGroup; import org.apache.zeppelin.interpreter.InterpreterHookListener; import org.apache.zeppelin.interpreter.InterpreterHookRegistry; import org.apache.zeppelin.interpreter.InterpreterHookRegistry.HookType; import org.apache.zeppelin.interpreter.InterpreterOutput; import org.apache.zeppelin.interpreter.InterpreterOutputListener; import org.apache.zeppelin.interpreter.InterpreterResult; import org.apache.zeppelin.interpreter.InterpreterResult.Code; import org.apache.zeppelin.interpreter.InterpreterResultMessage; import org.apache.zeppelin.interpreter.InterpreterResultMessageOutput; import org.apache.zeppelin.interpreter.LazyOpenInterpreter; import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion; import org.apache.zeppelin.interpreter.thrift.RegisterInfo; import org.apache.zeppelin.interpreter.thrift.RemoteApplicationResult; import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterContext; import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterEventService; import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterResult; import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterResultMessage; import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService; import org.apache.zeppelin.resource.DistributedResourcePool; import org.apache.zeppelin.resource.Resource; import org.apache.zeppelin.resource.ResourcePool; import org.apache.zeppelin.resource.ResourceSet; import org.apache.zeppelin.resource.WellKnownResourceName; import org.apache.zeppelin.scheduler.Job; import org.apache.zeppelin.scheduler.Job.Status; import org.apache.zeppelin.scheduler.JobListener; import org.apache.zeppelin.scheduler.Scheduler; import org.apache.zeppelin.user.AuthenticationInfo; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; import java.lang.reflect.Constructor; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.net.URL; import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.Iterator; import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Properties; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; /** * Entry point for Interpreter process. * Accepting thrift connections from ZeppelinServer. */ public class RemoteInterpreterServer extends Thread implements RemoteInterpreterService.Iface { private static Logger logger = LoggerFactory.getLogger(RemoteInterpreterServer.class); private String interpreterGroupId; private InterpreterGroup interpreterGroup; private AngularObjectRegistry angularObjectRegistry; private InterpreterHookRegistry hookRegistry; private DistributedResourcePool resourcePool; private ApplicationLoader appLoader; private Gson gson = new Gson(); private String intpEventServerHost; private String host; private int port; private TThreadPoolServer server; RemoteInterpreterEventService.Client intpEventServiceClient; RemoteInterpreterEventClient intpEventClient; private DependencyResolver depLoader; private final Map<String, RunningApplication> runningApplications = Collections .synchronizedMap(new HashMap<String, RunningApplication>()); private Map<String, Object> remoteWorksResponsePool; private final long DEFAULT_SHUTDOWN_TIMEOUT = 2000; // Hold information for manual progress update private ConcurrentMap<String, Integer> progressMap = new ConcurrentHashMap<>(); private boolean isTest; public RemoteInterpreterServer(String intpEventServerHost, int intpEventServerPort, String interpreterGroupId, String portRange) throws IOException, TTransportException { this(intpEventServerHost, intpEventServerPort, portRange, interpreterGroupId, false); } public RemoteInterpreterServer(String intpEventServerHost, int intpEventServerPort, String portRange, String interpreterGroupId, boolean isTest) throws TTransportException, IOException { logger.info("Starting remote interpreter server on port {}, intpEventServerAddress: {}:{}", port, intpEventServerHost, intpEventServerPort); if (null != intpEventServerHost) { this.intpEventServerHost = intpEventServerHost; if (!isTest) { TTransport transport = new TSocket(intpEventServerHost, intpEventServerPort); transport.open(); TProtocol protocol = new TBinaryProtocol(transport); intpEventServiceClient = new RemoteInterpreterEventService.Client(protocol); intpEventClient = new RemoteInterpreterEventClient(intpEventServiceClient); } } else { // DevInterpreter this.port = intpEventServerPort; } this.isTest = isTest; this.interpreterGroupId = interpreterGroupId; RemoteInterpreterService.Processor<RemoteInterpreterServer> processor = new RemoteInterpreterService.Processor<>( this); TServerSocket serverTransport; if (null == intpEventServerHost) { // Dev Interpreter serverTransport = new TServerSocket(intpEventServerPort); } else { serverTransport = RemoteInterpreterUtils.createTServerSocket(portRange); this.port = serverTransport.getServerSocket().getLocalPort(); this.host = RemoteInterpreterUtils.findAvailableHostAddress(); logger.info("Launching ThriftServer at " + this.host + ":" + this.port); } server = new TThreadPoolServer(new TThreadPoolServer.Args(serverTransport).processor(processor)); remoteWorksResponsePool = Collections.synchronizedMap(new HashMap<String, Object>()); } @Override public void run() { if (null != intpEventServerHost && !isTest) { new Thread(new Runnable() { boolean interrupted = false; @Override public void run() { while (!interrupted && !server.isServing()) { try { Thread.sleep(1000); } catch (InterruptedException e) { interrupted = true; } } if (!interrupted) { RegisterInfo registerInfo = new RegisterInfo(host, port, interpreterGroupId); try { intpEventServiceClient.registerInterpreterProcess(registerInfo); } catch (TException e) { logger.error("Error while registering interpreter: {}", registerInfo, e); try { shutdown(); } catch (TException e1) { logger.warn("Exception occurs while shutting down", e1); } } } } }).start(); } server.serve(); } @Override public void shutdown() throws TException { logger.info("Shutting down..."); if (interpreterGroup != null) { for (List<Interpreter> session : interpreterGroup.values()) { for (Interpreter interpreter : session) { try { interpreter.close(); } catch (InterpreterException e) { logger.warn("Fail to close interpreter", e); } } } } server.stop(); // server.stop() does not always finish server.serve() loop // sometimes server.serve() is hanging even after server.stop() call. // this case, need to force kill the process long startTime = System.currentTimeMillis(); while (System.currentTimeMillis() - startTime < DEFAULT_SHUTDOWN_TIMEOUT && server.isServing()) { try { Thread.sleep(300); } catch (InterruptedException e) { logger.info("Exception in RemoteInterpreterServer while shutdown, Thread.sleep", e); } } if (server.isServing()) { System.exit(0); } } public int getPort() { return port; } public boolean isRunning() { if (server == null) { return false; } else { return server.isServing(); } } public static void main(String[] args) throws TTransportException, InterruptedException, IOException { String zeppelinServerHost = null; int port = Constants.ZEPPELIN_INTERPRETER_DEFAUlT_PORT; String portRange = ":"; String interpreterGroupId = null; if (args.length > 0) { zeppelinServerHost = args[0]; port = Integer.parseInt(args[1]); interpreterGroupId = args[2]; if (args.length > 3) { portRange = args[3]; } } RemoteInterpreterServer remoteInterpreterServer = new RemoteInterpreterServer(zeppelinServerHost, port, interpreterGroupId, portRange); remoteInterpreterServer.start(); remoteInterpreterServer.join(); System.exit(0); } @Override public void createInterpreter(String interpreterGroupId, String sessionId, String className, Map<String, String> properties, String userName) throws TException { if (interpreterGroup == null) { interpreterGroup = new InterpreterGroup(interpreterGroupId); angularObjectRegistry = new AngularObjectRegistry(interpreterGroup.getId(), intpEventClient); hookRegistry = new InterpreterHookRegistry(); resourcePool = new DistributedResourcePool(interpreterGroup.getId(), intpEventClient); interpreterGroup.setInterpreterHookRegistry(hookRegistry); interpreterGroup.setAngularObjectRegistry(angularObjectRegistry); interpreterGroup.setResourcePool(resourcePool); intpEventClient.setIntpGroupId(interpreterGroupId); String localRepoPath = properties.get("zeppelin.interpreter.localRepo"); if (properties.containsKey("zeppelin.interpreter.output.limit")) { InterpreterOutput.limit = Integer.parseInt(properties.get("zeppelin.interpreter.output.limit")); } depLoader = new DependencyResolver(localRepoPath); appLoader = new ApplicationLoader(resourcePool, depLoader); } try { Class<Interpreter> replClass = (Class<Interpreter>) Object.class.forName(className); Properties p = new Properties(); p.putAll(properties); setSystemProperty(p); Constructor<Interpreter> constructor = replClass.getConstructor(new Class[] { Properties.class }); Interpreter repl = constructor.newInstance(p); repl.setClassloaderUrls(new URL[] {}); logger.info("Instantiate interpreter {}", className); repl.setInterpreterGroup(interpreterGroup); repl.setUserName(userName); interpreterGroup.addInterpreterToSession(new LazyOpenInterpreter(repl), sessionId); } catch (ClassNotFoundException | NoSuchMethodException | SecurityException | InstantiationException | IllegalAccessException | IllegalArgumentException | InvocationTargetException e) { logger.error(e.toString(), e); throw new TException(e); } } protected InterpreterGroup getInterpreterGroup() { return interpreterGroup; } protected ResourcePool getResourcePool() { return resourcePool; } protected RemoteInterpreterEventClient getIntpEventClient() { return intpEventClient; } private void setSystemProperty(Properties properties) { for (Object key : properties.keySet()) { if (!RemoteInterpreterUtils.isEnvString((String) key)) { String value = properties.getProperty((String) key); if (!StringUtils.isBlank(value)) { System.setProperty((String) key, properties.getProperty((String) key)); } } } } protected Interpreter getInterpreter(String sessionId, String className) throws TException { if (interpreterGroup == null) { throw new TException(new InterpreterException("Interpreter instance " + className + " not created")); } synchronized (interpreterGroup) { List<Interpreter> interpreters = interpreterGroup.get(sessionId); if (interpreters == null) { throw new TException(new InterpreterException("Interpreter " + className + " not initialized")); } for (Interpreter inp : interpreters) { if (inp.getClassName().equals(className)) { return inp; } } } throw new TException(new InterpreterException("Interpreter instance " + className + " not found")); } @Override public void open(String sessionId, String className) throws TException { logger.info(String.format("Open Interpreter %s for session %s ", className, sessionId)); Interpreter intp = getInterpreter(sessionId, className); try { intp.open(); } catch (InterpreterException e) { throw new TException("Fail to open interpreter", e); } } @Override public void close(String sessionId, String className) throws TException { // unload all applications for (String appId : runningApplications.keySet()) { RunningApplication appInfo = runningApplications.get(appId); // see NoteInterpreterLoader.SHARED_SESSION if (appInfo.noteId.equals(sessionId) || sessionId.equals("shared_session")) { try { logger.info("Unload App {} ", appInfo.pkg.getName()); appInfo.app.unload(); // see ApplicationState.Status.UNLOADED intpEventClient.onAppStatusUpdate(appInfo.noteId, appInfo.paragraphId, appId, "UNLOADED"); } catch (ApplicationException e) { logger.error(e.getMessage(), e); } } } // close interpreters List<Interpreter> interpreters; synchronized (interpreterGroup) { interpreters = interpreterGroup.get(sessionId); } if (interpreters != null) { Iterator<Interpreter> it = interpreters.iterator(); while (it.hasNext()) { Interpreter inp = it.next(); if (inp.getClassName().equals(className)) { try { inp.close(); } catch (InterpreterException e) { logger.warn("Fail to close interpreter", e); } it.remove(); break; } } } } @Override public RemoteInterpreterResult interpret(String sessionId, String className, String st, RemoteInterpreterContext interpreterContext) throws TException { if (logger.isDebugEnabled()) { logger.debug("st:\n{}", st); } Interpreter intp = getInterpreter(sessionId, className); InterpreterContext context = convert(interpreterContext); context.setInterpreterClassName(intp.getClassName()); Scheduler scheduler = intp.getScheduler(); InterpretJobListener jobListener = new InterpretJobListener(); InterpretJob job = new InterpretJob(interpreterContext.getParagraphId(), "RemoteInterpretJob_" + System.currentTimeMillis(), jobListener, intp, st, context); scheduler.submit(job); while (!job.isTerminated()) { synchronized (jobListener) { try { jobListener.wait(1000); } catch (InterruptedException e) { logger.info("Exception in RemoteInterpreterServer while interpret, jobListener.wait", e); } } } progressMap.remove(interpreterContext.getParagraphId()); InterpreterResult result = (InterpreterResult) job.getReturn(); // in case of job abort in PENDING status, result can be null if (result == null) { result = new InterpreterResult(Code.KEEP_PREVIOUS_RESULT); } return convert(result, context.getConfig(), context.getGui(), context.getNoteGui()); } class InterpretJobListener implements JobListener { @Override public void onProgressUpdate(Job job, int progress) { } @Override public void onStatusChange(Job job, Status before, Status after) { synchronized (this) { notifyAll(); } } } public static class InterpretJob extends Job<InterpreterResult> { private Interpreter interpreter; private String script; private InterpreterContext context; private Map<String, Object> infos; private InterpreterResult results; public InterpretJob(String jobId, String jobName, JobListener listener, Interpreter interpreter, String script, InterpreterContext context) { super(jobId, jobName, listener); this.interpreter = interpreter; this.script = script; this.context = context; } @Override public InterpreterResult getReturn() { return results; } @Override public int progress() { return 0; } @Override public Map<String, Object> info() { if (infos == null) { infos = new HashMap<>(); } return infos; } private void processInterpreterHooks(final String noteId) { InterpreterHookListener hookListener = new InterpreterHookListener() { @Override public void onPreExecute(String script) { String cmdDev = interpreter.getHook(noteId, HookType.PRE_EXEC_DEV.getName()); String cmdUser = interpreter.getHook(noteId, HookType.PRE_EXEC.getName()); // User defined hook should be executed before dev hook List<String> cmds = Arrays.asList(cmdDev, cmdUser); for (String cmd : cmds) { if (cmd != null) { script = cmd + '\n' + script; } } InterpretJob.this.script = script; } @Override public void onPostExecute(String script) { String cmdDev = interpreter.getHook(noteId, HookType.POST_EXEC_DEV.getName()); String cmdUser = interpreter.getHook(noteId, HookType.POST_EXEC.getName()); // User defined hook should be executed after dev hook List<String> cmds = Arrays.asList(cmdUser, cmdDev); for (String cmd : cmds) { if (cmd != null) { script += '\n' + cmd; } } InterpretJob.this.script = script; } }; hookListener.onPreExecute(script); hookListener.onPostExecute(script); } @Override public InterpreterResult jobRun() throws Throwable { ClassLoader currentThreadContextClassloader = Thread.currentThread().getContextClassLoader(); try { InterpreterContext.set(context); InterpreterResult result = null; // Open the interpreter instance prior to calling interpret(). // This is necessary because the earliest we can register a hook // is from within the open() method. LazyOpenInterpreter lazy = (LazyOpenInterpreter) interpreter; if (!lazy.isOpen()) { lazy.open(); result = lazy.executePrecode(context); } if (result == null || result.code() == Code.SUCCESS) { // Add hooks to script from registry. // note scope first, followed by global scope. // Here's the code after hooking: // global_pre_hook // note_pre_hook // script // note_post_hook // global_post_hook processInterpreterHooks(context.getNoteId()); processInterpreterHooks(null); logger.debug("Script after hooks: " + script); result = interpreter.interpret(script, context); } // data from context.out is prepended to InterpreterResult if both defined context.out.flush(); List<InterpreterResultMessage> resultMessages = context.out.toInterpreterResultMessage(); resultMessages.addAll(result.message()); for (InterpreterResultMessage msg : resultMessages) { if (msg.getType() == InterpreterResult.Type.IMG) { logger.debug("InterpreterResultMessage: IMAGE_DATA"); } else { logger.debug("InterpreterResultMessage: " + msg.toString()); } } // put result into resource pool if (resultMessages.size() > 0) { int lastMessageIndex = resultMessages.size() - 1; if (resultMessages.get(lastMessageIndex).getType() == InterpreterResult.Type.TABLE) { context.getResourcePool().put(context.getNoteId(), context.getParagraphId(), WellKnownResourceName.ZeppelinTableResult.toString(), resultMessages.get(lastMessageIndex)); } } return new InterpreterResult(result.code(), resultMessages); } finally { Thread.currentThread().setContextClassLoader(currentThreadContextClassloader); InterpreterContext.remove(); } } @Override protected boolean jobAbort() { return false; } @Override public void setResult(InterpreterResult result) { this.results = result; } } @Override public void cancel(String sessionId, String className, RemoteInterpreterContext interpreterContext) throws TException { logger.info("cancel {} {}", className, interpreterContext.getParagraphId()); Interpreter intp = getInterpreter(sessionId, className); String jobId = interpreterContext.getParagraphId(); Job job = intp.getScheduler().getJob(jobId); if (job != null && job.getStatus() == Status.PENDING) { job.setStatus(Status.ABORT); } else { try { intp.cancel(convert(interpreterContext, null)); } catch (InterpreterException e) { throw new TException("Fail to cancel", e); } } } @Override public int getProgress(String sessionId, String className, RemoteInterpreterContext interpreterContext) throws TException { Integer manuallyProvidedProgress = progressMap.get(interpreterContext.getParagraphId()); if (manuallyProvidedProgress != null) { return manuallyProvidedProgress; } else { Interpreter intp = getInterpreter(sessionId, className); if (intp == null) { throw new TException("No interpreter {} existed for session {}".format(className, sessionId)); } try { return intp.getProgress(convert(interpreterContext, null)); } catch (InterpreterException e) { throw new TException("Fail to getProgress", e); } } } @Override public String getFormType(String sessionId, String className) throws TException { Interpreter intp = getInterpreter(sessionId, className); try { return intp.getFormType().toString(); } catch (InterpreterException e) { throw new TException(e); } } @Override public List<InterpreterCompletion> completion(String sessionId, String className, String buf, int cursor, RemoteInterpreterContext remoteInterpreterContext) throws TException { Interpreter intp = getInterpreter(sessionId, className); try { return intp.completion(buf, cursor, convert(remoteInterpreterContext, null)); } catch (InterpreterException e) { throw new TException("Fail to get completion", e); } } private InterpreterContext convert(RemoteInterpreterContext ric) { return convert(ric, createInterpreterOutput(ric.getNoteId(), ric.getParagraphId())); } private InterpreterContext convert(RemoteInterpreterContext ric, InterpreterOutput output) { return InterpreterContext.builder().setNoteId(ric.getNoteId()).setNoteName(ric.getNoteName()) .setParagraphId(ric.getParagraphId()).setReplName(ric.getReplName()) .setParagraphTitle(ric.getParagraphTitle()).setParagraphText(ric.getParagraphText()) .setLocalProperties(ric.getLocalProperties()) .setAuthenticationInfo(AuthenticationInfo.fromJson(ric.getAuthenticationInfo())) .setGUI(GUI.fromJson(ric.getGui())) .setConfig(gson.fromJson(ric.getConfig(), new TypeToken<Map<String, Object>>() { }.getType())).setNoteGUI(GUI.fromJson(ric.getNoteGui())) .setAngularObjectRegistry(interpreterGroup.getAngularObjectRegistry()) .setResourcePool(interpreterGroup.getResourcePool()).setInterpreterOut(output) .setIntpEventClient(intpEventClient).setProgressMap(progressMap).build(); } protected InterpreterOutput createInterpreterOutput(final String noteId, final String paragraphId) { return new InterpreterOutput(new InterpreterOutputListener() { @Override public void onUpdateAll(InterpreterOutput out) { try { intpEventClient.onInterpreterOutputUpdateAll(noteId, paragraphId, out.toInterpreterResultMessage()); } catch (IOException e) { logger.error(e.getMessage(), e); } } @Override public void onAppend(int index, InterpreterResultMessageOutput out, byte[] line) { String output = new String(line); logger.debug("Output Append: {}", output); intpEventClient.onInterpreterOutputAppend(noteId, paragraphId, index, output); } @Override public void onUpdate(int index, InterpreterResultMessageOutput out) { String output; try { output = new String(out.toByteArray()); logger.debug("Output Update for index {}: {}", index, output); intpEventClient.onInterpreterOutputUpdate(noteId, paragraphId, index, out.getType(), output); } catch (IOException e) { logger.error(e.getMessage(), e); } } }); } private RemoteInterpreterResult convert(InterpreterResult result, Map<String, Object> config, GUI gui, GUI noteGui) { List<RemoteInterpreterResultMessage> msg = new LinkedList<>(); for (InterpreterResultMessage m : result.message()) { msg.add(new RemoteInterpreterResultMessage(m.getType().name(), m.getData())); } return new RemoteInterpreterResult(result.code().name(), msg, gson.toJson(config), gui.toJson(), noteGui.toJson()); } @Override public String getStatus(String sessionId, String jobId) throws TException { if (interpreterGroup == null) { return Status.UNKNOWN.name(); } synchronized (interpreterGroup) { List<Interpreter> interpreters = interpreterGroup.get(sessionId); if (interpreters == null) { logger.info("getStatus:" + Status.UNKNOWN.name()); return Status.UNKNOWN.name(); } for (Interpreter intp : interpreters) { Job job = intp.getScheduler().getJob(jobId); logger.info("job:" + job); if (job != null) { logger.info("getStatus: " + job.getStatus().name()); return job.getStatus().name(); } } } logger.info("getStatus:" + Status.UNKNOWN.name()); return Status.UNKNOWN.name(); } /** * called when object is updated in client (web) side. * * @param name * @param noteId noteId where the update issues * @param paragraphId paragraphId where the update issues * @param object * @throws TException */ @Override public void angularObjectUpdate(String name, String noteId, String paragraphId, String object) throws TException { AngularObjectRegistry registry = interpreterGroup.getAngularObjectRegistry(); // first try local objects AngularObject ao = registry.get(name, noteId, paragraphId); if (ao == null) { logger.debug("Angular object {} not exists", name); return; } if (object == null) { ao.set(null, false); return; } Object oldObject = ao.get(); Object value = null; if (oldObject != null) { // first try with previous object's type try { value = gson.fromJson(object, oldObject.getClass()); ao.set(value, false); return; } catch (Exception e) { // it's not a previous object's type. proceed to treat as a generic type logger.debug(e.getMessage(), e); } } // Generic java object type for json. if (value == null) { try { value = gson.fromJson(object, new TypeToken<Map<String, Object>>() { }.getType()); } catch (Exception e) { // it's not a generic json object, too. okay, proceed to threat as a string type logger.debug(e.getMessage(), e); } } // try string object type at last if (value == null) { value = gson.fromJson(object, String.class); } ao.set(value, false); } /** * When zeppelinserver initiate angular object add. * Dont't need to emit event to zeppelin server */ @Override public void angularObjectAdd(String name, String noteId, String paragraphId, String object) throws TException { AngularObjectRegistry registry = interpreterGroup.getAngularObjectRegistry(); // first try local objects AngularObject ao = registry.get(name, noteId, paragraphId); if (ao != null) { angularObjectUpdate(name, noteId, paragraphId, object); return; } // Generic java object type for json. Object value = null; try { value = gson.fromJson(object, new TypeToken<Map<String, Object>>() { }.getType()); } catch (Exception e) { // it's okay. proceed to treat object as a string logger.debug(e.getMessage(), e); } // try string object type at last if (value == null) { value = gson.fromJson(object, String.class); } registry.add(name, value, noteId, paragraphId, false); } @Override public void angularObjectRemove(String name, String noteId, String paragraphId) throws TException { AngularObjectRegistry registry = interpreterGroup.getAngularObjectRegistry(); registry.remove(name, noteId, paragraphId, false); } @Override public List<String> resourcePoolGetAll() throws TException { logger.debug("Request resourcePoolGetAll from ZeppelinServer"); List<String> result = new LinkedList<>(); if (resourcePool == null) { return result; } ResourceSet resourceSet = resourcePool.getAll(false); for (Resource r : resourceSet) { result.add(r.toJson()); } return result; } @Override public boolean resourceRemove(String noteId, String paragraphId, String resourceName) throws TException { Resource resource = resourcePool.remove(noteId, paragraphId, resourceName); return resource != null; } @Override public ByteBuffer resourceGet(String noteId, String paragraphId, String resourceName) throws TException { logger.debug("Request resourceGet {} from ZeppelinServer", resourceName); Resource resource = resourcePool.get(noteId, paragraphId, resourceName, false); if (resource == null || resource.get() == null || !resource.isSerializable()) { return ByteBuffer.allocate(0); } else { try { return Resource.serializeObject(resource.get()); } catch (IOException e) { logger.error(e.getMessage(), e); return ByteBuffer.allocate(0); } } } @Override public ByteBuffer resourceInvokeMethod(String noteId, String paragraphId, String resourceName, String invokeMessage) { InvokeResourceMethodEventMessage message = InvokeResourceMethodEventMessage.fromJson(invokeMessage); Resource resource = resourcePool.get(noteId, paragraphId, resourceName, false); if (resource == null || resource.get() == null) { return ByteBuffer.allocate(0); } else { try { Object o = resource.get(); Method method = o.getClass().getMethod(message.methodName, message.getParamTypes()); Object ret = method.invoke(o, message.params); if (message.shouldPutResultIntoResourcePool()) { // if return resource name is specified, // then put result into resource pool // and return empty byte buffer resourcePool.put(noteId, paragraphId, message.returnResourceName, ret); return ByteBuffer.allocate(0); } else { // if return resource name is not specified, // then return serialized result ByteBuffer serialized = Resource.serializeObject(ret); if (serialized == null) { return ByteBuffer.allocate(0); } else { return serialized; } } } catch (Exception e) { logger.error(e.getMessage(), e); return ByteBuffer.allocate(0); } } } // /** // * Get payload of resource from remote // * // * @param invokeResourceMethodEventMessage json serialized InvokeResourcemethodEventMessage // * @param object java serialized of the object // * @throws TException // */ // @Override // public void resourceResponseInvokeMethod( // String invokeResourceMethodEventMessage, ByteBuffer object) throws TException { // InvokeResourceMethodEventMessage message = // InvokeResourceMethodEventMessage.fromJson(invokeResourceMethodEventMessage); // // if (message.shouldPutResultIntoResourcePool()) { // Resource resource = resourcePool.get( // message.resourceId.getNoteId(), // message.resourceId.getParagraphId(), // message.returnResourceName, // true); // eventClient.putResponseInvokeMethod(message, resource); // } else { // eventClient.putResponseInvokeMethod(message, object); // } // } @Override public void angularRegistryPush(String registryAsString) throws TException { try { Map<String, Map<String, AngularObject>> deserializedRegistry = gson.fromJson(registryAsString, new TypeToken<Map<String, Map<String, AngularObject>>>() { }.getType()); interpreterGroup.getAngularObjectRegistry().setRegistry(deserializedRegistry); } catch (Exception e) { logger.info("Exception in RemoteInterpreterServer while angularRegistryPush, nolock", e); } } protected InterpreterOutput createAppOutput(final String noteId, final String paragraphId, final String appId) { return new InterpreterOutput(new InterpreterOutputListener() { @Override public void onUpdateAll(InterpreterOutput out) { } @Override public void onAppend(int index, InterpreterResultMessageOutput out, byte[] line) { intpEventClient.onAppOutputAppend(noteId, paragraphId, index, appId, new String(line)); } @Override public void onUpdate(int index, InterpreterResultMessageOutput out) { try { intpEventClient.onAppOutputUpdate(noteId, paragraphId, index, appId, out.getType(), new String(out.toByteArray())); } catch (IOException e) { logger.error(e.getMessage(), e); } } }); } private ApplicationContext getApplicationContext(HeliumPackage packageInfo, String noteId, String paragraphId, String applicationInstanceId) { InterpreterOutput out = createAppOutput(noteId, paragraphId, applicationInstanceId); return new ApplicationContext(noteId, paragraphId, applicationInstanceId, new HeliumAppAngularObjectRegistry(angularObjectRegistry, noteId, applicationInstanceId), out); } @Override public RemoteApplicationResult loadApplication(String applicationInstanceId, String packageInfo, String noteId, String paragraphId) throws TException { if (runningApplications.containsKey(applicationInstanceId)) { logger.warn("Application instance {} is already running"); return new RemoteApplicationResult(true, ""); } HeliumPackage pkgInfo = HeliumPackage.fromJson(packageInfo); ApplicationContext context = getApplicationContext(pkgInfo, noteId, paragraphId, applicationInstanceId); try { Application app = null; logger.info("Loading application {}({}), artifact={}, className={} into note={}, paragraph={}", pkgInfo.getName(), applicationInstanceId, pkgInfo.getArtifact(), pkgInfo.getClassName(), noteId, paragraphId); app = appLoader.load(pkgInfo, context); runningApplications.put(applicationInstanceId, new RunningApplication(pkgInfo, app, noteId, paragraphId)); return new RemoteApplicationResult(true, ""); } catch (Exception e) { logger.error(e.getMessage(), e); return new RemoteApplicationResult(false, e.getMessage()); } } @Override public RemoteApplicationResult unloadApplication(String applicationInstanceId) throws TException { RunningApplication runningApplication = runningApplications.remove(applicationInstanceId); if (runningApplication != null) { try { logger.info("Unloading application {}", applicationInstanceId); runningApplication.app.unload(); } catch (ApplicationException e) { logger.error(e.getMessage(), e); return new RemoteApplicationResult(false, e.getMessage()); } } return new RemoteApplicationResult(true, ""); } @Override public RemoteApplicationResult runApplication(String applicationInstanceId) throws TException { logger.info("run application {}", applicationInstanceId); RunningApplication runningApp = runningApplications.get(applicationInstanceId); if (runningApp == null) { logger.error("Application instance {} not exists", applicationInstanceId); return new RemoteApplicationResult(false, "Application instance does not exists"); } else { ApplicationContext context = runningApp.app.context(); try { context.out.clear(); context.out.setType(InterpreterResult.Type.ANGULAR); ResourceSet resource = appLoader.findRequiredResourceSet(runningApp.pkg.getResources(), context.getNoteId(), context.getParagraphId()); for (Resource res : resource) { System.err.println("Resource " + res.get()); } runningApp.app.run(resource); context.out.flush(); InterpreterResultMessageOutput out = context.out.getOutputAt(0); intpEventClient.onAppOutputUpdate(context.getNoteId(), context.getParagraphId(), 0, applicationInstanceId, out.getType(), new String(out.toByteArray())); return new RemoteApplicationResult(true, ""); } catch (ApplicationException | IOException e) { return new RemoteApplicationResult(false, e.getMessage()); } } } private static class RunningApplication { public final Application app; public final HeliumPackage pkg; public final String noteId; public final String paragraphId; RunningApplication(HeliumPackage pkg, Application app, String noteId, String paragraphId) { this.app = app; this.pkg = pkg; this.noteId = noteId; this.paragraphId = paragraphId; } } ; }