io.hops.hopsworks.api.zeppelin.socket.NotebookServerImpl.java Source code

Java tutorial

Introduction

Here is the source code for io.hops.hopsworks.api.zeppelin.socket.NotebookServerImpl.java

Source

/*
 * Changes to this file committed after and not including commit-id: ccc0d2c5f9a5ac661e60e6eaf138de7889928b8b
 * are released under the following license:
 *
 * This file is part of Hopsworks
 * Copyright (C) 2018, Logical Clocks AB. All rights reserved
 *
 * Hopsworks is free software: you can redistribute it and/or modify it under the terms of
 * the GNU Affero General Public License as published by the Free Software Foundation,
 * either version 3 of the License, or (at your option) any later version.
 *
 * Hopsworks is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
 * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
 * PURPOSE.  See the GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License along with this program.
 * If not, see <https://www.gnu.org/licenses/>.
 *
 * Changes to this file committed before and including commit-id: ccc0d2c5f9a5ac661e60e6eaf138de7889928b8b
 * are released under the following license:
 *
 * Copyright (C) 2013 - 2018, Logical Clocks AB and RISE SICS AB. All rights reserved
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy of this
 * software and associated documentation files (the "Software"), to deal in the Software
 * without restriction, including without limitation the rights to use, copy, modify, merge,
 * publish, distribute, sublicense, and/or sell copies of the Software, and to permit
 * persons to whom the Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in all copies or
 * substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS  OR IMPLIED, INCLUDING
 * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL  THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
 * DAMAGES OR  OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 */

package io.hops.hopsworks.api.zeppelin.socket;

import com.github.eirslett.maven.plugins.frontend.lib.TaskRunnerException;
import com.google.common.base.Strings;
import com.google.common.collect.Queues;
import com.google.common.collect.Sets;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.google.gson.reflect.TypeToken;
import io.hops.hopsworks.api.filter.AllowedProjectRoles;
import io.hops.hopsworks.api.zeppelin.rest.exception.ForbiddenException;
import io.hops.hopsworks.api.zeppelin.server.ZeppelinConfig;
import io.hops.hopsworks.api.zeppelin.server.ZeppelinConfigFactory;
import io.hops.hopsworks.api.zeppelin.types.InterpreterSettingsList;
import io.hops.hopsworks.api.zeppelin.util.InterpreterBindingUtils;
import io.hops.hopsworks.api.zeppelin.util.SecurityUtils;
import io.hops.hopsworks.api.zeppelin.util.ZeppelinResource;
import io.hops.hopsworks.common.dao.certificates.CertsFacade;
import io.hops.hopsworks.common.dao.certificates.ProjectGenericUserCerts;
import io.hops.hopsworks.common.dao.project.Project;
import io.hops.hopsworks.common.dao.project.team.ProjectTeamFacade;
import io.hops.hopsworks.common.dao.user.Users;
import io.hops.hopsworks.common.dao.user.activity.Activity;
import io.hops.hopsworks.common.dao.user.activity.ActivityFacade;
import io.hops.hopsworks.common.hdfs.DistributedFileSystemOps;
import io.hops.hopsworks.common.hdfs.DistributedFsService;
import io.hops.hopsworks.common.security.CertificateMaterializer;
import io.hops.hopsworks.common.security.CertificatesMgmService;
import io.hops.hopsworks.common.util.HopsUtils;
import io.hops.hopsworks.common.util.Settings;
import java.io.IOException;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Queue;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import javax.websocket.CloseReason;
import javax.websocket.Session;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.vfs2.FileSystemException;
import org.apache.zeppelin.conf.ZeppelinConfiguration;
import org.apache.zeppelin.display.AngularObject;
import org.apache.zeppelin.display.AngularObjectRegistry;
import org.apache.zeppelin.display.AngularObjectRegistryListener;
import org.apache.zeppelin.display.Input;
import org.apache.zeppelin.helium.ApplicationEventListener;
import org.apache.zeppelin.helium.HeliumPackage;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.InterpreterContextRunner;
import org.apache.zeppelin.interpreter.InterpreterGroup;
import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.InterpreterResultMessage;
import org.apache.zeppelin.interpreter.InterpreterSetting;
import org.apache.zeppelin.interpreter.remote.RemoteAngularObjectRegistry;
import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcessListener;
import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
import org.apache.zeppelin.notebook.Folder;
import org.apache.zeppelin.notebook.JobListenerFactory;
import org.apache.zeppelin.notebook.Note;
import org.apache.zeppelin.notebook.Notebook;
import org.apache.zeppelin.notebook.NotebookAuthorization;
import org.apache.zeppelin.notebook.NotebookEventListener;
import org.apache.zeppelin.notebook.NotebookImportDeserializer;
import org.apache.zeppelin.notebook.Paragraph;
import org.apache.zeppelin.notebook.ParagraphJobListener;
import org.apache.zeppelin.notebook.repo.NotebookRepo;
import org.apache.zeppelin.notebook.socket.WatcherMessage;
import org.apache.zeppelin.scheduler.Job;
import org.apache.zeppelin.user.AuthenticationInfo;
import org.apache.zeppelin.util.WatcherSecurityKey;
import org.joda.time.DateTime;
import org.joda.time.format.DateTimeFormat;
import org.joda.time.format.DateTimeFormatter;
import org.quartz.SchedulerException;
import org.sonatype.aether.RepositoryException;

public class NotebookServerImpl implements JobListenerFactory, AngularObjectRegistryListener,
        RemoteInterpreterProcessListener, ApplicationEventListener {

    private static final Logger LOG = Logger.getLogger(NotebookServerImpl.class.getName());
    private static Gson gson = new GsonBuilder().setDateFormat("yyyy-MM-dd'T'HH:mm:ssZ")
            .registerTypeAdapter(Date.class, new NotebookImportDeserializer()).setPrettyPrinting()
            .registerTypeAdapterFactory(Input.TypeAdapterFactory).create();

    private final Map<String, List<Session>> noteSocketMap = new HashMap<>();
    private final Queue<Session> connectedSockets = new ConcurrentLinkedQueue<>();
    private final Map<String, Queue<Session>> userConnectedSockets = new ConcurrentHashMap<>();
    /**
     * This is a special endpoint in the notebook websoket, Every connection in this Queue
     * will be able to watch every websocket event, it doesnt need to be listed into the map of
     * noteSocketMap. This can be used to get information about websocket traffic and watch what
     * is going on.
     */
    private static final Queue<Session> watcherSockets = Queues.newConcurrentLinkedQueue();

    private ZeppelinConfig conf;
    private Notebook notebook;
    private Project project;
    private Settings settings;
    private CertsFacade certsFacade;
    private final ProjectTeamFacade projectTeamFacade;
    private final ActivityFacade activityFacade;
    private final CertificatesMgmService certificatesMgmService;

    private String certPwd;

    public NotebookServerImpl(Project project, ZeppelinConfigFactory zeppelin, CertsFacade certsFacade,
            Settings settings, ProjectTeamFacade projectTeamFacade, ActivityFacade activityFacade,
            CertificatesMgmService certificatesMgmService)
            throws IOException, RepositoryException, TaskRunnerException {
        this.project = project;
        this.conf = zeppelin.getZeppelinConfig(project.getName(), this);
        this.notebook = this.conf.getNotebook();
        this.settings = settings;
        this.certsFacade = certsFacade;
        this.projectTeamFacade = projectTeamFacade;
        this.activityFacade = activityFacade;
        this.certificatesMgmService = certificatesMgmService;
    }

    public Notebook notebook() {
        return this.notebook;
    }

    public ZeppelinConfig getConf() {
        return conf;
    }

    private void addConnectionToNote(String noteId, Session socket) {
        synchronized (noteSocketMap) {
            removeConnectionFromAllNote(socket); // make sure a socket relates only a single note.
            List<Session> socketList = noteSocketMap.get(noteId);
            if (socketList == null) {
                socketList = new LinkedList<>();
                noteSocketMap.put(noteId, socketList);
            }
            if (!socketList.contains(socket)) {
                socketList.add(socket);
            }
        }
    }

    private void removeConnectionFromNote(String noteId, Session socket) {
        synchronized (noteSocketMap) {
            List<Session> socketList = noteSocketMap.get(noteId);
            if (socketList != null) {
                socketList.remove(socket);
            }
        }
    }

    private void removeNote(String noteId) {
        synchronized (noteSocketMap) {
            noteSocketMap.remove(noteId);
        }
    }

    public void removeConnectionFromAllNote(Session socket) {
        synchronized (noteSocketMap) {
            Set<String> keys = noteSocketMap.keySet();
            for (String noteId : keys) {
                removeConnectionFromNote(noteId, socket);
            }
        }
    }

    private String getOpenNoteId(Session socket) {
        String id = null;
        synchronized (noteSocketMap) {
            Set<String> keys = noteSocketMap.keySet();
            for (String noteId : keys) {
                List<Session> sockets = noteSocketMap.get(noteId);
                if (sockets.contains(socket)) {
                    id = noteId;
                }
            }
        }

        return id;
    }

    private void broadcast(String noteId, Message m) {
        synchronized (noteSocketMap) {
            broadcastToWatchers(noteId, StringUtils.EMPTY, m);
            List<Session> socketLists = noteSocketMap.get(noteId);
            if (socketLists == null || socketLists.isEmpty()) {
                return;
            }
            LOG.log(Level.FINE, "SEND >> {0}", m.op);
            for (Session conn : socketLists) {
                try {
                    sendMsg(conn, serializeMessage(m));
                } catch (IOException ex) {
                    LOG.log(Level.SEVERE, "Unable to send message " + m, ex);
                }
            }
        }
    }

    private void broadcastToNoteBindedInterpreter(String interpreterGroupId, Message m) {
        Notebook notebook = notebook();
        List<Note> notes = notebook.getAllNotes();
        for (Note note : notes) {
            List<String> ids = notebook.getInterpreterSettingManager().getInterpreters(note.getId());
            for (String id : ids) {
                if (id.equals(interpreterGroupId)) {
                    broadcast(note.getId(), m);
                }
            }
        }
    }

    private void broadcastExcept(String noteId, Message m, Session exclude) {
        synchronized (noteSocketMap) {
            broadcastToWatchers(noteId, StringUtils.EMPTY, m);
            List<Session> socketLists = noteSocketMap.get(noteId);
            if (socketLists == null || socketLists.isEmpty()) {
                return;
            }
            LOG.log(Level.FINE, "SEND >> {0}", m.op);
            for (Session conn : socketLists) {
                if (exclude.equals(conn)) {
                    continue;
                }
                try {
                    sendMsg(conn, serializeMessage(m));
                } catch (IOException ex) {
                    LOG.log(Level.SEVERE, "Unable to send message " + m, ex);
                }
            }
        }
    }

    private void multicastToUser(String user, Message m) {
        if (!userConnectedSockets.containsKey(user)) {
            LOG.log(Level.SEVERE, "Multicasting to user {0} that is not in connections map", user);
            return;
        }

        for (Session conn : userConnectedSockets.get(user)) {
            unicast(m, conn);
        }
    }

    public void unicast(Message m, Session conn) {
        try {
            sendMsg(conn, serializeMessage(m));
        } catch (IOException e) {
            LOG.log(Level.SEVERE, "socket error", e);
        }
        broadcastToWatchers(StringUtils.EMPTY, StringUtils.EMPTY, m);
    }

    public void unicastNoteJobInfo(Session conn, Message fromMessage) throws IOException {
        addConnectionToNote(NotebookServer.JOB_MANAGER_SERVICE.JOB_MANAGER_PAGE.getKey(), conn);
        AuthenticationInfo subject = new AuthenticationInfo(fromMessage.principal);
        List<Map<String, Object>> noteJobs = notebook().getJobListByUnixTime(false, 0, subject);
        Map<String, Object> response = new HashMap<>();

        response.put("lastResponseUnixTime", System.currentTimeMillis());
        response.put("jobs", noteJobs);

        sendMsg(conn, serializeMessage(new Message(Message.OP.LIST_NOTE_JOBS).put("noteJobs", response)));
    }

    public void broadcastUpdateNoteJobInfo(long lastUpdateUnixTime) throws IOException {
        List<Map<String, Object>> noteJobs = new LinkedList<>();
        Notebook notebookObject = notebook();
        List<Map<String, Object>> jobNotes = null;
        if (notebookObject != null) {
            jobNotes = notebook().getJobListByUnixTime(false, lastUpdateUnixTime, null);
            noteJobs = jobNotes == null ? noteJobs : jobNotes;
        }

        Map<String, Object> response = new HashMap<>();
        response.put("lastResponseUnixTime", System.currentTimeMillis());
        response.put("jobs", noteJobs != null ? noteJobs : new LinkedList<>());

        broadcast(NotebookServer.JOB_MANAGER_SERVICE.JOB_MANAGER_PAGE.getKey(),
                new Message(Message.OP.LIST_UPDATE_NOTE_JOBS).put("noteRunningJobs", response));
    }

    public void unsubscribeNoteJobInfo(Session conn) {
        removeConnectionFromNote(NotebookServer.JOB_MANAGER_SERVICE.JOB_MANAGER_PAGE.getKey(), conn);
    }

    public void saveInterpreterBindings(Session conn, Message fromMessage, ZeppelinResource zeppelinResource) {
        String noteId = (String) fromMessage.data.get("noteId");
        try {
            List<String> settingIdList = gson.fromJson(String.valueOf(fromMessage.data.get("selectedSettingIds")),
                    new TypeToken<ArrayList<String>>() {
                    }.getType());
            AuthenticationInfo subject = new AuthenticationInfo(this.project.getName());
            notebook().bindInterpretersToNote(subject.getUser(), noteId, settingIdList);
            broadcastInterpreterBindings(noteId,
                    InterpreterBindingUtils.getInterpreterBindings(notebook(), noteId));
            zeppelinResource.persistToDB(this.project);
        } catch (Exception e) {
            LOG.log(Level.SEVERE, "Error while saving interpreter bindings", e);
        }
    }

    public void getInterpreterBindings(Session conn, Message fromMessage) throws IOException {
        String noteId = (String) fromMessage.data.get("noteId");
        List<InterpreterSettingsList> settingList = InterpreterBindingUtils.getInterpreterBindings(notebook(),
                noteId);
        sendMsg(conn, serializeMessage(
                new Message(Message.OP.INTERPRETER_BINDINGS).put("interpreterBindings", settingList)));
    }

    public List<Map<String, String>> generateNotesInfo(boolean needsReload, AuthenticationInfo subject,
            Set<String> userAndRoles) {

        Notebook notebook = notebook();

        ZeppelinConfiguration conf = notebook.getConf();
        String homescreenNoteId = conf.getString(ZeppelinConfiguration.ConfVars.ZEPPELIN_NOTEBOOK_HOMESCREEN);
        boolean hideHomeScreenNotebookFromList = conf
                .getBoolean(ZeppelinConfiguration.ConfVars.ZEPPELIN_NOTEBOOK_HOMESCREEN_HIDE);

        if (needsReload) {
            try {
                notebook.reloadAllNotes(subject);
            } catch (IOException e) {
                LOG.log(Level.SEVERE, "Fail to reload notes from repository");
            }
        }

        List<Note> notes = notebook.getAllNotes(userAndRoles);
        List<Map<String, String>> notesInfo = new LinkedList<>();
        for (Note note : notes) {
            Map<String, String> info = new HashMap<>();

            if (hideHomeScreenNotebookFromList && note.getId().equals(homescreenNoteId)) {
                continue;
            }

            info.put("id", note.getId());
            info.put("name", note.getName());
            notesInfo.add(info);
        }

        return notesInfo;
    }

    public void broadcastNote(Note note) {
        broadcast(note.getId(), new Message(Message.OP.NOTE).put("note", note));
    }

    public void broadcastInterpreterBindings(String noteId, List settingList) {
        broadcast(noteId, new Message(Message.OP.INTERPRETER_BINDINGS).put("interpreterBindings", settingList));
    }

    public void unicastParagraph(Note note, Paragraph p, String user) {
        if (!note.isPersonalizedMode() || p == null || user == null) {
            return;
        }

        if (!userConnectedSockets.containsKey(user)) {
            LOG.log(Level.WARNING, "Failed to send unicast. user {} that is not in connections map", user);
            return;
        }

        for (Session conn : userConnectedSockets.get(user)) {
            Message m = new Message(Message.OP.PARAGRAPH).put("paragraph", p);
            unicast(m, conn);
        }
    }

    public void broadcastParagraph(Note note, Paragraph p) {
        if (note.isPersonalizedMode()) {
            broadcastParagraphs(p.getUserParagraphMap(), p);
        } else {
            broadcast(note.getId(), new Message(Message.OP.PARAGRAPH).put("paragraph", p));
        }
    }

    public void broadcastParagraphs(Map<String, Paragraph> userParagraphMap, Paragraph defaultParagraph) {
        if (null != userParagraphMap) {
            for (String user : userParagraphMap.keySet()) {
                multicastToUser(user,
                        new Message(Message.OP.PARAGRAPH).put("paragraph", userParagraphMap.get(user)));
            }
        }
    }

    private void broadcastNewParagraph(Note note, Paragraph para) {
        LOG.info("Broadcasting paragraph on run call instead of note.");
        int paraIndex = note.getParagraphs().indexOf(para);
        broadcast(note.getId(),
                new Message(Message.OP.PARAGRAPH_ADDED).put("paragraph", para).put("index", paraIndex));
    }

    public void broadcastNoteList(AuthenticationInfo subject, HashSet userAndRoles) {
        if (subject == null) {
            subject = new AuthenticationInfo(StringUtils.EMPTY);
        }
        //send first to requesting user
        List<Map<String, String>> notesInfo = generateNotesInfo(false, subject, userAndRoles);
        multicastToUser(subject.getUser(), new Message(Message.OP.NOTES_INFO).put("notes", notesInfo));
        //to others afterwards
        broadcastNoteListExcept(notesInfo, subject);
    }

    public void unicastNoteList(Session conn, AuthenticationInfo subject, HashSet<String> userAndRoles) {
        List<Map<String, String>> notesInfo = generateNotesInfo(false, subject, userAndRoles);
        unicast(new Message(Message.OP.NOTES_INFO).put("notes", notesInfo), conn);
    }

    public void broadcastReloadedNoteList(AuthenticationInfo subject, HashSet userAndRoles) {
        if (subject == null) {
            subject = new AuthenticationInfo(StringUtils.EMPTY);
        }

        //reload and reply first to requesting user
        List<Map<String, String>> notesInfo = generateNotesInfo(true, subject, userAndRoles);
        multicastToUser(subject.getUser(), new Message(Message.OP.NOTES_INFO).put("notes", notesInfo));
        //to others afterwards
        broadcastNoteListExcept(notesInfo, subject);
    }

    private void broadcastNoteListExcept(List<Map<String, String>> notesInfo, AuthenticationInfo subject) {
        Set<String> userAndRoles;
        NotebookAuthorization authInfo = NotebookAuthorization.getInstance();
        for (String user : userConnectedSockets.keySet()) {
            if (subject.getUser().equals(user)) {
                continue;
            }
            //reloaded already above; parameter - false
            userAndRoles = authInfo.getRoles(user);
            userAndRoles.add(user);
            notesInfo = generateNotesInfo(false, new AuthenticationInfo(user), userAndRoles);
            multicastToUser(user, new Message(Message.OP.NOTES_INFO).put("notes", notesInfo));
        }
    }

    void permissionError(Session conn, String op, String userName, Set<String> userAndRoles, Set<String> allowed,
            Users user) throws IOException {
        LOG.log(Level.INFO, "Cannot {0}. Connection readers {1}. Allowed readers {2}",
                new Object[] { op, userAndRoles, allowed });
        sendMsg(conn,
                serializeMessage(new Message(Message.OP.AUTH_INFO).put("info",
                        "Insufficient privileges to " + op + "note.\n\n" + "Allowed users or roles: "
                                + allowed.toString() + "\n\n" + "But the user " + user.getLname() + " belongs to: "
                                + userAndRoles.toString())));
    }

    /**
     * @return false if user doesn't have reader permission for this paragraph
     */
    private boolean hasParagraphReaderPermission(Session conn, Notebook notebook, String noteId,
            HashSet<String> userAndRoles, String principal, String op, Users user) throws IOException {

        NotebookAuthorization notebookAuthorization = notebook.getNotebookAuthorization();
        if (!notebookAuthorization.isReader(noteId, userAndRoles)) {
            permissionError(conn, op, principal, userAndRoles, notebookAuthorization.getOwners(noteId), user);
            return false;
        }

        return true;
    }

    /**
     * @return false if user doesn't have writer permission for this paragraph
     */
    private boolean hasParagraphWriterPermission(Session conn, Notebook notebook, String noteId,
            HashSet<String> userAndRoles, String principal, String op, Users user) throws IOException {

        NotebookAuthorization notebookAuthorization = notebook.getNotebookAuthorization();
        if (!notebookAuthorization.isWriter(noteId, userAndRoles)) {
            permissionError(conn, op, principal, userAndRoles, notebookAuthorization.getOwners(noteId), user);
            return false;
        }

        return true;
    }

    /**
     * @return false if user doesn't have owner permission for this paragraph
     */
    private boolean hasParagraphOwnerPermission(Session conn, Notebook notebook, String noteId,
            HashSet<String> userAndRoles, String principal, String op, Users user) throws IOException {

        NotebookAuthorization notebookAuthorization = notebook.getNotebookAuthorization();
        if (!notebookAuthorization.isOwner(noteId, userAndRoles)) {
            permissionError(conn, op, principal, userAndRoles, notebookAuthorization.getOwners(noteId), user);
            return false;
        }

        return true;
    }

    public void sendNote(Session conn, HashSet<String> userAndRoles, Notebook notebook, Message fromMessage,
            Users user, String hdfsUser) throws IOException {

        LOG.log(Level.FINE, "New operation from {0} : {1} : {2}",
                new Object[] { fromMessage.principal, fromMessage.op, fromMessage.get("id") });

        String noteId = (String) fromMessage.get("id");
        if (noteId == null) {
            return;
        }

        Note note = notebook.getNote(noteId);
        if (note != null) {

            if (!hasParagraphReaderPermission(conn, notebook, noteId, userAndRoles, fromMessage.principal, "read",
                    user)) {
                return;
            }

            addConnectionToNote(note.getId(), conn);

            if (note.isPersonalizedMode()) {
                note = note.getUserNote(hdfsUser);
            }
            sendMsg(conn, serializeMessage(new Message(Message.OP.NOTE).put("note", note)));
            sendAllAngularObjects(note, hdfsUser, conn);
        } else {
            sendMsg(conn, serializeMessage(new Message(Message.OP.NOTE).put("note", null)));
        }
    }

    public void sendHomeNote(Session conn, HashSet<String> userAndRoles, Notebook notebook, Message fromMessage,
            Users user) throws IOException {
        String noteId = notebook.getConf().getString(ZeppelinConfiguration.ConfVars.ZEPPELIN_NOTEBOOK_HOMESCREEN);
        String projectName = this.project.getName();
        Note note = null;
        if (noteId != null) {
            note = notebook.getNote(noteId);
        }

        if (note != null) {
            if (!hasParagraphReaderPermission(conn, notebook, noteId, userAndRoles, fromMessage.principal, "read",
                    user)) {
                return;
            }

            addConnectionToNote(note.getId(), conn);
            sendMsg(conn, serializeMessage(new Message(Message.OP.NOTE).put("note", note)));
            sendAllAngularObjects(note, projectName, conn);
        } else {
            removeConnectionFromAllNote(conn);
            sendMsg(conn, serializeMessage(new Message(Message.OP.NOTE).put("note", null)));
        }
    }

    public void updateNote(Session conn, HashSet<String> userAndRoles, Notebook notebook, Message fromMessage,
            Users user) throws SchedulerException, IOException {
        String noteId = (String) fromMessage.get("id");
        String name = (String) fromMessage.get("name");
        Map<String, Object> config = (Map<String, Object>) fromMessage.get("config");
        if (noteId == null) {
            return;
        }
        if (config == null) {
            return;
        }

        if (!hasParagraphWriterPermission(conn, notebook, noteId, userAndRoles, fromMessage.principal, "update",
                user)) {
            return;
        }

        Note note = notebook.getNote(noteId);
        if (note != null) {
            boolean cronUpdated = isCronUpdated(config, note.getConfig());
            note.setName(name);
            note.setConfig(config);
            if (cronUpdated) {
                notebook.refreshCron(note.getId());
            }

            AuthenticationInfo subject = new AuthenticationInfo(this.project.getName());
            note.persist(subject);
            broadcast(note.getId(), new Message(Message.OP.NOTE_UPDATED).put("name", name).put("config", config)
                    .put("info", note.getInfo()));
            broadcastNoteList(subject, userAndRoles);
        }
    }

    public void updatePersonalizedMode(Session conn, HashSet<String> userAndRoles, Notebook notebook,
            Message fromMessage, Users user) throws SchedulerException, IOException {
        String noteId = (String) fromMessage.get("id");
        String personalized = (String) fromMessage.get("personalized");
        boolean isPersonalized = personalized.equals("true") ? true : false;

        if (noteId == null) {
            return;
        }

        if (!hasParagraphOwnerPermission(conn, notebook, noteId, userAndRoles, fromMessage.principal,
                "persoanlized", user)) {
            return;
        }

        Note note = notebook.getNote(noteId);
        if (note != null) {
            note.setPersonalizedMode(isPersonalized);
            AuthenticationInfo subject = new AuthenticationInfo(fromMessage.principal);
            note.persist(subject);
            broadcastNote(note);
        }
    }

    public void renameNote(Session conn, HashSet<String> userAndRoles, Notebook notebook, Message fromMessage,
            Users user) throws SchedulerException, IOException {
        renameNote(conn, userAndRoles, notebook, fromMessage, "rename", user);
    }

    private void renameNote(Session conn, HashSet<String> userAndRoles, Notebook notebook, Message fromMessage,
            String op, Users user) throws SchedulerException, IOException {
        String noteId = (String) fromMessage.get("id");
        String name = (String) fromMessage.get("name");

        if (noteId == null) {
            return;
        }

        if (!hasParagraphOwnerPermission(conn, notebook, noteId, userAndRoles, fromMessage.principal, "rename",
                user)) {
            return;
        }

        Note note = notebook.getNote(noteId);
        if (note != null) {
            note.setName(name);

            AuthenticationInfo subject = new AuthenticationInfo(fromMessage.principal);
            note.persist(subject);
            broadcastNote(note);
            broadcastNoteList(subject, userAndRoles);
        }
    }

    public void renameFolder(Session conn, HashSet<String> userAndRoles, Notebook notebook, Message fromMessage,
            Users user) throws SchedulerException, IOException {
        renameFolder(conn, userAndRoles, notebook, fromMessage, "rename", user);
    }

    private void renameFolder(Session conn, HashSet<String> userAndRoles, Notebook notebook, Message fromMessage,
            String op, Users user) throws SchedulerException, IOException {
        String oldFolderId = (String) fromMessage.get("id");
        String newFolderId = (String) fromMessage.get("name");

        if (oldFolderId == null) {
            return;
        }

        for (Note note : notebook.getNotesUnderFolder(oldFolderId)) {
            String noteId = note.getId();
            if (!hasParagraphOwnerPermission(conn, notebook, noteId, userAndRoles, fromMessage.principal,
                    op + " folder of '" + note.getName() + "'", user)) {
                return;
            }
        }

        Folder oldFolder = notebook.renameFolder(oldFolderId, newFolderId);

        if (oldFolder != null) {
            AuthenticationInfo subject = new AuthenticationInfo(fromMessage.principal);

            List<Note> renamedNotes = oldFolder.getNotesRecursively();
            for (Note note : renamedNotes) {
                note.persist(subject);
                broadcastNote(note);
            }

            broadcastNoteList(subject, userAndRoles);
        }
    }

    private boolean isCronUpdated(Map<String, Object> configA, Map<String, Object> configB) {
        boolean cronUpdated = false;
        if (configA.get("cron") != null && configB.get("cron") != null
                && configA.get("cron").equals(configB.get("cron"))) {
            cronUpdated = true;
        } else if (configA.get("cron") == null && configB.get("cron") == null) {
            cronUpdated = false;
        } else if (configA.get("cron") != null || configB.get("cron") != null) {
            cronUpdated = true;
        }

        return cronUpdated;
    }

    public void createNote(Session conn, HashSet<String> userAndRoles, Notebook notebook, Message message)
            throws IOException {
        AuthenticationInfo subject = new AuthenticationInfo(message.principal);

        try {
            Note note = null;

            String defaultInterpreterId = (String) message.get("defaultInterpreterId");
            if (!StringUtils.isEmpty(defaultInterpreterId)) {
                List<String> interpreterSettingIds = new LinkedList<>();
                interpreterSettingIds.add(defaultInterpreterId);
                for (String interpreterSettingId : notebook.getInterpreterSettingManager()
                        .getDefaultInterpreterSettingList()) {
                    if (!interpreterSettingId.equals(defaultInterpreterId)) {
                        interpreterSettingIds.add(interpreterSettingId);
                    }
                }
                note = notebook.createNote(interpreterSettingIds, subject);
            } else {
                note = notebook.createNote(subject);
            }

            note.addNewParagraph(subject); // it's an empty note. so add one paragraph
            if (message != null) {
                String noteName = (String) message.get("name");
                if (StringUtils.isEmpty(noteName)) {
                    noteName = "Note " + note.getId();
                }
                note.setName(noteName);
            }

            note.persist(subject);
            addConnectionToNote(note.getId(), (Session) conn);
            sendMsg(conn, serializeMessage(new Message(Message.OP.NEW_NOTE).put("note", note)));
        } catch (FileSystemException e) {
            LOG.log(Level.SEVERE, "Exception from createNote", e);
            sendMsg(conn,
                    serializeMessage(new Message(Message.OP.ERROR_INFO).put("info",
                            "Oops! There is something wrong with the notebook file system. "
                                    + "Please check the logs for more details.")));
            return;
        }
        broadcastNoteList(subject, userAndRoles);
    }

    public void removeNote(Session conn, HashSet<String> userAndRoles, Notebook notebook, Message fromMessage,
            Users user) throws IOException {
        String noteId = (String) fromMessage.get("id");
        if (noteId == null) {
            return;
        }

        if (!hasParagraphOwnerPermission(conn, notebook, noteId, userAndRoles, fromMessage.principal, "remove",
                user)) {
            return;
        }

        AuthenticationInfo subject = new AuthenticationInfo(fromMessage.principal);
        notebook.removeNote(noteId, subject);
        removeNote(noteId);
        broadcastNoteList(subject, userAndRoles);
    }

    public void removeFolder(Session conn, HashSet<String> userAndRoles, Notebook notebook, Message fromMessage,
            Users user) throws SchedulerException, IOException {
        String folderId = (String) fromMessage.get("id");
        if (folderId == null) {
            return;
        }

        String hopsworksUserRole = projectTeamFacade.findCurrentRole(project, user);
        if (hopsworksUserRole == null) {
            String errorMsg = "Hopsworks role for user " + user.getUsername() + " should not be null";
            sendMsg(conn, serializeMessage(new Message(Message.OP.ERROR_INFO).put("info", errorMsg)));
            throw new IOException(errorMsg);
        }

        if (!hopsworksUserRole.equals(AllowedProjectRoles.DATA_OWNER)) {
            String errorMsg = "User " + user.getUsername() + " is not a DATA "
                    + "OWNER for this project and is not allowed to empty the Trash";
            sendMsg(conn, serializeMessage(new Message(Message.OP.ERROR_INFO).put("info", errorMsg)));
            throw new IOException(errorMsg);
        }
        List<Note> notes = notebook.getNotesUnderFolder(folderId);
        for (Note note : notes) {
            String noteId = note.getId();

            if (!hasParagraphOwnerPermission(conn, notebook, noteId, userAndRoles, fromMessage.principal,
                    "remove folder of '" + note.getName() + "'", user)) {
                return;
            }
        }

        AuthenticationInfo subject = new AuthenticationInfo(fromMessage.principal);
        for (Note note : notes) {
            notebook.removeNote(note.getId(), subject);
            removeNote(note.getId());
        }
        broadcastNoteList(subject, userAndRoles);
    }

    private void logTrashActivity(String activityMsg, Users user) {
        Activity activity = new Activity();
        Date now = new Date();
        activity.setActivity(activityMsg);
        activity.setFlag(ActivityFacade.FLAG_PROJECT);
        activity.setProject(project);
        activity.setTimestamp(now);
        activity.setUser(user);
        activityFacade.persistActivity(activity);
    }

    public void moveNoteToTrash(Session conn, HashSet<String> userAndRoles, Notebook notebook, Message fromMessage,
            Users user) throws SchedulerException, IOException {
        String noteId = (String) fromMessage.get("id");
        if (noteId == null) {
            return;
        }

        Note note = notebook.getNote(noteId);
        if (note != null && !note.isTrash()) {
            String noteName = note.getName();
            fromMessage.put("name", Folder.TRASH_FOLDER_ID + "/" + note.getName());
            renameNote(conn, userAndRoles, notebook, fromMessage, "move", user);
            notebook.moveNoteToTrash(note.getId());

            logTrashActivity(ActivityFacade.TRASH_NOTEBOOK + "notebook " + noteName, user);
        }
    }

    public void moveFolderToTrash(Session conn, HashSet<String> userAndRoles, Notebook notebook,
            Message fromMessage, Users user) throws SchedulerException, IOException {
        String folderId = (String) fromMessage.get("id");
        if (folderId == null) {
            return;
        }

        Folder folder = notebook.getFolder(folderId);
        if (folder != null && !folder.isTrash()) {
            String trashFolderId = Folder.TRASH_FOLDER_ID + "/" + folderId;
            if (notebook.hasFolder(trashFolderId)) {
                DateTime currentDate = new DateTime();
                DateTimeFormatter formatter = DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss");
                trashFolderId += Folder.TRASH_FOLDER_CONFLICT_INFIX + formatter.print(currentDate);
            }

            fromMessage.put("name", trashFolderId);
            renameFolder(conn, userAndRoles, notebook, fromMessage, "move", user);

            logTrashActivity(ActivityFacade.TRASH_NOTEBOOK + "notebook folder " + folderId + "/", user);
        }
    }

    public void restoreNote(Session conn, HashSet<String> userAndRoles, Notebook notebook, Message fromMessage,
            Users user) throws SchedulerException, IOException {
        String noteId = (String) fromMessage.get("id");

        if (noteId == null) {
            return;
        }

        Note note = notebook.getNote(noteId);
        if (note != null && note.isTrash()) {
            fromMessage.put("name", note.getName().replaceFirst(Folder.TRASH_FOLDER_ID + "/", ""));
            renameNote(conn, userAndRoles, notebook, fromMessage, "restore", user);
        }
    }

    public void restoreFolder(Session conn, HashSet<String> userAndRoles, Notebook notebook, Message fromMessage,
            Users user) throws SchedulerException, IOException {
        String folderId = (String) fromMessage.get("id");

        if (folderId == null) {
            return;
        }

        Folder folder = notebook.getFolder(folderId);
        if (folder != null && folder.isTrash()) {
            String restoreName = folder.getId().replaceFirst(Folder.TRASH_FOLDER_ID + "/", "").trim();

            // if the folder had conflict when it had moved to trash before
            Pattern p = Pattern.compile("\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}$");
            Matcher m = p.matcher(restoreName);
            restoreName = m.replaceAll("").trim();

            fromMessage.put("name", restoreName);
            renameFolder(conn, userAndRoles, notebook, fromMessage, "restore", user);
        }
    }

    public void restoreAll(Session conn, HashSet<String> userAndRoles, Notebook notebook, Message fromMessage,
            Users user) throws SchedulerException, IOException {
        Folder trashFolder = notebook.getFolder(Folder.TRASH_FOLDER_ID);
        if (trashFolder != null) {
            fromMessage.data = new HashMap<>();
            fromMessage.put("id", Folder.TRASH_FOLDER_ID);
            fromMessage.put("name", Folder.ROOT_FOLDER_ID);
            renameFolder(conn, userAndRoles, notebook, fromMessage, "restore trash", user);
        }
    }

    public void emptyTrash(Session conn, HashSet<String> userAndRoles, Notebook notebook, Message fromMessage,
            Users user) throws SchedulerException, IOException {
        fromMessage.data = new HashMap<>();
        fromMessage.put("id", Folder.TRASH_FOLDER_ID);
        removeFolder(conn, userAndRoles, notebook, fromMessage, user);
    }

    public void updateParagraph(Session conn, HashSet<String> userAndRoles, Notebook notebook, Message fromMessage,
            Users user) throws IOException {
        String paragraphId = (String) fromMessage.get("id");
        if (paragraphId == null) {
            return;
        }

        Map<String, Object> params = (Map<String, Object>) fromMessage.get("params");
        Map<String, Object> config = (Map<String, Object>) fromMessage.get("config");
        String noteId = getOpenNoteId(conn);

        if (!hasParagraphWriterPermission(conn, notebook, noteId, userAndRoles, fromMessage.principal, "write",
                user)) {
            return;
        }

        final Note note = notebook.getNote(noteId);
        Paragraph p = note.getParagraph(paragraphId);

        p.settings.setParams(params);
        p.setConfig(config);
        p.setTitle((String) fromMessage.get("title"));
        p.setText((String) fromMessage.get("paragraph"));

        AuthenticationInfo subject = new AuthenticationInfo(fromMessage.principal);
        if (note.isPersonalizedMode()) {
            p = p.getUserParagraph(subject.getUser());
            p.settings.setParams(params);
            p.setConfig(config);
            p.setTitle((String) fromMessage.get("title"));
            p.setText((String) fromMessage.get("paragraph"));
        }

        note.persist(subject);

        if (note.isPersonalizedMode()) {
            Map<String, Paragraph> userParagraphMap = note.getParagraph(paragraphId).getUserParagraphMap();
            broadcastParagraphs(userParagraphMap, p);
        } else {
            broadcastParagraph(note, p);
        }
    }

    public void cloneNote(Session conn, HashSet<String> userAndRoles, Notebook notebook, Message fromMessage)
            throws IOException, CloneNotSupportedException {
        String noteId = getOpenNoteId(conn);
        String name = (String) fromMessage.get("name");
        Note newNote = notebook.cloneNote(noteId, name, new AuthenticationInfo(fromMessage.principal));
        AuthenticationInfo subject = new AuthenticationInfo(fromMessage.principal);
        addConnectionToNote(newNote.getId(), conn);
        sendMsg(conn, serializeMessage(new Message(Message.OP.NEW_NOTE).put("note", newNote)));
        broadcastNoteList(subject, userAndRoles);
    }

    public void clearAllParagraphOutput(Session conn, HashSet<String> userAndRoles, Notebook notebook,
            Message fromMessage, Users user) throws IOException {
        final String noteId = (String) fromMessage.get("id");
        if (StringUtils.isBlank(noteId)) {
            return;
        }

        if (!hasParagraphWriterPermission(conn, notebook, noteId, userAndRoles, fromMessage.principal,
                "clear output", user)) {
            return;
        }

        Note note = notebook.getNote(noteId);
        note.clearAllParagraphOutput();
        broadcastNote(note);
    }

    protected Note importNote(Session conn, HashSet<String> userAndRoles, Notebook notebook, Message fromMessage)
            throws IOException {
        Note note = null;
        if (fromMessage != null) {
            String noteName = (String) ((Map) fromMessage.get("note")).get("name");
            String noteJson = gson.toJson(fromMessage.get("note"));
            AuthenticationInfo subject = null;
            if (fromMessage.principal != null) {
                subject = new AuthenticationInfo(fromMessage.principal);
            } else {
                subject = new AuthenticationInfo("anonymous");
            }
            note = notebook.importNote(noteJson, noteName, subject);
            note.persist(subject);
            broadcastNote(note);
            broadcastNoteList(subject, userAndRoles);
        }
        return note;
    }

    public void removeParagraph(Session conn, HashSet<String> userAndRoles, Notebook notebook, Message fromMessage,
            Users user) throws IOException {
        final String paragraphId = (String) fromMessage.get("id");
        if (paragraphId == null) {
            return;
        }
        String noteId = getOpenNoteId(conn);

        if (!hasParagraphWriterPermission(conn, notebook, noteId, userAndRoles, fromMessage.principal, "write",
                user)) {
            return;
        }

        /**
         * We dont want to remove the last paragraph
         */
        final Note note = notebook.getNote(noteId);
        if (!note.isLastParagraph(paragraphId)) {
            AuthenticationInfo subject = new AuthenticationInfo(fromMessage.principal);
            Paragraph para = note.removeParagraph(subject.getUser(), paragraphId);
            note.persist(subject);
            if (para != null) {
                broadcast(note.getId(), new Message(Message.OP.PARAGRAPH_REMOVED).put("id", para.getId()));
            }
        }
    }

    public void clearParagraphOutput(Session conn, HashSet<String> userAndRoles, Notebook notebook,
            Message fromMessage, Users user, String hdfsUser) throws IOException {
        final String paragraphId = (String) fromMessage.get("id");
        if (paragraphId == null) {
            return;
        }

        String noteId = getOpenNoteId(conn);
        if (!hasParagraphWriterPermission(conn, notebook, noteId, userAndRoles, fromMessage.principal, "write",
                user)) {
            return;
        }

        final Note note = notebook.getNote(noteId);
        if (note.isPersonalizedMode()) {
            Paragraph p = note.clearPersonalizedParagraphOutput(paragraphId, hdfsUser);
            unicastParagraph(note, p, hdfsUser);
        } else {
            note.clearParagraphOutput(paragraphId);
            Paragraph paragraph = note.getParagraph(paragraphId);
            broadcastParagraph(note, paragraph);
        }
    }

    public void completion(Session conn, HashSet<String> userAndRoles, Notebook notebook, Message fromMessage)
            throws IOException {
        String paragraphId = (String) fromMessage.get("id");
        String buffer = (String) fromMessage.get("buf");
        int cursor = (int) Double.parseDouble(fromMessage.get("cursor").toString());
        Message resp = new Message(Message.OP.COMPLETION_LIST).put("id", paragraphId);
        if (paragraphId == null) {
            sendMsg(conn, serializeMessage(resp));
            return;
        }

        final Note note = notebook.getNote(getOpenNoteId(conn));
        List<InterpreterCompletion> candidates = note.completion(paragraphId, buffer, cursor);
        resp.put("completions", candidates);
        sendMsg(conn, serializeMessage(resp));
    }

    /**
     * When angular object updated from client
     *
     * @param conn the web socket.
     * @param notebook the notebook.
     * @param fromMessage the message.
     */
    public void angularObjectUpdated(Session conn, HashSet<String> userAndRoles, Notebook notebook,
            Message fromMessage) {
        String noteId = (String) fromMessage.get("noteId");
        String paragraphId = (String) fromMessage.get("paragraphId");
        String interpreterGroupId = (String) fromMessage.get("interpreterGroupId");
        String varName = (String) fromMessage.get("name");
        Object varValue = fromMessage.get("value");
        String user = this.project.getName();
        AngularObject ao = null;
        boolean global = false;
        // propagate change to (Remote) AngularObjectRegistry
        Note note = notebook.getNote(noteId);
        if (note != null) {
            List<InterpreterSetting> settings = notebook.getInterpreterSettingManager()
                    .getInterpreterSettings(note.getId());
            for (InterpreterSetting setting : settings) {
                if (setting.getInterpreterGroup(user, note.getId()) == null) {
                    continue;
                }
                if (interpreterGroupId.equals(setting.getInterpreterGroup(user, note.getId()).getId())) {
                    AngularObjectRegistry angularObjectRegistry = setting.getInterpreterGroup(user, note.getId())
                            .getAngularObjectRegistry();

                    // first trying to get local registry
                    ao = angularObjectRegistry.get(varName, noteId, paragraphId);
                    if (ao == null) {
                        // then try notebook scope registry
                        ao = angularObjectRegistry.get(varName, noteId, null);
                        if (ao == null) {
                            // then try global scope registry
                            ao = angularObjectRegistry.get(varName, null, null);
                            if (ao == null) {
                                LOG.log(Level.WARNING, "Object {0} is not binded", varName);
                            } else {
                                // path from client -> server
                                ao.set(varValue, false);
                                global = true;
                            }
                        } else {
                            // path from client -> server
                            ao.set(varValue, false);
                            global = false;
                        }
                    } else {
                        ao.set(varValue, false);
                        global = false;
                    }
                    break;
                }
            }
        }

        if (global) { // broadcast change to all web session that uses related
            // interpreter.
            for (Note n : notebook.getAllNotes()) {
                List<InterpreterSetting> settings = notebook.getInterpreterSettingManager()
                        .getInterpreterSettings(note.getId());
                for (InterpreterSetting setting : settings) {
                    if (setting.getInterpreterGroup(user, n.getId()) == null) {
                        continue;
                    }
                    if (interpreterGroupId.equals(setting.getInterpreterGroup(user, n.getId()).getId())) {
                        AngularObjectRegistry angularObjectRegistry = setting.getInterpreterGroup(user, n.getId())
                                .getAngularObjectRegistry();
                        this.broadcastExcept(n.getId(),
                                new Message(Message.OP.ANGULAR_OBJECT_UPDATE).put("angularObject", ao)
                                        .put("interpreterGroupId", interpreterGroupId).put("noteId", n.getId())
                                        .put("paragraphId", ao.getParagraphId()),
                                conn);
                    }
                }
            }
        } else { // broadcast to all web session for the note
            this.broadcastExcept(note.getId(),
                    new Message(Message.OP.ANGULAR_OBJECT_UPDATE).put("angularObject", ao)
                            .put("interpreterGroupId", interpreterGroupId).put("noteId", note.getId())
                            .put("paragraphId", ao.getParagraphId()),
                    conn);
        }
    }

    /**
     * Push the given Angular variable to the target
     * interpreter angular registry given a noteId
     * and a paragraph id
     */
    protected void angularObjectClientBind(Session conn, HashSet<String> userAndRoles, Notebook notebook,
            Message fromMessage) throws Exception {
        String noteId = fromMessage.getType("noteId");
        String varName = fromMessage.getType("name");
        Object varValue = fromMessage.get("value");
        String paragraphId = fromMessage.getType("paragraphId");
        Note note = notebook.getNote(noteId);

        if (paragraphId == null) {
            throw new IllegalArgumentException("target paragraph not specified for " + "angular value bind");
        }

        if (note != null) {
            final InterpreterGroup interpreterGroup = findInterpreterGroupForParagraph(note, paragraphId);

            final AngularObjectRegistry registry = interpreterGroup.getAngularObjectRegistry();
            if (registry instanceof RemoteAngularObjectRegistry) {

                RemoteAngularObjectRegistry remoteRegistry = (RemoteAngularObjectRegistry) registry;
                pushAngularObjectToRemoteRegistry(noteId, paragraphId, varName, varValue, remoteRegistry,
                        interpreterGroup.getId(), conn);

            } else {
                pushAngularObjectToLocalRepo(noteId, paragraphId, varName, varValue, registry,
                        interpreterGroup.getId(), conn);
            }
        }
    }

    /**
     * Remove the given Angular variable to the target
     * interpreter(s) angular registry given a noteId
     * and an optional list of paragraph id(s)
     */
    protected void angularObjectClientUnbind(Session conn, HashSet<String> userAndRoles, Notebook notebook,
            Message fromMessage) throws Exception {
        String noteId = fromMessage.getType("noteId");
        String varName = fromMessage.getType("name");
        String paragraphId = fromMessage.getType("paragraphId");
        Note note = notebook.getNote(noteId);

        if (paragraphId == null) {
            throw new IllegalArgumentException("target paragraph not specified for " + "angular value unBind");
        }

        if (note != null) {
            final InterpreterGroup interpreterGroup = findInterpreterGroupForParagraph(note, paragraphId);

            final AngularObjectRegistry registry = interpreterGroup.getAngularObjectRegistry();

            if (registry instanceof RemoteAngularObjectRegistry) {
                RemoteAngularObjectRegistry remoteRegistry = (RemoteAngularObjectRegistry) registry;
                removeAngularFromRemoteRegistry(noteId, paragraphId, varName, remoteRegistry,
                        interpreterGroup.getId(), conn);
            } else {
                removeAngularObjectFromLocalRepo(noteId, paragraphId, varName, registry, interpreterGroup.getId(),
                        conn);
            }
        }
    }

    private InterpreterGroup findInterpreterGroupForParagraph(Note note, String paragraphId) throws Exception {
        final Paragraph paragraph = note.getParagraph(paragraphId);
        if (paragraph == null) {
            throw new IllegalArgumentException("Unknown paragraph with id : " + paragraphId);
        }
        return paragraph.getCurrentRepl().getInterpreterGroup();
    }

    private void pushAngularObjectToRemoteRegistry(String noteId, String paragraphId, String varName,
            Object varValue, RemoteAngularObjectRegistry remoteRegistry, String interpreterGroupId, Session conn) {

        final AngularObject ao = remoteRegistry.addAndNotifyRemoteProcess(varName, varValue, noteId, paragraphId);

        this.broadcastExcept(noteId,
                new Message(Message.OP.ANGULAR_OBJECT_UPDATE).put("angularObject", ao)
                        .put("interpreterGroupId", interpreterGroupId).put("noteId", noteId)
                        .put("paragraphId", paragraphId),
                conn);
    }

    private void removeAngularFromRemoteRegistry(String noteId, String paragraphId, String varName,
            RemoteAngularObjectRegistry remoteRegistry, String interpreterGroupId, Session conn) {
        final AngularObject ao = remoteRegistry.removeAndNotifyRemoteProcess(varName, noteId, paragraphId);
        this.broadcastExcept(noteId,
                new Message(Message.OP.ANGULAR_OBJECT_REMOVE).put("angularObject", ao)
                        .put("interpreterGroupId", interpreterGroupId).put("noteId", noteId)
                        .put("paragraphId", paragraphId),
                conn);
    }

    private void pushAngularObjectToLocalRepo(String noteId, String paragraphId, String varName, Object varValue,
            AngularObjectRegistry registry, String interpreterGroupId, Session conn) {
        AngularObject angularObject = registry.get(varName, noteId, paragraphId);
        if (angularObject == null) {
            angularObject = registry.add(varName, varValue, noteId, paragraphId);
        } else {
            angularObject.set(varValue, true);
        }

        this.broadcastExcept(noteId,
                new Message(Message.OP.ANGULAR_OBJECT_UPDATE).put("angularObject", angularObject)
                        .put("interpreterGroupId", interpreterGroupId).put("noteId", noteId)
                        .put("paragraphId", paragraphId),
                conn);
    }

    private void removeAngularObjectFromLocalRepo(String noteId, String paragraphId, String varName,
            AngularObjectRegistry registry, String interpreterGroupId, Session conn) {
        final AngularObject removed = registry.remove(varName, noteId, paragraphId);
        if (removed != null) {
            this.broadcastExcept(noteId,
                    new Message(Message.OP.ANGULAR_OBJECT_REMOVE).put("angularObject", removed)
                            .put("interpreterGroupId", interpreterGroupId).put("noteId", noteId)
                            .put("paragraphId", paragraphId),
                    conn);
        }
    }

    public void moveParagraph(Session conn, HashSet<String> userAndRoles, Notebook notebook, Message fromMessage,
            Users user) throws IOException {
        final String paragraphId = (String) fromMessage.get("id");
        if (paragraphId == null) {
            return;
        }

        final int newIndex = (int) Double.parseDouble(fromMessage.get("index").toString());
        String noteId = getOpenNoteId(conn);
        final Note note = notebook.getNote(noteId);

        if (!hasParagraphWriterPermission(conn, notebook, noteId, userAndRoles, fromMessage.principal, "write",
                user)) {
            return;
        }

        AuthenticationInfo subject = new AuthenticationInfo(fromMessage.principal);
        note.moveParagraph(paragraphId, newIndex);
        note.persist(subject);
        broadcast(note.getId(),
                new Message(Message.OP.PARAGRAPH_MOVED).put("id", paragraphId).put("index", newIndex));
    }

    public String insertParagraph(Session conn, HashSet<String> userAndRoles, Notebook notebook,
            Message fromMessage, Users user) throws IOException {
        final int index = (int) Double.parseDouble(fromMessage.get("index").toString());
        String noteId = getOpenNoteId(conn);
        final Note note = notebook.getNote(noteId);
        AuthenticationInfo subject = new AuthenticationInfo(fromMessage.principal);

        if (!hasParagraphWriterPermission(conn, notebook, noteId, userAndRoles, fromMessage.principal, "write",
                user)) {
            return null;
        }
        Map<String, Object> config;
        if (fromMessage.get("config") != null) {
            config = (Map<String, Object>) fromMessage.get("config");
        } else {
            config = new HashMap<>();
        }

        Paragraph newPara = note.insertNewParagraph(index, subject);
        newPara.setConfig(config);
        note.persist(subject);
        broadcastNewParagraph(note, newPara);

        return newPara.getId();
    }

    public void copyParagraph(Session conn, HashSet<String> userAndRoles, Notebook notebook, Message fromMessage,
            Users user) throws IOException {
        String newParaId = insertParagraph(conn, userAndRoles, notebook, fromMessage, user);

        if (newParaId == null) {
            return;
        }
        fromMessage.put("id", newParaId);

        updateParagraph(conn, userAndRoles, notebook, fromMessage, user);
    }

    public void cancelParagraph(Session conn, HashSet<String> userAndRoles, Notebook notebook, Message fromMessage,
            Users user) throws IOException {
        final String paragraphId = (String) fromMessage.get("id");
        if (paragraphId == null) {
            return;
        }

        String noteId = getOpenNoteId(conn);

        if (!hasParagraphWriterPermission(conn, notebook, noteId, userAndRoles, fromMessage.principal, "write",
                user)) {
            return;
        }

        final Note note = notebook.getNote(noteId);
        Paragraph p = note.getParagraph(paragraphId);
        p.abort();
    }

    public void runAllParagraphs(Session conn, HashSet<String> userAndRoles, Notebook notebook, Message fromMessage,
            Users user, CertificateMaterializer certificateMaterializer, Settings settings,
            DistributedFsService dfsService) throws IOException {
        final String noteId = (String) fromMessage.get("noteId");
        if (StringUtils.isBlank(noteId)) {
            return;
        }

        if (!hasParagraphWriterPermission(conn, notebook, noteId, userAndRoles, fromMessage.principal,
                "run all paragraphs", user)) {
            return;
        }

        List<Map<String, Object>> paragraphs = gson.fromJson(String.valueOf(fromMessage.data.get("paragraphs")),
                new TypeToken<List<Map<String, Object>>>() {
                }.getType());

        for (Map<String, Object> raw : paragraphs) {
            String paragraphId = (String) raw.get("id");
            if (paragraphId == null) {
                continue;
            }

            String text = (String) raw.get("paragraph");
            String title = (String) raw.get("title");
            Map<String, Object> params = (Map<String, Object>) raw.get("params");
            Map<String, Object> config = (Map<String, Object>) raw.get("config");

            Note note = notebook.getNote(noteId);
            Paragraph p = setParagraphUsingMessage(note, fromMessage, paragraphId, text, title, params, config);

            // Hack: in case the interpreter is hopshive, put the keystore/truststore password in the ticket
            // so that it can be retrieved from the zeppelin-hopshive interpreter
            if ((p.getRequiredReplName() != null && p.getRequiredReplName().equals("hopshive"))
                    || (p.getRequiredReplName() == null && notebook().getInterpreterSettingManager()
                            .getDefaultInterpreterSetting(note.getId()).getName().equals("hopshive"))) {
                setCertificatePassword(p, fromMessage);
            }

            persistAndExecuteSingleParagraph(conn, note, p, certificateMaterializer, settings, dfsService);
        }
    }

    public void broadcastSpellExecution(Session conn, HashSet<String> userAndRoles, Notebook notebook,
            Message fromMessage, Users user) throws IOException {

        final String paragraphId = (String) fromMessage.get("id");
        if (paragraphId == null) {
            return;
        }

        String noteId = getOpenNoteId(conn);

        if (!hasParagraphWriterPermission(conn, notebook, noteId, userAndRoles, fromMessage.principal, "write",
                user)) {
            return;
        }

        String text = (String) fromMessage.get("paragraph");
        String title = (String) fromMessage.get("title");
        Job.Status status = Job.Status.valueOf((String) fromMessage.get("status"));
        Map<String, Object> params = (Map<String, Object>) fromMessage.get("params");
        Map<String, Object> config = (Map<String, Object>) fromMessage.get("config");

        final Note note = notebook.getNote(noteId);
        Paragraph p = setParagraphUsingMessage(note, fromMessage, paragraphId, text, title, params, config);
        p.setResult(fromMessage.get("results"));
        p.setErrorMessage((String) fromMessage.get("errorMessage"));
        p.setStatusWithoutNotification(status);

        // Spell uses ISO 8601 formatted string generated from moment
        String dateStarted = (String) fromMessage.get("dateStarted");
        String dateFinished = (String) fromMessage.get("dateFinished");
        SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSX");

        try {
            p.setDateStarted(df.parse(dateStarted));
        } catch (ParseException e) {
            LOG.log(Level.SEVERE, "Failed parse dateStarted: {0}", e);
        }

        try {
            p.setDateFinished(df.parse(dateFinished));
        } catch (ParseException e) {
            LOG.log(Level.SEVERE, "Failed parse dateFinished: {0}", e);
        }

        addNewParagraphIfLastParagraphIsExecuted(note, p);
        if (!persistNoteWithAuthInfo(conn, note, p)) {
            return;
        }

        // broadcast to other clients only
        broadcastExcept(note.getId(), new Message(Message.OP.RUN_PARAGRAPH_USING_SPELL).put("paragraph", p), conn);
    }

    public void runParagraph(Session conn, HashSet<String> userAndRoles, Notebook notebook, Message fromMessage,
            Users user, CertificateMaterializer certificateMaterializer, Settings settings,
            DistributedFsService dfsService) throws IOException {
        final String paragraphId = (String) fromMessage.get("id");
        if (paragraphId == null) {
            return;
        }

        String noteId = getOpenNoteId(conn);
        Properties props = notebook.getInterpreterSettingManager().getInterpreterSettings(noteId).get(0)
                .getFlatProperties();
        if (!hasParagraphWriterPermission(conn, notebook, noteId, userAndRoles, fromMessage.principal, "write",
                user)) {
            return;
        }

        // 1. clear paragraph only if personalized,
        // otherwise this will be handed in `onOutputClear`
        final Note note = notebook.getNote(noteId);
        if (note.isPersonalizedMode()) {
            Paragraph p = note.clearPersonalizedParagraphOutput(paragraphId, fromMessage.principal);
            unicastParagraph(note, p, fromMessage.principal);
        }

        // 2. set paragraph values
        String text = (String) fromMessage.get("paragraph");
        String title = (String) fromMessage.get("title");
        Map<String, Object> params = (Map<String, Object>) fromMessage.get("params");
        Map<String, Object> config = (Map<String, Object>) fromMessage.get("config");

        Paragraph p = setParagraphUsingMessage(note, fromMessage, paragraphId, text, title, params, config);

        // Hack: in case the interpreter is hopshive, put the keystore/truststore password in the ticket
        // so that it can be retrieved from the zeppelin-hopshive interpreter
        if ((p.getRequiredReplName() != null && p.getRequiredReplName().equals("hopshive"))
                || (p.getRequiredReplName() == null && notebook().getInterpreterSettingManager()
                        .getDefaultInterpreterSetting(note.getId()).getName().equals("hopshive"))) {
            setCertificatePassword(p, fromMessage);
        }

        persistAndExecuteSingleParagraph(conn, note, p, certificateMaterializer, settings, dfsService);
    }

    private void setCertificatePassword(Paragraph p, Message fromMessage) {
        if (certPwd == null || certPwd.isEmpty()) {
            Users user = project.getOwner();
            try {
                ProjectGenericUserCerts serviceCert = certsFacade
                        .findProjectGenericUserCerts(project.getProjectGenericUser());
                certPwd = HopsUtils.decrypt(user.getPassword(), serviceCert.getCertificatePassword(),
                        certificatesMgmService.getMasterEncryptionPassword());
            } catch (Exception e) {
                LOG.log(Level.SEVERE, "Cannot retrieve user " + project.getName() + " keystore password. " + e);
                certPwd = "";
            }
        }
        p.setAuthenticationInfo(new AuthenticationInfo(fromMessage.principal, fromMessage.roles, certPwd));
    }

    private void addNewParagraphIfLastParagraphIsExecuted(Note note, Paragraph p) {
        // if it's the last paragraph and empty, let's add a new one
        boolean isTheLastParagraph = note.isLastParagraph(p.getId());
        if (!(Strings.isNullOrEmpty(p.getText()) || p.getText().trim().equals(p.getMagic()))
                && isTheLastParagraph) {
            Paragraph newPara = note.addNewParagraph(p.getAuthenticationInfo());
            broadcastNewParagraph(note, newPara);
        }
    }

    /**
     * @return false if failed to save a note
     */
    private boolean persistNoteWithAuthInfo(Session conn, Note note, Paragraph p) throws IOException {
        try {
            note.persist(p.getAuthenticationInfo());
            return true;
        } catch (FileSystemException ex) {
            LOG.log(Level.SEVERE, "Exception from run", ex);
            sendMsg(conn,
                    serializeMessage(new Message(Message.OP.ERROR_INFO).put("info",
                            "Oops! There is something wrong with the notebook file system. "
                                    + "Please check the logs for more details.")));
            // don't run the paragraph when there is error on persisting the note information
            return false;
        }
    }

    private void persistAndExecuteSingleParagraph(Session conn, Note note, Paragraph p,
            CertificateMaterializer certificateMaterializer, Settings settings, DistributedFsService dfsService)
            throws IOException {
        addNewParagraphIfLastParagraphIsExecuted(note, p);
        if (!persistNoteWithAuthInfo(conn, note, p)) {
            return;
        }

        // Materialize certificates both in local filesystem and
        // in HDFS for the interpreters
        if (certificateMaterializer.openedInterpreter(project.getId())) {

            DistributedFileSystemOps dfso = null;
            try {
                dfso = dfsService.getDfsOps();
                HopsUtils.materializeCertificatesForProject(project.getName(), settings.getHdfsTmpCertDir(), dfso,
                        certificateMaterializer, settings);
            } catch (IOException ex) {
                LOG.log(Level.SEVERE, "Error while materializing certificates for Zeppelin", ex);
                certificateMaterializer.closedInterpreter(project.getId());
                HopsUtils.cleanupCertificatesForProject(project.getName(), settings.getHdfsTmpCertDir(),
                        certificateMaterializer, settings);
                throw ex;
            } finally {
                if (null != dfso) {
                    dfso.close();
                    dfso = null;
                }
            }
        }

        try {
            note.run(p.getId());
        } catch (Exception ex) {
            LOG.log(Level.SEVERE, "Exception from run", ex);
            if (p != null) {
                p.setReturn(new InterpreterResult(InterpreterResult.Code.ERROR, ex.getMessage()), ex);
                p.setStatus(Job.Status.ERROR);
                broadcast(note.getId(), new Message(Message.OP.PARAGRAPH).put("paragraph", p));
            }
        }
    }

    private Paragraph setParagraphUsingMessage(Note note, Message fromMessage, String paragraphId, String text,
            String title, Map<String, Object> params, Map<String, Object> config) {
        Paragraph p = note.getParagraph(paragraphId);
        p.setText(text);
        p.setTitle(title);
        AuthenticationInfo subject = new AuthenticationInfo(fromMessage.principal, fromMessage.roles,
                fromMessage.ticket);
        p.setAuthenticationInfo(subject);
        p.settings.setParams(params);
        p.setConfig(config);

        if (note.isPersonalizedMode()) {
            p = note.getParagraph(paragraphId);
            p.setText(text);
            p.setTitle(title);
            p.setAuthenticationInfo(subject);
            p.settings.setParams(params);
            p.setConfig(config);
        }

        return p;
    }

    public void sendAllConfigurations(Session conn, HashSet<String> userAndRoles, Notebook notebook)
            throws IOException {
        ZeppelinConfiguration conf = notebook.getConf();

        Map<String, String> configurations = conf.dumpConfigurations(conf,
                new ZeppelinConfiguration.ConfigurationKeyPredicate() {
                    @Override
                    public boolean apply(String key) {
                        return !key.contains("password") && !key
                                .equals(ZeppelinConfiguration.ConfVars.ZEPPELIN_NOTEBOOK_AZURE_CONNECTION_STRING
                                        .getVarName());
                    }
                });

        sendMsg(conn, serializeMessage(
                new Message(Message.OP.CONFIGURATIONS_INFO).put("configurations", configurations)));
    }

    public void checkpointNote(Session conn, Notebook notebook, Message fromMessage) throws IOException {
        String noteId = (String) fromMessage.get("noteId");
        String commitMessage = (String) fromMessage.get("commitMessage");
        AuthenticationInfo subject = new AuthenticationInfo(fromMessage.principal);
        NotebookRepo.Revision revision = notebook.checkpointNote(noteId, commitMessage, subject);
        if (!NotebookRepo.Revision.isEmpty(revision)) {
            List<NotebookRepo.Revision> revisions = notebook.listRevisionHistory(noteId, subject);
            sendMsg(conn,
                    serializeMessage(new Message(Message.OP.LIST_REVISION_HISTORY).put("revisionList", revisions)));
        } else {
            sendMsg(conn,
                    serializeMessage(new Message(Message.OP.ERROR_INFO).put("info",
                            "Couldn't checkpoint note revision: possibly storage "
                                    + "doesn't support versioning. Please check the logs for" + " more details.")));
        }
    }

    public void listRevisionHistory(Session conn, Notebook notebook, Message fromMessage) throws IOException {
        String noteId = (String) fromMessage.get("noteId");
        AuthenticationInfo subject = new AuthenticationInfo(fromMessage.principal);
        List<NotebookRepo.Revision> revisions = notebook.listRevisionHistory(noteId, subject);

        sendMsg(conn,
                serializeMessage(new Message(Message.OP.LIST_REVISION_HISTORY).put("revisionList", revisions)));
    }

    public void setNoteRevision(Session conn, HashSet<String> userAndRoles, Notebook notebook, Message fromMessage,
            Users user) throws IOException {

        String noteId = (String) fromMessage.get("noteId");
        String revisionId = (String) fromMessage.get("revisionId");
        AuthenticationInfo subject = new AuthenticationInfo(fromMessage.principal);

        if (!hasParagraphWriterPermission(conn, notebook, noteId, userAndRoles, fromMessage.principal, "update",
                user)) {
            return;
        }

        Note headNote = null;
        boolean setRevisionStatus;
        try {
            headNote = notebook.setNoteRevision(noteId, revisionId, subject);
            setRevisionStatus = headNote != null;
        } catch (Exception e) {
            setRevisionStatus = false;
            LOG.log(Level.SEVERE, "Failed to set given note revision", e);
        }
        if (setRevisionStatus) {
            notebook.loadNoteFromRepo(noteId, subject);
        }

        sendMsg(conn, serializeMessage(new Message(Message.OP.SET_NOTE_REVISION).put("status", setRevisionStatus)));

        if (setRevisionStatus) {
            Note reloadedNote = notebook.getNote(headNote.getId());
            broadcastNote(reloadedNote);
        } else {
            sendMsg(conn, serializeMessage(new Message(Message.OP.ERROR_INFO).put("info",
                    "Couldn't set note to the given revision. " + "Please check the logs for more details.")));
        }
    }

    public void getNoteByRevision(Session conn, Notebook notebook, Message fromMessage) throws IOException {
        String noteId = (String) fromMessage.get("noteId");
        String revisionId = (String) fromMessage.get("revisionId");
        AuthenticationInfo subject = new AuthenticationInfo(fromMessage.principal);
        Note revisionNote = notebook.getNoteByRevision(noteId, revisionId, subject);
        sendMsg(conn, serializeMessage(new Message(Message.OP.NOTE_REVISION).put("noteId", noteId)
                .put("revisionId", revisionId).put("note", revisionNote)));
    }

    /**
     * This callback is for the paragraph that runs on ZeppelinServer
     *
     * @param output output to append
     */
    @Override
    public void onOutputAppend(String noteId, String paragraphId, int index, String output) {
        Message msg = new Message(Message.OP.PARAGRAPH_APPEND_OUTPUT).put("noteId", noteId)
                .put("paragraphId", paragraphId).put("index", index).put("data", output);
        broadcast(noteId, msg);
    }

    /**
     * This callback is for the paragraph that runs on ZeppelinServer
     *
     * @param output output to update (replace)
     */
    @Override
    public void onOutputUpdated(String noteId, String paragraphId, int index, InterpreterResult.Type type,
            String output) {
        Message msg = new Message(Message.OP.PARAGRAPH_UPDATE_OUTPUT).put("noteId", noteId)
                .put("paragraphId", paragraphId).put("index", index).put("type", type).put("data", output);
        Note note = notebook().getNote(noteId);
        if (note.isPersonalizedMode()) {
            String user = note.getParagraph(paragraphId).getUser();
            if (null != user) {
                multicastToUser(user, msg);
            }
        } else {
            broadcast(noteId, msg);
        }
    }

    /**
     * This callback is for the paragraph that runs on ZeppelinServer
     */
    @Override
    public void onOutputClear(String noteId, String paragraphId) {
        Notebook notebook = notebook();
        final Note note = notebook.getNote(noteId);
        note.clearParagraphOutput(paragraphId);
        Paragraph paragraph = note.getParagraph(paragraphId);
        broadcastParagraph(note, paragraph);
    }

    /**
     * When application append output
     */
    @Override
    public void onOutputAppend(String noteId, String paragraphId, int index, String appId, String output) {
        Message msg = new Message(Message.OP.APP_APPEND_OUTPUT).put("noteId", noteId)
                .put("paragraphId", paragraphId).put("index", index).put("appId", appId).put("data", output);
        broadcast(noteId, msg);
    }

    /**
     * When application update output
     */
    @Override
    public void onOutputUpdated(String noteId, String paragraphId, int index, String appId,
            InterpreterResult.Type type, String output) {
        Message msg = new Message(Message.OP.APP_UPDATE_OUTPUT).put("noteId", noteId)
                .put("paragraphId", paragraphId).put("index", index).put("type", type).put("appId", appId)
                .put("data", output);
        broadcast(noteId, msg);
    }

    @Override
    public void onLoad(String noteId, String paragraphId, String appId, HeliumPackage pkg) {
        Message msg = new Message(Message.OP.APP_LOAD).put("noteId", noteId).put("paragraphId", paragraphId)
                .put("appId", appId).put("pkg", pkg);
        broadcast(noteId, msg);
    }

    @Override
    public void onStatusChange(String noteId, String paragraphId, String appId, String status) {
        Message msg = new Message(Message.OP.APP_STATUS_CHANGE).put("noteId", noteId)
                .put("paragraphId", paragraphId).put("appId", appId).put("status", status);
        broadcast(noteId, msg);
    }

    @Override
    public void onGetParagraphRunners(String noteId, String paragraphId,
            RemoteInterpreterProcessListener.RemoteWorksEventListener callback) {
        Notebook notebookIns = notebook();
        List<InterpreterContextRunner> runner = new LinkedList<>();

        if (notebookIns == null) {
            LOG.info("intepreter request notebook instance is null");
            callback.onFinished(notebookIns);
        }

        try {
            Note note = notebookIns.getNote(noteId);
            if (note != null) {
                if (paragraphId != null) {
                    Paragraph paragraph = note.getParagraph(paragraphId);
                    if (paragraph != null) {
                        runner.add(paragraph.getInterpreterContextRunner());
                    }
                } else {
                    for (Paragraph p : note.getParagraphs()) {
                        runner.add(p.getInterpreterContextRunner());
                    }
                }
            }
            callback.onFinished(runner);
        } catch (NullPointerException e) {
            LOG.log(Level.SEVERE, e.getMessage());
            callback.onError();
        }
    }

    @Override
    public void onRemoteRunParagraph(String noteId, String paragraphId) throws Exception {
        Notebook notebookIns = notebook();
        try {
            if (notebookIns == null) {
                throw new Exception("onRemoteRunParagraph notebook instance is null");
            }
            Note noteIns = notebookIns.getNote(noteId);
            if (noteIns == null) {
                throw new Exception(String.format("Can't found note id %s", noteId));
            }

            Paragraph paragraph = noteIns.getParagraph(paragraphId);
            if (paragraph == null) {
                throw new Exception(String.format("Can't found paragraph %s %s", noteId, paragraphId));
            }

            Set<String> userAndRoles = Sets.newHashSet();
            userAndRoles.add(SecurityUtils.getPrincipal());
            userAndRoles.addAll(SecurityUtils.getRoles());
            if (!notebookIns.getNotebookAuthorization().hasWriteAuthorization(userAndRoles, noteId)) {
                throw new ForbiddenException(String.format("can't execute note %s", noteId));
            }

            AuthenticationInfo subject = new AuthenticationInfo(SecurityUtils.getPrincipal());
            paragraph.setAuthenticationInfo(subject);

            noteIns.run(paragraphId);

        } catch (Exception e) {
            throw e;
        }
    }

    /**
     * Notebook Information Change event
     */
    public static class NotebookInformationListener implements NotebookEventListener {

        private NotebookServerImpl notebookServer;

        public NotebookInformationListener(NotebookServerImpl notebookServer) {
            this.notebookServer = notebookServer;
        }

        @Override
        public void onParagraphRemove(Paragraph p) {
            try {
                notebookServer.broadcastUpdateNoteJobInfo(System.currentTimeMillis() - 5000);
            } catch (IOException ioe) {
                LOG.log(Level.SEVERE, "can not broadcast for job manager {}", ioe.getMessage());
            }
        }

        @Override
        public void onNoteRemove(Note note) {
            try {
                notebookServer.broadcastUpdateNoteJobInfo(System.currentTimeMillis() - 5000);
            } catch (IOException ioe) {
                LOG.log(Level.SEVERE, "can not broadcast for job manager {}", ioe.getMessage());
            }

            List<Map<String, Object>> notesInfo = new LinkedList<>();
            Map<String, Object> info = new HashMap<>();
            info.put("noteId", note.getId());
            // set paragraphs
            List<Map<String, Object>> paragraphsInfo = new LinkedList<>();

            // notebook json object root information.
            info.put("isRunningJob", false);
            info.put("unixTimeLastRun", 0);
            info.put("isRemoved", true);
            info.put("paragraphs", paragraphsInfo);
            notesInfo.add(info);

            Map<String, Object> response = new HashMap<>();
            response.put("lastResponseUnixTime", System.currentTimeMillis());
            response.put("jobs", notesInfo);

            notebookServer.broadcast(NotebookServer.JOB_MANAGER_SERVICE.JOB_MANAGER_PAGE.getKey(),
                    new Message(Message.OP.LIST_UPDATE_NOTE_JOBS).put("noteRunningJobs", response));

        }

        @Override
        public void onParagraphCreate(Paragraph p) {
            Notebook notebook = notebookServer.notebook();
            List<Map<String, Object>> notebookJobs = notebook.getJobListByParagraphId(p.getId());
            Map<String, Object> response = new HashMap<>();
            response.put("lastResponseUnixTime", System.currentTimeMillis());
            response.put("jobs", notebookJobs);

            notebookServer.broadcast(NotebookServer.JOB_MANAGER_SERVICE.JOB_MANAGER_PAGE.getKey(),
                    new Message(Message.OP.LIST_UPDATE_NOTE_JOBS).put("noteRunningJobs", response));
        }

        @Override
        public void onNoteCreate(Note note) {
            Notebook notebook = notebookServer.notebook();
            List<Map<String, Object>> notebookJobs = notebook.getJobListByNoteId(note.getId());
            Map<String, Object> response = new HashMap<>();
            response.put("lastResponseUnixTime", System.currentTimeMillis());
            response.put("jobs", notebookJobs);

            notebookServer.broadcast(NotebookServer.JOB_MANAGER_SERVICE.JOB_MANAGER_PAGE.getKey(),
                    new Message(Message.OP.LIST_UPDATE_NOTE_JOBS).put("noteRunningJobs", response));
        }

        @Override
        public void onParagraphStatusChange(Paragraph p, Job.Status status) {
            Notebook notebook = notebookServer.notebook();
            List<Map<String, Object>> notebookJobs = notebook.getJobListByParagraphId(p.getId());

            Map<String, Object> response = new HashMap<>();
            response.put("lastResponseUnixTime", System.currentTimeMillis());
            response.put("jobs", notebookJobs);

            notebookServer.broadcast(NotebookServer.JOB_MANAGER_SERVICE.JOB_MANAGER_PAGE.getKey(),
                    new Message(Message.OP.LIST_UPDATE_NOTE_JOBS).put("noteRunningJobs", response));
        }

        @Override
        public void onUnbindInterpreter(Note note, InterpreterSetting setting) {
            Notebook notebook = notebookServer.notebook();
            List<Map<String, Object>> notebookJobs = notebook.getJobListByNoteId(note.getId());
            Map<String, Object> response = new HashMap<>();
            response.put("lastResponseUnixTime", System.currentTimeMillis());
            response.put("jobs", notebookJobs);

            notebookServer.broadcast(NotebookServer.JOB_MANAGER_SERVICE.JOB_MANAGER_PAGE.getKey(),
                    new Message(Message.OP.LIST_UPDATE_NOTE_JOBS).put("noteRunningJobs", response));
        }
    }

    /**
     * Need description here.
     */
    public static class ParagraphListenerImpl implements ParagraphJobListener {

        private NotebookServerImpl notebookServer;
        private Note note;

        public ParagraphListenerImpl(NotebookServerImpl notebookServer, Note note) {
            this.notebookServer = notebookServer;
            this.note = note;
        }

        @Override
        public void onProgressUpdate(Job job, int progress) {
            notebookServer.broadcast(note.getId(),
                    new Message(Message.OP.PROGRESS).put("id", job.getId()).put("progress", progress));
        }

        @Override
        public void beforeStatusChange(Job job, Job.Status before, Job.Status after) {
        }

        @Override
        public void afterStatusChange(Job job, Job.Status before, Job.Status after) {
            if (after == Job.Status.ERROR) {
                if (job.getException() != null) {
                    LOG.log(Level.INFO, "Error", job.getException());
                }
            }

            if (job.isTerminated()) {
                if (job.getStatus() == Job.Status.FINISHED) {
                    LOG.log(Level.INFO, "Job {0} is finished successfully, status: {1}",
                            new Object[] { job.getId(), job.getStatus() });
                } else {
                    LOG.log(Level.SEVERE, "Job {0} is finished, status: {1}, exception: {2}, result: {3}",
                            new Object[] { job.getId(), job.getStatus(), job.getException(), job.getReturn() });
                }

                try {
                    //TODO(khalid): may change interface for JobListener and pass subject from interpreter
                    note.persist(job instanceof Paragraph ? ((Paragraph) job).getAuthenticationInfo() : null);
                } catch (IOException e) {
                    LOG.log(Level.SEVERE, e.toString(), e);
                }
            }
            if (job instanceof Paragraph) {
                Paragraph p = (Paragraph) job;
                p.setStatusToUserParagraph(job.getStatus());
                notebookServer.broadcastParagraph(note, p);
            }
            try {
                notebookServer.broadcastUpdateNoteJobInfo(System.currentTimeMillis() - 5000);
            } catch (IOException e) {
                LOG.log(Level.SEVERE, "can not broadcast for job manager {}", e);
            }
        }

        /**
         * This callback is for paragraph that runs on RemoteInterpreterProcess
         */
        @Override
        public void onOutputAppend(Paragraph paragraph, int idx, String output) {
            Message msg = new Message(Message.OP.PARAGRAPH_APPEND_OUTPUT).put("noteId", paragraph.getNote().getId())
                    .put("paragraphId", paragraph.getId()).put("data", output);

            notebookServer.broadcast(paragraph.getNote().getId(), msg);
        }

        /**
         * This callback is for paragraph that runs on RemoteInterpreterProcess
         */
        @Override
        public void onOutputUpdate(Paragraph paragraph, int idx, InterpreterResultMessage result) {
            String output = result.getData();
            Message msg = new Message(Message.OP.PARAGRAPH_UPDATE_OUTPUT).put("noteId", paragraph.getNote().getId())
                    .put("paragraphId", paragraph.getId()).put("data", output);

            notebookServer.broadcast(paragraph.getNote().getId(), msg);
        }

        @Override
        public void onOutputUpdateAll(Paragraph paragraph, List<InterpreterResultMessage> msgs) {
            // TODO
        }
    }

    @Override
    public ParagraphJobListener getParagraphJobListener(Note note) {
        return new ParagraphListenerImpl(this, note);
    }

    public NotebookEventListener getNotebookInformationListener() {
        return new NotebookInformationListener(this);
    }

    private void sendAllAngularObjects(Note note, String user, Session conn) throws IOException {
        List<InterpreterSetting> settings = notebook().getInterpreterSettingManager()
                .getInterpreterSettings(note.getId());
        if (settings == null || settings.size() == 0) {
            return;
        }

        for (InterpreterSetting intpSetting : settings) {
            AngularObjectRegistry registry = intpSetting.getInterpreterGroup(user, note.getId())
                    .getAngularObjectRegistry();
            List<AngularObject> objects = registry.getAllWithGlobal(note.getId());
            for (AngularObject object : objects) {
                sendMsg(conn,
                        serializeMessage(new Message(Message.OP.ANGULAR_OBJECT_UPDATE).put("angularObject", object)
                                .put("interpreterGroupId",
                                        intpSetting.getInterpreterGroup(user, note.getId()).getId())
                                .put("noteId", note.getId()).put("paragraphId", object.getParagraphId())));
            }
        }
    }

    @Override
    public void onAdd(String interpreterGroupId, AngularObject object) {
        onUpdate(interpreterGroupId, object);
    }

    @Override
    public void onUpdate(String interpreterGroupId, AngularObject object) {
        Notebook notebook = notebook();
        if (notebook == null) {
            return;
        }

        List<Note> notes = notebook.getAllNotes();
        for (Note note : notes) {
            if (object.getNoteId() != null && !note.getId().equals(object.getNoteId())) {
                continue;
            }

            List<InterpreterSetting> intpSettings = notebook.getInterpreterSettingManager()
                    .getInterpreterSettings(note.getId());
            if (intpSettings.isEmpty()) {
                continue;
            }

            broadcast(note.getId(),
                    new Message(Message.OP.ANGULAR_OBJECT_UPDATE).put("angularObject", object)
                            .put("interpreterGroupId", interpreterGroupId).put("noteId", note.getId())
                            .put("paragraphId", object.getParagraphId()));
        }
    }

    @Override
    public void onRemove(String interpreterGroupId, String name, String noteId, String paragraphId) {
        Notebook notebook = notebook();
        List<Note> notes = notebook.getAllNotes();
        for (Note note : notes) {
            if (noteId != null && !note.getId().equals(noteId)) {
                continue;
            }

            List<String> settingIds = notebook.getInterpreterSettingManager().getInterpreters(note.getId());
            for (String id : settingIds) {
                if (interpreterGroupId.contains(id)) {
                    broadcast(note.getId(), new Message(Message.OP.ANGULAR_OBJECT_REMOVE).put("name", name)
                            .put("noteId", noteId).put("paragraphId", paragraphId));
                    break;
                }
            }
        }
    }

    public void getEditorSetting(Session conn, Message fromMessage) throws IOException {
        String paragraphId = (String) fromMessage.get("paragraphId");
        String replName = (String) fromMessage.get("magic");
        String noteId = getOpenNoteId(conn);
        String user = fromMessage.principal;
        Message resp = new Message(Message.OP.EDITOR_SETTING);
        resp.put("paragraphId", paragraphId);
        Interpreter interpreter = notebook().getInterpreterFactory().getInterpreter(user, noteId, replName);
        resp.put("editor",
                notebook().getInterpreterSettingManager().getEditorSetting(interpreter, user, noteId, replName));
        sendMsg(conn, serializeMessage(resp));
    }

    public void getInterpreterSettings(Session conn, AuthenticationInfo subject) throws IOException {
        List<InterpreterSetting> availableSettings = notebook().getInterpreterSettingManager().get();
        sendMsg(conn, serializeMessage(
                new Message(Message.OP.INTERPRETER_SETTINGS).put("interpreterSettings", availableSettings)));
    }

    @Override
    public void onMetaInfosReceived(String settingId, Map<String, String> metaInfos) {
        InterpreterSetting interpreterSetting = notebook().getInterpreterSettingManager().get(settingId);
        interpreterSetting.setInfos(metaInfos);
    }

    public void switchConnectionToWatcher(Session conn, Message messagereceived, String hdfsUsername,
            NotebookServerImplFactory notebookServerImplFactory) throws IOException {
        if (!isSessionAllowedToSwitchToWatcher(conn)) {
            LOG.log(Level.SEVERE, "Cannot switch this client to watcher, invalid security key");
            return;
        }
        LOG.log(Level.INFO, "Going to add {} to watcher socket", conn);
        // add the connection to the watcher.
        if (watcherSockets.contains(conn)) {
            LOG.info("connection alrerady present in the watcher");
            return;
        }
        watcherSockets.add(conn);

        // remove this connection from regular zeppelin ws usage.
        removeConnectionFromAllNote(conn);
        removeConnectedSockets(conn, notebookServerImplFactory);
        removeUserConnection(hdfsUsername, conn);
        removeUserConnection(project.getProjectGenericUser(), conn);
    }

    public synchronized void addConnectedSocket(Session conn) {
        connectedSockets.add(conn);
    }

    public synchronized void removeConnectedSockets(Session conn,
            NotebookServerImplFactory notebookServerImplFactory) {
        connectedSockets.remove(conn);
        if (connectedSockets.isEmpty()) {
            notebookServerImplFactory.removeNotebookServerImpl(this.project.getName());
        }
    }

    public synchronized boolean connectedSocketsIsEmpty() {
        return connectedSockets.isEmpty();
    }

    public boolean userConnectedSocketsContainsKey(String user) {
        return userConnectedSockets.containsKey(user);
    }

    public Queue<Session> getUserConnectedSocket(String user) {
        return userConnectedSockets.get(user);
    }

    public void putUserConnectedSocket(String user, Queue<Session> conn) {
        userConnectedSockets.put(user, conn);
    }

    public void removeUserConnection(String user, Session conn) {
        if (userConnectedSockets.containsKey(user)) {
            userConnectedSockets.get(user).remove(conn);
        } else {
            LOG.log(Level.SEVERE, "Closing connection that is absent in user connections");
        }
    }

    private boolean isSessionAllowedToSwitchToWatcher(Session session) {
        String watcherSecurityKey = (String) session.getUserProperties().get(WatcherSecurityKey.HTTP_HEADER);
        return !(StringUtils.isBlank(watcherSecurityKey)
                || !watcherSecurityKey.equals(WatcherSecurityKey.getKey()));
    }

    private void broadcastToWatchers(String noteId, String subject, Message message) {
        synchronized (watcherSockets) {
            if (watcherSockets.isEmpty()) {
                return;
            }
            for (Session watcher : watcherSockets) {
                try {
                    sendMsg(watcher, WatcherMessage.builder(noteId).subject(subject)
                            .message(serializeMessage(message)).build().toJson());
                } catch (IOException e) {
                    LOG.log(Level.SEVERE, "Cannot broadcast message to watcher", e);
                }
            }
        }
    }

    @Override
    public void onParaInfosReceived(String noteId, String paragraphId, String interpreterSettingId,
            Map<String, String> metaInfos) {
        Note note = notebook().getNote(noteId);
        if (note != null) {
            Paragraph paragraph = note.getParagraph(paragraphId);
            if (paragraph != null) {
                InterpreterSetting setting = notebook().getInterpreterSettingManager().get(interpreterSettingId);
                setting.addNoteToPara(noteId, paragraphId);
                String label = metaInfos.get("label");
                String tooltip = metaInfos.get("tooltip");
                List<String> keysToRemove = Arrays.asList("noteId", "paraId", "label", "tooltip");
                for (String removeKey : keysToRemove) {
                    metaInfos.remove(removeKey);
                }
                paragraph.updateRuntimeInfos(label, tooltip, metaInfos, setting.getGroup(), setting.getId());
                broadcast(note.getId(), new Message(Message.OP.PARAS_INFO).put("id", paragraphId).put("infos",
                        paragraph.getRuntimeInfos()));
            }
        }
    }

    public void clearParagraphRuntimeInfo(InterpreterSetting setting) {
        Map<String, Set<String>> noteIdAndParaMap = setting.getNoteIdAndParaMap();
        if (noteIdAndParaMap != null && !noteIdAndParaMap.isEmpty()) {
            for (String noteId : noteIdAndParaMap.keySet()) {
                Set<String> paraIdSet = noteIdAndParaMap.get(noteId);
                if (paraIdSet != null && !paraIdSet.isEmpty()) {
                    for (String paraId : paraIdSet) {
                        Note note = notebook().getNote(noteId);
                        if (note != null) {
                            Paragraph paragraph = note.getParagraph(paraId);
                            if (paragraph != null) {
                                paragraph.clearRuntimeInfo(setting.getId());
                                broadcast(noteId, new Message(Message.OP.PARAGRAPH).put("paragraph", paragraph));
                            }
                        }
                    }
                }
            }
        }
        setting.clearNoteIdAndParaMap();
    }

    public void sendMsg(Session conn, String msg) throws IOException {
        if (conn == null || !conn.isOpen()) {
            LOG.log(Level.SEVERE, "Can't handle message. The connection has been closed.");
            return;
        }
        conn.getBasicRemote().sendText(msg);
    }

    protected String serializeMessage(Message m) {
        return gson.toJson(m);
    }

    public void closeConnections(NotebookServerImplFactory notebookServerImplFactory) {
        for (Map.Entry<String, Queue<Session>> entry : userConnectedSockets.entrySet()) {
            for (Session session : entry.getValue()) {
                closeConnection(session, entry.getKey(), notebookServerImplFactory);
            }
        }
    }

    public void closeConnection(Session session, String hdfsUsername,
            NotebookServerImplFactory notebookServerImplFactory) {
        try {
            if (session.isOpen()) {
                session.getBasicRemote().sendText("Restarting zeppelin.");
                session.close(new CloseReason(CloseReason.CloseCodes.SERVICE_RESTART, "Restarting zeppelin."));
            }
            removeConnectionFromAllNote(session);
            removeConnectedSockets(session, notebookServerImplFactory);
            removeUserConnection(hdfsUsername, session);
            removeUserConnection(project.getProjectGenericUser(), session);
        } catch (IOException ex) {
            LOG.log(Level.SEVERE, null, ex);
        }
    }
}