controllers.TargetController.java Source code

Introduction

Here is the source code for controllers.TargetController.java

Source

package controllers;

import static play.data.Form.form;

import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.PrintWriter;
import java.io.StringWriter;
import java.net.MalformedURLException;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
import java.text.DateFormat;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Date;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.TimeZone;

import com.avaje.ebean.*;
import org.apache.commons.lang3.StringUtils;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.ss.usermodel.Cell;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Workbook;
import org.apache.poi.ss.usermodel.WorkbookFactory;
import org.apache.commons.validator.routines.UrlValidator;

import models.Collection;
import models.CrawlPermission;
import models.FieldUrl;
import models.Flag;
import models.License;
import models.Organisation;
import models.QaIssue;
import models.Subject;
import models.Tag;
import models.Target;
import models.Taxonomy;
import models.User;
import play.Logger;
import play.Play;
import play.data.DynamicForm;
import play.data.Form;
import play.data.validation.ValidationError;
import play.libs.Json;
import play.mvc.BodyParser;
import play.mvc.Http.MultipartFormData;
import play.mvc.Http.MultipartFormData.FilePart;
import play.mvc.Result;
import play.mvc.Security;
import play.mvc.With;
import uk.bl.Const;
import uk.bl.Const.CrawlFrequency;
import uk.bl.api.Utils;
import uk.bl.api.models.CrawlFeedItem;
import uk.bl.exception.ActException;
import uk.bl.exception.WhoisException;
import uk.bl.scope.Scope;
import views.html.collections.sites;
import views.html.licence.ukwalicenceresult;
import views.html.infomessage;

import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.JsonNodeFactory;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.rabbitmq.client.AMQP.BasicProperties;
import com.rabbitmq.client.ConnectionFactory;
import com.rabbitmq.client.Connection;
import com.rabbitmq.client.Channel;

import views.html.targets.blank;
import views.html.targets.edit;
import views.html.targets.list;
import views.html.targets.lookup;
import views.html.targets.view;
import views.html.users.usersites;

/**
 * Describe W3ACT project.
 */
@Security.Authenticated(SecuredController.class)
public class TargetController extends AbstractController {

    final static Form<Target> targetForm = new Form<Target>(Target.class);
    private static final String DEFAULT_SORT_BY = "title";
    private static final String DEFAULT_ORDER = "asc";

    /**
     * Display the targets.
     */
    public static Result index() {
        return GO_HOME;

    }

    /**
     * Display the paginated list of targets.
     *
     * @param page   Current page number (starts from 0)
     * @param sortBy Column to be sorted
     * @param order  Sort order (either asc or desc)
     * @param filter Filter applied on target urls
     */
    public static Result lookup(int pageNo, String sortBy, String order, String filter) {
        Logger.debug("TargetController.lookup()");

        String url = filter;
        if (url.startsWith("http://")) {
            url = url.replace("http://", "");
        }
        if (url.startsWith("https://")) {
            url = url.replace("https://", "");
        }

        // Re-enable www-ignoring part of search for lookup:
        if (url.startsWith("www.")) {
            url = url.replace("www.", "");
        }

        if (url.endsWith("/")) {
            url = url.substring(0, url.length() - 1);
        }

        Logger.debug("After processing Filter::" + url);

        Query<FieldUrl> query = FieldUrl.find.fetch("target").fetch("target.organisation").setDistinct(true).where()
                .add(Expr.or(Expr.icontains("target.synonyms", url),
                        Expr.or(Expr.icontains("url", url), Expr.icontains("target.title", url))))
                .query();

        // Set up the sorting:
        if ("title".equals(sortBy)) {
            query = query.orderBy("target.title" + " " + order);
        } else {
            if ("organisation".equals(sortBy)) {
                query = query.orderBy("target.organisation.id" + " " + order);
            } else {
                if ("seeds".equals(sortBy)) {
                    query = query.orderBy("url" + " " + order);
                } else {
                    if ("frequency".equals(sortBy)) {
                        query = query.orderBy("target.crawlFrequency" + " " + order);
                    } else {
                        if ("active".equals(sortBy)) {
                            query = query.orderBy("target.active" + " " + order);
                        }
                    }
                }
            }
        }
        // Finish the query:
        Page<FieldUrl> pages = query.findPagingList(20).setFetchAhead(false).getPage(pageNo);

        Logger.debug("Total: " + pages.getTotalRowCount());

        // Also check for clear match:
        Target matchTarget = null;
        try {
            FieldUrl isExistingFieldUrl = FieldUrl.hasDuplicate(filter.trim());
            if (isExistingFieldUrl != null) {
                matchTarget = isExistingFieldUrl.target;
            }
        } catch (Exception e) {
            Logger.error("Problem looking up duplicate URLs.", e);
        }

        return ok(lookup.render("Lookup " + filter, matchTarget, User.findByEmail(request().username()), filter,
                pages, sortBy, order));
    }

    /**
     * Display the paginated list of targets.
     *
     * @param page           Current page number (starts from 0)
     * @param sortBy         Column to be sorted
     * @param order          Sort order (either asc or desc)
     * @param filter         Filter applied on target urls
     * @param curatorId      Author of the target
     * @param organisation   The author's organisation
     * @param subject        Target subject
     * @param crawlFrequency The crawl frequency
     * @param depth          The crawl depth
     * @param collection     The associated collection
     * @param license        The license name
     * @param pageSize       The number of Target entries on the page
     * @param flag           The flag assigned by user
     */
    public static Result list(int pageNo, String sortBy, String order, String filter, Long curatorId,
            Long organisationId, String subject, String crawlFrequencyName, String depthName, String collection,
            Long licenseId, int pageSize, Long flagId) {

        String url = filter;
        if (url.startsWith("http://")) {
            url = url.replace("http://", "");
        }
        if (url.startsWith("https://")) {
            url = url.replace("https://", "");
        }

        // The list page does NOT match so easily, www prefix NOT ignored:
        //if(url.startsWith("www.")) {
        //    url = url.replace("www.", "");
        //}

        if (url.endsWith("/")) {
            url = url.substring(0, url.length() - 1);
        }

        if (StringUtils.isBlank(sortBy)) {
            sortBy = DEFAULT_SORT_BY;
            order = DEFAULT_ORDER;
        }

        Logger.debug("After processing Filter::" + url);

        Logger.debug("Pre Targets.list(): " + pageNo + " - " + url + " - " + curatorId + " - " + organisationId
                + " - " + subject + " - " + crawlFrequencyName + " - " + depthName + " - " + collection + " - "
                + licenseId + " - " + pageSize + " - " + flagId);

        Page<Target> pageTargets = Target.pageTargets(pageNo, pageSize, sortBy, order, url, curatorId,
                organisationId, subject, crawlFrequencyName, depthName, collection, licenseId, flagId);

        User user = User.findByEmail(request().username());
        List<License> licenses = License.findAllLicenses();

        List<Long> subjectIds = new ArrayList<Long>();
        String[] subjects = subject.split(", ");
        for (String sId : subjects) {
            if (StringUtils.isNotEmpty(sId)) {
                Long subjectId = Long.valueOf(sId);
                subjectIds.add(subjectId);
            }
        }
        JsonNode subjectData = getSubjectsDataByIds(subjectIds);

        List<Long> collectionIds = new ArrayList<Long>();
        String[] collections = collection.split(", ");
        for (String cId : collections) {
            if (StringUtils.isNotEmpty(cId)) {
                Long collectionId = Long.valueOf(cId);
                collectionIds.add(collectionId);
            }
        }
        JsonNode collectionData = getCollectionsDataByIds(collectionIds);

        List<User> users = User.findAllSorted();
        List<Organisation> organisations = Organisation.findAllSorted();
        CrawlFrequency[] crawlFrequencies = Const.CrawlFrequency.values();
        List<Flag> flags = Flag.findAllFlags();

        Logger.debug("getTotalRowCount: " + pageTargets.getTotalRowCount());

        return ok(list.render("Targets", user, filter, pageTargets, sortBy, order, curatorId, organisationId,
                subject, crawlFrequencyName, depthName, collection, licenseId, pageSize, flagId, licenses,
                collectionData, subjectData, users, organisations, crawlFrequencies, flags));
    }

    public static String getTitle(Long id) {
        Target target = Target.findById(id);
        if (target != null) {
            return target.title;
        } else {
            return "Blank Target";
        }
    }

    public static Result view(Long id) {
        Target target = Target.findById(id);
        if (target != null) {
            if (request().accepts("text/html")) {
                User user = User.findByEmail(request().username());
                return ok(view.render(target, user));
            } else {
                return ok(Json.toJson(target));
            }
        } else {
            return notFound("There is no Target with ID " + id);
        }
    }

    public static Result viewAct(String url) {
        Target target = Target.findByUrl(url);
        User user = User.findByEmail(request().username());
        return ok(view.render(target, user));
    }

    public static Result viewWct(String url) {
        Target target = Target.findByWct(url);
        User user = User.findByEmail(request().username());
        return ok(view.render(target, user));
    }

    //    @BodyParser.Of(BodyParser.Json.class)
    public static Result filterByJson(String url) {
        if (url.startsWith("http://")) {
            url = url.replace("http://", "");
        }
        if (url.startsWith("https://")) {
            url = url.replace("https://", "");
        }

        // Leave this out for now, making AJAX matches more precise:
        //if(url.startsWith("www.")) {
        //    url = url.replace("www.", "");
        //}

        if (url.endsWith("/")) {
            url = url.substring(0, url.length() - 1);
        }

        Logger.debug("search url is: " + url);

        JsonNode jsonData = null;
        if (url != null && url.length() >= 3) {
            Query<FieldUrl> query = FieldUrl.find.fetch("target").fetch("target.organisation").where()
                    .add(Expr.or(Expr.icontains("url", url), Expr.icontains("target.title", url))).orderBy()
                    .asc("url");

            // Only grab the first few...
            Page<FieldUrl> pages = query.findPagingList(20).setFetchAhead(false).getPage(0);

            List<Target> targets = new ArrayList<Target>();
            for (FieldUrl f : pages.getList()) {
                targets.add(f.target);
            }
            jsonData = Json.toJson(targets);
        }
        if (jsonData != null) {
            return ok(jsonData);
        } else {
            return notFound();
        }
    }

    /**
     * This method enables searching for given URL and redirection in order to add new entry
     * if required.
     *
     * @return
     */
    public static Result searchTargets() {
        DynamicForm requestData = Form.form().bindFromRequest();
        if (requestData.get("pageSize") == null || (requestData.get("pageSize") != null
                && !Utils.INSTANCE.isNumeric(requestData.get("pageSize")))) {
            flash("message", "You may only enter a numeric page size.");
            return GO_HOME;
        }

        String action = requestData.get("action");
        String filter = requestData.get("filter");

        //       if (StringUtils.isBlank(query)) {
        //         Logger.debug("Target name is empty. Please write name in search window.");
        //         flash("message", "Please enter a name in the search window");
        //           return GO_HOME;
        //       }       

        int pageNo = Integer.parseInt(requestData.get("p"));
        String sort = requestData.get("s");
        String order = requestData.get("o");

        int pageSize = Integer.parseInt(requestData.get("pageSize"));
        Long curatorId = Long.parseLong(requestData.get("curator"));
        Long organisationId = Long.parseLong(requestData.get("organisation"));
        Long licenseId = Long.parseLong(requestData.get("license"));
        String depthName = requestData.get("depth");
        String crawlFrequencyName = requestData.get("crawlFrequency");
        Long flagId = Long.parseLong(requestData.get("flag"));

        String subjectSelect = requestData.get("subjectSelect").replace("\"", "");
        String collectionSelect = requestData.get("collectionSelect").replace("\"", "");

        Logger.debug(filter + " " + pageNo + " " + sort + " " + order + " " + pageSize + " " + curatorId + " "
                + organisationId + " " + licenseId + " " + depthName + " " + crawlFrequencyName + " " + flagId + " "
                + collectionSelect + " " + subjectSelect);

        if (StringUtils.isEmpty(action)) {
            return badRequest("You must provide a valid action");
        } else {
            if (action.equals("add")) {
                return redirect(routes.TargetController.newForm(filter));
            } else {
                if (action.equals("clear")) {
                    return GO_HOME;
                } else {
                    if (action.equals("export")) {
                        List<Target> exportTargets = new ArrayList<Target>();
                        //              Page<Target> page = Target.pageTargets(0, pageSize, sort, order, query, curator, organisation, subject, crawlFrequency, depth, collection, license, flag);
                        // TODO: no time to do it now but this needs redoing with straight SQL/JPA to get the count
                        Page<Target> page = Target.pageTargets(pageNo, pageSize, sort, order, filter, curatorId,
                                organisationId, subjectSelect, crawlFrequencyName, depthName, collectionSelect,
                                licenseId, flagId);
                        int rowCount = page.getTotalRowCount();
                        //              Page<Target> pageAll = Target.pageTargets(0, rowCount, sort, order, query, curator, organisation, subject, crawlFrequency, depth, collection, license, flag);
                        Page<Target> pageAll = Target.pageTargets(pageNo, rowCount, sort, order, filter, curatorId,
                                organisationId, subjectSelect, crawlFrequencyName, depthName, collectionSelect,
                                licenseId, flagId);
                        exportTargets.addAll(pageAll.getList());
                        Logger.debug("export size: " + exportTargets.size());
                        String csvData = Utils.INSTANCE.export(exportTargets);
                        //              return redirect(routes.TargetController.list(pageNo, sort, order, query, curator, organisation,
                        //                    subject, crawlFrequency, depth, collection, license, pageSize, flag));

                        response().setContentType("text/csv; charset=utf-8");
                        response().setHeader("Content-disposition", "attachment; filename=\"target-export.csv");
                        return ok(csvData);
                        //               return redirect(routes.TargetController.list(pageNo, sort, order, filter, curatorId, organisationId, subjectSelect, crawlFrequencyName, depthName, collectionSelect, licenseId, pageSize, flagId));
                    } else {
                        if (action.equals("search") || action.equals("apply")) {
                            return redirect(routes.TargetController.list(pageNo, sort, order, filter, curatorId,
                                    organisationId, subjectSelect, crawlFrequencyName, depthName, collectionSelect,
                                    licenseId, pageSize, flagId));
                        } else {
                            return badRequest("This action is not allowed");
                        }
                    }
                }
            }
        }
    }

    /**
     * This method enables searching for given URL and redirection in order to add new entry
     * if required.
     *
     * @return
     */
    public static Result search() {

        DynamicForm form = DynamicForm.form().bindFromRequest();

        String action = form.get("action");
        String query = form.get("filter");
        int pageNo = Integer.parseInt(form.get("p"));
        String sort = form.get("s");
        String order = form.get("o");

        if (StringUtils.isEmpty(action)) {
            return badRequest("You must provide a valid action");
        } else {

            // check url

            Logger.debug("Query: " + query);
            boolean isValidUrl = Utils.INSTANCE.validUrl(query);

            Logger.debug("valid? " + isValidUrl);
            if (!isValidUrl) {
                ValidationError ve = new ValidationError("formUrl", "Invalid URL");
                form.reject(ve);
                flash("message",
                        "The URL entered is not valid. Please check and correct it, and click Search again");
                return redirect(routes.TargetController.lookup(pageNo, sort, order, query));
            }

            if (action.equals("add")) {
                return redirect(routes.TargetController.newForm(query));
            } else {
                if (action.equals("search")) {
                    Logger.debug("searching " + pageNo + " " + sort + " " + order);
                    return redirect(routes.TargetController.lookup(pageNo, sort, order, query));
                } else {
                    return badRequest("This action is not allowed");
                }
            }
        }
    }

    /**
     * Display the paginated list of targets.
     *
     * @param page           Current page number (starts from 0)
     * @param sortBy         Column to be sorted
     * @param order          Sort order (either asc or desc)
     * @param filter         Filter applied on target urls
     * @param collection_url Collection where targets search occurs
     */
    public static Result collectionTargets(int pageNo, int pageSize, String sortBy, String order, String filter,
            Long collectionId) {
        Logger.debug("Targets.collectionTargets()");
        Collection collection = Collection.findById(collectionId);
        Logger.debug("Collection: " + collection);
        Page<Target> pages = Target.pageCollectionTargets(pageNo, pageSize, sortBy, order, filter, collection.id);
        return ok(sites.render(collection, User.findByEmail(request().username()), filter, pages, sortBy, order,
                pageSize));
    }

    /**
     * @return
     */
    public static Result allTargetsIDsAsJson() {
        List<Target> targets = Target.findAllActive();
        Logger.debug("all targets: " + targets.size());
        List<Long> target_ids = new ArrayList<Long>(targets.size());
        for (Target t : targets) {
            target_ids.add(t.id);
        }
        return ok(Json.toJson(target_ids));
    }

    /**
     * Optional flag parameter - allow it to be overridden as a query parameter. Default is false.
     * If true (http://localhost:9000/act/api/targets?syncFlag=true),
     * then sync all active targets (includes runChecks and Update for each, may take more than 2hrs with 80K targets)
     *
     * @return
     */
    public static Result allTargetsAsJson(int pageNo, int pageLength, boolean flag) {
        List<Target> targets = Target.findAllActive();
        if (flag) {
            Logger.debug("Starting Sync for All active targets. Targets size = " + targets.size());
            // Transaction start
            Ebean.beginTransaction();
            try {
                targets.forEach(target -> {
                    target.runChecks();
                    target.update();
                });
                Ebean.commitTransaction();
            } finally {
                Ebean.endTransaction();
            }
            // Transaction end
            return ok("OK. Active Target Sync was done successfully.");
        } else { //flag == false
            int offset = pageNo * pageLength;
            if (offset > targets.size()) {
                return notFound("There are only " + targets.size() + " targets!");
            }
            List<Target> targets_page = new ArrayList<Target>(pageLength);
            for (int i = 0; i < pageLength; i++) {
                if (offset + i >= targets.size())
                    break;
                targets_page.add(targets.get(offset + i));
            }
            return ok(Json.toJson(targets_page));
        }
    }

    /**
     * @param collectionId
     * @return
     */
    public static Result collectionTargetsAsJson(Long collectionId) {
        Collection collection = Collection.findById(collectionId);
        if (collection != null) {
            List<Target> targets = Target.allCollectionTargets(collection.id);
            Logger.debug("collections targets: " + targets.size());
            List<Long> target_ids = new ArrayList<Long>(targets.size());
            for (Target t : targets) {
                target_ids.add(t.id);
            }
            return ok(Json.toJson(target_ids));
        } else {
            return notFound("There is not collection with ID " + collectionId);
        }
    }

    /**
     * Display the paginated list of targets.
     *
     * @param page        Current page number (starts from 0)
     * @param sortBy      Column to be sorted
     * @param order       Sort order (either asc or desc)
     * @param filter      Filter applied on target urls
     * @param subject_url Subject where targets search occurs
     */
    public static Result subjectTargets(int pageNo, String sortBy, String order, String filter, Long subjectId) {
        Logger.debug("Targets.subjectTargets()");
        return ok(views.html.subjects.sites.render(Subject.findById(subjectId),
                User.findByEmail(request().username()), filter,
                Target.pageSubjectTargets(pageNo, 10, sortBy, order, filter, subjectId), sortBy, order));
    }

    /**
     * Display the paginated list of targets for given organisation.
     *
     * @param page           Current page number (starts from 0)
     * @param sortBy         Column to be sorted
     * @param order          Sort order (either asc or desc)
     * @param filter         Filter applied on target urls
     * @param collection_url Collection where targets search occurs
     */
    public static Result organisationTargets(int pageNo, String sortBy, String order, String filter,
            Long organisationId) {
        Logger.debug("Targets.organisationTargets()");

        User user = User.findByEmail(request().username());

        Organisation organisation = Organisation.findById(organisationId);

        Page<Target> pageTargets = Target.pageOrganisationTargets(pageNo, 10, sortBy, order, filter,
                organisation.id);

        return ok(views.html.organisations.sites.render(organisation, user, filter, pageTargets, sortBy, order));
    }

    /**
     * This method enables searching for given URL and particular collection.
     *
     * @return
     */
    public static Result searchTargetsByCollection() {

        DynamicForm form = DynamicForm.form().bindFromRequest();
        String action = form.get("action");
        String query = form.get("url");

        int pageNo = Integer.parseInt(form.get(Const.PAGE_NO));
        int pageSize = Integer.parseInt(form.get(Const.PAGE_SIZE));

        String sort = form.get(Const.SORT_BY);
        String order = form.get(Const.ORDER);
        String collection_url = form.get(Const.COLLECTION_URL);

        Collection collection = Collection.findByUrl(collection_url);
        if (StringUtils.isEmpty(action)) {
            return badRequest("You must provide a valid action");
        } else {
            if (Const.SEARCH.equals(action)) {
                Logger.debug("searching " + pageNo + " " + pageSize + " " + sort + " " + order);
                return redirect(routes.TargetController.collectionTargets(pageNo, pageSize, sort, order, query,
                        collection.id));
            } else {
                return badRequest("This action is not allowed");
            }
        }
    }

    /**
     * This method enables searching for given URL and particular subject.
     *
     * @return
     */
    public static Result searchTargetsBySubject() {

        DynamicForm form = DynamicForm.form().bindFromRequest();
        String action = form.get("action");
        String query = form.get("url");

        int pageNo = Integer.parseInt(form.get(Const.PAGE_NO));
        String sort = form.get(Const.SORT_BY);
        String order = form.get(Const.ORDER);
        String subject_url = form.get("subject_url");
        Long subjectId = Long.valueOf(subject_url);

        if (StringUtils.isEmpty(action)) {
            return badRequest("You must provide a valid action");
        } else {
            if (Const.SEARCH.equals(action)) {
                Logger.debug("searching " + pageNo + " " + sort + " " + order);
                return redirect(routes.TargetController.subjectTargets(pageNo, sort, order, query, subjectId));
            } else {
                return badRequest("This action is not allowed");
            }
        }
    }

    /**
     * This method enables searching for given URL and particular organisation.
     *
     * @return
     */
    public static Result searchTargetsByOrganisation() {

        DynamicForm form = DynamicForm.form().bindFromRequest();
        String action = form.get(Const.ACTION);
        String query = form.get(Const.URL);

        int pageNo = Integer.parseInt(form.get(Const.PAGE_NO));
        String sort = form.get(Const.SORT_BY);
        String order = form.get(Const.ORDER);
        String organisation_id = form.get("organisation_id");
        Long organisationId = Long.valueOf(organisation_id);

        if (StringUtils.isEmpty(action)) {
            return badRequest("You must provide a valid action");
        } else {
            if (Const.SEARCH.equals(action)) {
                Logger.debug("searching " + pageNo + " " + sort + " " + order);
                return redirect(
                        routes.TargetController.organisationTargets(pageNo, sort, order, query, organisationId));
            } else {
                return badRequest("This action is not allowed");
            }
        }
    }

    /**
     * Display the paginated list of targets.
     *
     * @param page         Current page number (starts from 0)
     * @param sortBy       Column to be sorted
     * @param order        Sort order (either asc or desc)
     * @param filter       Filter applied on target urls
     * @param user_url     User for whom targets search occurs
     * @param fastSubjects Taxonomy of type subject
     * @param collection   Taxonomy of type collection
     */
    public static Result userTargets(int pageNo, String sortBy, String order, String filter, Long userId,
            Long subjectId, Long collectionId) {
        User curator = User.findById(userId);
        User user = User.findByEmail(request().username());

        Page<Target> pageTargets = Target.pageUserTargets(pageNo, 10, sortBy, order, filter, userId, subjectId,
                collectionId);

        List<Subject> subjects = Subject.findAllSubjects();
        List<Collection> collections = Collection.findAllCollections();

        Logger.debug("Targets.collectionTargets() " + userId + ", " + subjectId + ", " + collectionId);

        return ok(usersites.render(curator, user, filter, pageTargets, sortBy, order, subjectId, collectionId,
                subjects, collections));
    }

    /**
     * This method enables searching for given URL and particular user.
     *
     * @return
     */
    public static Result searchTargetsByUser() {

        DynamicForm requestData = DynamicForm.form().bindFromRequest();
        String action = requestData.get("action");
        String query = requestData.get("url");

        String user_id = requestData.get("userId");
        Long userId = Long.valueOf(user_id);

        int pageNo = Integer.parseInt(requestData.get("p"));
        String sort = requestData.get("s");
        String order = requestData.get("o");

        String subject = requestData.get("subjectId");
        Long subjectId = null;
        if (StringUtils.isNotBlank(subject)) {
            subjectId = Long.valueOf(subject);
        }

        String collection = requestData.get("collectionId");
        Long collectionId = null;
        if (StringUtils.isNotBlank(collection)) {
            collectionId = Long.valueOf(collection);
        }

        //       if (StringUtils.isBlank(query)) {
        //         Logger.debug("Target name is empty. Please write name in search window.");
        //         flash("message", "Please enter a name in the search window");
        //           return redirect(routes.Targets.userTargets(pageNo, sort, order, query, user_url, subject, collection));
        //       }       

        if (StringUtils.isEmpty(action)) {
            return badRequest("You must provide a valid action");
        } else {
            if (action.equals("add")) {
                String title = requestData.get("filter");
                return redirect(routes.TargetController.newForm(title));
            }
            if (action.equals("search")) {
                Logger.debug("searching " + pageNo + " " + sort + " " + order + " " + userId + ", " + subjectId
                        + ", " + collectionId);
                return redirect(routes.TargetController.userTargets(pageNo, sort, order, query, userId, subjectId,
                        collectionId));
            } else {
                return badRequest("This action is not allowed");
            }
        }
    }

    public static Result GO_HOME = redirect(
            routes.TargetController.list(0, "title", "asc", "", 0, 0, "", "", "", "", 0, 10, 0));

    public static Result newForm(String title) {
        User user = User.findByEmail(request().username());
        Form<Target> filledForm = Form.form(Target.class);
        Target target = new Target();
        target.setDefaultValues();
        if (StringUtils.isNotBlank(title)) {
            try {
                new URL(title);
                target.formUrl = title;
            } catch (MalformedURLException e) {
                target.title = title;
            }
        }
        target.revision = Const.INITIAL_REVISION;
        target.active = Boolean.TRUE;

        target.authorUser = user;
        target.organisation = user.organisation;
        target.language = Const.TargetLanguage.EN.toString();

        Logger.debug("add target with url: " + target.url);
        Logger.debug("target title: " + target.title);

        filledForm = filledForm.fill(target);

        JsonNode collectionData = getCollectionsData();
        JsonNode subjectData = getSubjectsData();

        Map<String, String> authors = User.options();
        List<Tag> tags = Tag.findAllTags();
        List<Flag> flags = Flag.findAllFlags();
        Map<String, String> qaIssues = QaIssue.options();
        Map<String, String> languages = Const.TargetLanguage.options();
        Map<String, String> selectionTypes = Const.SelectionType.options();
        Map<String, String> scopeTypes = Const.ScopeType.options();
        Map<String, String> depthTypes = Const.DepthType.options();
        List<License> licenses = License.findAllLicenses();
        Map<String, String> licenseStatuses = License.LicenseStatus.options();
        Map<String, String> crawlFrequencies = Const.CrawlFrequency.options();
        Map<String, String> siteStatuses = Const.SiteStatus.options();
        Map<String, String> organisations = Organisation.options();

        return ok(edit.render(filledForm, user, null, collectionData, subjectData, authors, tags, flags, qaIssues,
                languages, selectionTypes, scopeTypes, depthTypes, licenses, licenseStatuses, crawlFrequencies,
                siteStatuses, organisations, null, null, null, null));
    }

    /**
     * Display the target edit panel for this URL.
     *
     * @param url The target identifier URL
     */
    public static Result edit(Long id) {
        Logger.debug("Targets.edit() id::::: " + id);

        Target target = Target.findById(id);
        if (target == null) {
            return notFound("There is no Target with ID " + id);
        }

        // Make sure scope checks are up to date:
        target.runChecks();
        target.update();

        target.formUrl = target.fieldUrl();
        target.subjectSelect = target.subjectIdsAsString();
        target.collectionSelect = target.collectionIdsAsString();
        Form<Target> filledForm = targetForm.fill(target);
        if (target.watchedTarget != null) {
            filledForm.data().putAll(FastSubjects.getFormData(target.watchedTarget.fastSubjects));
        }
        User user = User.findByEmail(request().username());
        target.language = Const.TargetLanguage.EN.toString();
        JsonNode collectionData = getCollectionsData(target.collections);
        JsonNode subjectData = getSubjectsData(target.subjects);

        Map<String, String> authors = User.options();
        List<Tag> tags = Tag.findAllTags();
        List<Tag> targetTags = target.tags;
        List<Flag> flags = Flag.findAllFlags();
        List<Flag> targetFlags = target.flags;
        Map<String, String> qaIssues = QaIssue.options();
        Map<String, String> languages = Const.TargetLanguage.options();
        Map<String, String> selectionTypes = Const.SelectionType.options();
        Map<String, String> scopeTypes = Const.ScopeType.options();
        Map<String, String> depthTypes = Const.DepthType.options();
        List<License> licenses = License.findAllLicenses();
        List<License> targetLicenses = target.licenses;
        Map<String, String> licenseStatuses = License.LicenseStatus.options();
        Map<String, String> crawlFrequencies = Const.CrawlFrequency.options();
        Map<String, String> siteStatuses = Const.SiteStatus.options();
        Map<String, String> organisations = Organisation.options();
        target.fieldUrl = target.fieldUrl();
        //      Logger.debug("DATE::::::::::::::::::::::::::::: "+filledForm.get().crawlEndDate.toString());
        //      DateFormat formatter = new SimpleDateFormat("dd-MM-yyyy HH:mm");
        //      formatter.setTimeZone(TimeZone.getTimeZone("UTC"));
        //      try {
        //         Date date = formatter.parse(filledForm.get().crawlEndDate.toString());
        //         filledForm.get().crawlEndDate = date;
        //      } catch (ParseException e) {
        //         e.printStackTrace();
        //      }

        Logger.debug("collections: " + target.collections.size());
        return ok(edit.render(filledForm, user, id, collectionData, subjectData, authors, tags, flags, qaIssues,
                languages, selectionTypes, scopeTypes, depthTypes, licenses, licenseStatuses, crawlFrequencies,
                siteStatuses, organisations, null, targetTags, targetFlags, targetLicenses));
    }

    public static Result delete(Long id) {
        Target target = Target.findById(id);

        Form<Target> filledForm = Form.form(Target.class);
        filledForm = filledForm.fill(target);

        if (!target.isDeletable()) {
            ValidationError ve = new ValidationError("formUrl",
                    "Unable to delete Target as it references Instance(s), License(s) and/or Collection(s)");
            filledForm.reject(ve);
            return info(filledForm, id);
        }

        if (target.hasDocuments()) {
            ValidationError ve = new ValidationError("watched",
                    "Watched Targets with existing crawled documents can not be deleted.");
            filledForm.reject(ve);
            return info(filledForm, id);
        }
        if (target.watchedTarget != null) {
            //Ebean.delete(target.watchedTarget.documents);
            Ebean.delete(target.watchedTarget.journalTitles);
        }
        target.delete();
        return redirect(routes.TargetController.index());
    }

    /**
     * This method shows selected revision of a Target by given ID.
     *
     * @param id
     * @return
     */
    public static Result viewrevision(Long id) {
        Target target = Target.findById(id);
        User user = User.findByEmail(request().username());
        return ok(view.render(target, user));
    }

    /**
     * Lists IDs for all targets of a given frequency, no matter what the status/schedule:
     *
     * @param frequency
     * @return
     */
    @BodyParser.Of(BodyParser.Json.class)
    public static Result idsForFrequencyJson(String frequency) {
        JsonNode jsonData = null;
        if (frequency != null) {
            List<Target> targets = new ArrayList<Target>();
            targets = Target.getByFrequency(frequency);
            List<Long> targetIds = new ArrayList<Long>();
            for (Target t : targets) {
                targetIds.add(t.id);
            }
            jsonData = Json.toJson(targetIds);
        }
        return ok(jsonData);
    }

    /**
     * This method provides data exports for each possible crawl-frequency.
     * For each frequency this contains a list of Targets and associated
     * crawl metadata.
     *
     * @param frequency The crawl frequency e.g. 'daily'
     * @return list of Target objects
     */
    @BodyParser.Of(BodyParser.Json.class)
    public static Result exportByFrequencyJson(String frequency) {
        JsonNode jsonData = null;
        if (frequency != null) {
            List<Target> targets = new ArrayList<Target>();
            targets = Target.exportByFrequency(frequency);
            jsonData = Json.toJson(targets);
        }
        return ok(jsonData);
    }

    /**
     * As above, but only lists IDs
     *
     * @param frequency
     * @return
     */
    @BodyParser.Of(BodyParser.Json.class)
    public static Result idsToCrawlByFrequencyJson(String frequency) {
        JsonNode jsonData = null;
        if (frequency != null) {
            List<Target> targets = new ArrayList<Target>();
            targets = Target.exportByFrequency(frequency);
            List<Long> targetIds = new ArrayList<Long>();
            for (Target t : targets) {
                targetIds.add(t.id);
            }
            jsonData = Json.toJson(targetIds);
        }
        return ok(jsonData);
    }

    /**
     * @param frequency
     * @return
     */
    @BodyParser.Of(BodyParser.Json.class)
    public static Result crawlFeedByFrequencyJson(String frequency) {
        JsonNode jsonData = null;
        if (frequency != null) {
            List<Target> targets = new ArrayList<Target>();
            targets = Target.exportByFrequency(frequency);
            List<CrawlFeedItem> targetIds = new ArrayList<CrawlFeedItem>();
            for (Target t : targets) {
                targetIds.add(new CrawlFeedItem(t));
            }
            jsonData = Json.toJson(targetIds);
        }
        return ok(jsonData);
    }

    /**
     * @param frequency
     * @return
     */
    @BodyParser.Of(BodyParser.Json.class)
    public static Result crawlFeedOAFrequencyJson(String frequency) {
        JsonNode jsonData = null;
        if (frequency != null) {
            List<Target> targets = new ArrayList<Target>();
            targets = Target.exportOAFrequency(frequency);
            List<CrawlFeedItem> targetIds = new ArrayList<CrawlFeedItem>();
            for (Target t : targets) {
                targetIds.add(new CrawlFeedItem(t));
            }
            jsonData = Json.toJson(targetIds);
        }
        return ok(jsonData);
    }

    /**
     * This method provides data exports for each possible crawl-frequency that are in legal deposit.
     * For each frequency this contains a list of Targets and associated
     * crawl metadata.
     *
     * @param frequency The crawl frequency e.g. 'daily'
     * @return list of Target objects
     * @throws ActException
     */
    @BodyParser.Of(BodyParser.Json.class)
    public static Result exportLdFrequencyJson(String frequency) throws ActException {
        JsonNode jsonData = null;
        if (frequency != null) {
            List<Target> targets = new ArrayList<Target>();
            targets = Target.exportLdFrequency(frequency);
            jsonData = Json.toJson(targets);
        }
        return ok(jsonData);
    }

    /**
     * As above, but only lists IDs.
     *
     * @param frequency
     * @return
     * @throws ActException
     */
    @BodyParser.Of(BodyParser.Json.class)
    public static Result idsToCrawlLdFrequencyJson(String frequency) {
        JsonNode jsonData = null;
        if (frequency != null) {
            List<Target> targets = new ArrayList<Target>();
            targets = Target.exportLdFrequency(frequency);
            List<Long> targetIds = new ArrayList<Long>();
            for (Target t : targets) {
                targetIds.add(t.id);
            }
            jsonData = Json.toJson(targetIds);
        }
        return ok(jsonData);
    }

    /**
     * @param frequency
     * @param whether to generate the paywall feed or not - i.e. this can exclude paywalled Targets, or be composed only of paywalled Targets.
     * @return
     * @throws ActException
     */
    @BodyParser.Of(BodyParser.Json.class)
    public static Result crawlFeedLdFrequencyJson(String frequency, boolean generatePaywallFeed) {
        JsonNode jsonData = null;
        if (frequency != null) {
            List<Target> targets = new ArrayList<Target>();
            targets = Target.exportLdFrequency(frequency);
            List<CrawlFeedItem> targetIds = new ArrayList<CrawlFeedItem>();
            for (Target t : targets) {
                // Only add 
                if (!generatePaywallFeed && t.secretId == null) {
                    targetIds.add(new CrawlFeedItem(t));
                } else if (generatePaywallFeed && t.secretId != null) {
                    targetIds.add(new CrawlFeedItem(t));
                }
            }
            jsonData = Json.toJson(targetIds);
        }
        return ok(jsonData);
    }

    /**
     * Example form with validation
     *
     * @return blank form for data entry
     */
    public static Result blank() {
        Logger.debug("blank()");
        return ok(blank.render(targetForm, User.findByEmail(request().username())));
    }

    public static Result saveBlank() {
        Form<Target> filledForm = targetForm.bindFromRequest();
        if (filledForm.hasErrors()) {
            Logger.debug("hasErrors: " + filledForm.hasErrors());
            for (String key : filledForm.errors().keySet()) {
                Logger.debug("" + key);
            }
            return badRequest(blank.render(filledForm, User.findByEmail(request().username())));
        } else {
            flash("success", "You've saved");
            Logger.debug("saved");
            return ok(blank.render(filledForm, User.findByEmail(request().username())));
        }
    }

    /**
     * This method indicates to the user in a target record if data has been entered
     * by other users relating to NPLD status in another target record at a higher
     * level in the domain.
     * This is to avoid duplication of effort: users should not need to spend time
     * (outside ACT) doing the necessary research to populate the 'UK Postal Address',
     * 'Via Correspondence', and 'Professional Judgment' fields for abc.co.uk/directory
     * if those fields are already populated in a target record for abc.co.uk
     * @param fieldUrl The target URL
     * @return result as a flag. Result is true if:
     *          (i) one or more of the three fields named above is not null in any other
     *             target record at a higher level within the same domain AND
     *         (ii) where both 'UK hosting' and 'UK top-level domain' = No.
     */
    //    public static boolean indicateNpldStatus(String fieldUrl) {
    //       boolean res = false;
    //       if (Target.getNpldStatusList(fieldUrl).size() > 0) {
    //          res = true;
    //       }
    //       Logger.debug("indicateNpldStatus() res: " + res);
    //       return res;
    //    }

    /**
     * This method should give a list of the Target Titles and URLs for the
     * first three examples, in descending order of date of creation of the record.
     *
     * @param fieldUrl The target URL
     * @return result as a string
     */
    public static String showNpldStatusList(String fieldUrl) {
        String res = "";
        //        try {
        //            StringBuilder sb = new StringBuilder();
        //           List<Target> targets = Target.getNpldStatusList(fieldUrl);
        //           Iterator<Target> itr = targets.iterator();
        //           while (itr.hasNext()) {
        //              Target target = itr.next();
        //              sb.append(target.title + " " + target.fieldUrl());
        //                sb.append(System.getProperty("line.separator"));
        //           }
        //            res = sb.toString();
        //       } catch (Exception e) {
        //            Logger.error("showNpldStatusList() " + e.getMessage());
        //        }
        return res;
    }

    /**
     * This method prepares Target form for sending info message
     * about errors
     *
     * @return edit page with form and info message
     */
    public static Result info(Form<Target> form, Long id) {

        User user = User.findByEmail(request().username());

        Map<String, String> authors = User.options();
        List<Tag> tags = Tag.findAllTags();
        List<Flag> flags = Flag.findAllFlags();
        Map<String, String> qaIssues = QaIssue.options();
        Map<String, String> languages = Const.TargetLanguage.options();
        Map<String, String> selectionTypes = Const.SelectionType.options();
        Map<String, String> scopeTypes = Const.ScopeType.options();
        Map<String, String> depthTypes = Const.DepthType.options();
        List<License> licenses = License.findAllLicenses();
        Map<String, String> licenseStatuses = License.LicenseStatus.options();
        Map<String, String> crawlFrequencies = Const.CrawlFrequency.options();
        Map<String, String> siteStatuses = Const.SiteStatus.options();
        Map<String, String> organisations = Organisation.options();

        DynamicForm requestData = Form.form().bindFromRequest();
        String tabStatus = requestData.get("tabstatus");

        JsonNode collectionData = null;
        JsonNode subjectData = null;
        List<Tag> targetTags = null;
        List<Flag> targetFlags = null;
        List<License> targetLicenses = null;

        if (!form.hasErrors()) {
            Target target = form.get();

            collectionData = getCollectionsData(target.collections);
            subjectData = getSubjectsData(target.subjects);
            targetTags = target.tags;
            targetFlags = target.flags;
            targetLicenses = target.licenses;

        }

        return badRequest(edit.render(form, user, id, collectionData, subjectData, authors, tags, flags, qaIssues,
                languages, selectionTypes, scopeTypes, depthTypes, licenses, licenseStatuses, crawlFrequencies,
                siteStatuses, organisations, tabStatus, targetTags, targetFlags, targetLicenses));
    }

    private static String urlNoTrailingSlash(String url) {
        if (url.endsWith("/")) {
            url = url.substring(0, url.length() - 1);
        }
        return url;
    }

    /**
     * @param requestData
     * @param id
     * @return
     */
    private static Result validateForm(DynamicForm requestData, Long id) throws ActException {
        Map<String, String[]> formParams = request().body().asFormUrlEncoded();
        Form<Target> filledForm = form(Target.class).bindFromRequest();
        User currentUser = User.findByEmail(request().username());
        Target original = Target.findById(id);
        Logger.debug("hasGlobalErrors: " + filledForm.hasGlobalErrors());

        if (filledForm.hasErrors()) {
            Logger.debug("hasErrors: " + filledForm.errors());
            return info(filledForm, id);
        }

        String title = requestData.get("title");

        if (title.trim().length() == 0) {

            flash("message", "Blank Title");
            return info(filledForm, id);
        }

        String wctId = requestData.get("wctId");

        if (StringUtils.isNotBlank(wctId) && !Utils.INSTANCE.isNumeric(wctId)) {
            flash("message", "Only numeric values are valid identifiers. Please check field 'WCT ID'.");
            return info(filledForm, id);
        }

        String sptId = requestData.get("sptId");

        if (StringUtils.isNotBlank(sptId) && !Utils.INSTANCE.isNumeric(sptId)) {
            Logger.debug("Only numeric values are valid identifiers. Please check field 'SPT ID'.");
            flash("message", "Only numeric values are valid identifiers. Please check field 'SPT ID'.");
            return info(filledForm, id);
        }

        String legacySiteId = requestData.get("legacySiteId");
        if (StringUtils.isNotBlank(legacySiteId) && !Utils.INSTANCE.isNumeric(legacySiteId)) {
            Logger.debug("Only numeric values are valid identifiers. Please check field 'LEGACY SITE ID'.");
            flash("message", "Only numeric values are valid identifiers. Please check field 'LEGACY SITE ID'.");
            return info(filledForm, id);
        }

        String author = requestData.get("authorUser.id");
        if (StringUtils.isBlank(author)) {
            flash("message", "Please choose a Selector");
            return info(filledForm, id);
        }

        List<License> newLicenses = new ArrayList<License>();
        String[] licenseValues = formParams.get("licensesList");

        // if it was originally set then let it go
        String openUkwa = requestData.get("openUkwaLicense");

        if (licenseValues != null) {
            for (String licenseValue : licenseValues) {
                try {
                    Long licenseId = Long.valueOf(licenseValue);
                    License license = License.findById(licenseId);
                    // could just use the ID instead
                    if (StringUtils.isEmpty(openUkwa) && license.name.equals(Const.OPEN_UKWA_LICENSE)) {
                        ValidationError ve = new ValidationError("licensesList",
                                "It is not possible to attach an Open UKWA Licence directly to a target in this way. Please initiate the licensing process using the green button below");
                        filledForm.reject(ve);
                        return info(filledForm, id);
                    }
                    newLicenses.add(license);
                } catch (NumberFormatException e) {
                    Logger.debug("No license selected for " + filledForm.get().title);
                }
            }
            filledForm.get().licenses = newLicenses;
        }

        String webFormDate = requestData.get("webFormDateText");
        if (StringUtils.isNotEmpty(webFormDate)) {
            DateFormat formatter = new SimpleDateFormat("dd-MM-yyyy");
            formatter.setTimeZone(TimeZone.getTimeZone("UTC"));
            try {
                Date date = formatter.parse(webFormDate);
                filledForm.get().webFormDate = date;
                Logger.debug("webFormDate:::::::: " + date);
            } catch (ParseException e) {
                e.printStackTrace();
                return info(filledForm, id);
            }
        } else {
            if (id != null) {
                Ebean.createUpdate(Target.class, "update target SET web_form_date=null where id=:id")
                        .setParameter("id", id).execute();
            }
        }

        String crawlFrequency = filledForm.get().crawlFrequency;

        if (StringUtils.isNotEmpty(crawlFrequency)) {
            String crawlStartDate = requestData.get("crawlStartDateText");

            if (StringUtils.isNotEmpty(crawlStartDate)) {
                DateFormat formatter = new SimpleDateFormat("dd-MM-yyyy HH:mm");
                formatter.setTimeZone(TimeZone.getTimeZone("UTC"));
                try {
                    Date date = formatter.parse(crawlStartDate);
                    filledForm.get().crawlStartDate = date;
                    Logger.debug("crawlStartDate:::::::: " + date);
                    if (date.before(Calendar.getInstance().getTime())) {
                        flash("warning",
                                "<b>Warning! The crawl start date is in the past!</b><br/>This is normal for existing targets, but should not be the case for new targets.");
                    }
                } catch (ParseException e) {
                    e.printStackTrace();
                    return info(filledForm, id);
                }
            } else {
                if (id != null) {
                    Ebean.createUpdate(Target.class, "update target SET crawl_start_date=null where id=:id")
                            .setParameter("id", id).execute();
                }
            }

            Logger.debug("crawl frequency: " + crawlFrequency);
            Logger.debug("crawlStartDate: " + crawlStartDate);

            if (!CrawlFrequency.DOMAINCRAWL.name().equals(crawlFrequency) && StringUtils.isEmpty(crawlStartDate)) {
                ValidationError ve = new ValidationError("crawlStartDateText",
                        "Start Date is required when any crawl frequency other than 'Domain Crawl Only' is selected");
                filledForm.reject(ve);
                return info(filledForm, id);
            }

            // Don't let non-archivists add or remove NEVERCRAWL status:
            if (original != null && CrawlFrequency.NEVERCRAWL.name().equals(original.crawlFrequency)) {
                // Do not remove:
                if ((!CrawlFrequency.NEVERCRAWL.name().equals(crawlFrequency))
                        && !(currentUser.isArchivist() || currentUser.isSysAdmin())) {
                    ValidationError ve = new ValidationError("crawlFrequency",
                            "Only an archivist can change the crawl frequency from 'never crawl' to something else.");
                    filledForm.reject(ve);
                    return info(filledForm, id);
                }
            } else {
                // Do not add:
                if ((CrawlFrequency.NEVERCRAWL.name().equals(crawlFrequency))
                        && !(currentUser.isArchivist() || currentUser.isSysAdmin())) {
                    ValidationError ve = new ValidationError("crawlFrequency",
                            "Only an archivist can change the crawl frequency to 'never crawl'.");
                    filledForm.reject(ve);
                    return info(filledForm, id);
                }
            }

            String crawlEndDate = requestData.get("crawlEndDateText");
            if (StringUtils.isNotEmpty(crawlEndDate)) {
                DateFormat formatter = new SimpleDateFormat("dd-MM-yyyy HH:mm");
                formatter.setTimeZone(TimeZone.getTimeZone("UTC"));
                try {
                    Date date = formatter.parse(crawlEndDate);
                    filledForm.get().crawlEndDate = date;
                    Logger.debug("crawlEndDate in date:::::::: " + date);
                } catch (ParseException e) {
                    e.printStackTrace();
                    return info(filledForm, id);
                }
            } else {
                if (id != null) {
                    Ebean.createUpdate(Target.class, "update target SET crawl_end_date=null where id=:id")
                            .setParameter("id", id).execute();
                }
            }
        }

        List<Tag> newTags = new ArrayList<Tag>();
        String[] tagValues = formParams.get("tagsList");

        if (original != null && tagValues != null) {
            for (String tagValue : tagValues) {
                Long tagId = Long.valueOf(tagValue);
                Tag tag = Tag.findTagById(tagId);
                newTags.add(tag);
                if (!original.tags.contains(tag)) {
                    original.tags.add(tag);
                }
            }
            Ebean.update(original);
            filledForm.get().tags = newTags;
        } else {
            if (original != null && tagValues == null) {
                original.tags.clear();
                Ebean.update(original);
            }
        }

        String[] flagValues = formParams.get("flagsList");

        if (original != null && flagValues != null) {
            List<Flag> newFlags = new ArrayList<>();
            for (String flagValue : flagValues) {
                Long flagId = Long.valueOf(flagValue);
                Flag flag = Flag.findById(flagId);
                newFlags.add(flag);
                if (!original.flags.contains(flag)) {
                    original.flags.add(flag);
                }
            }

            for (Iterator<Flag> it = original.flags.iterator(); it.hasNext();) {
                if (!newFlags.contains(it.next())) {
                    it.remove();
                }
            }

            Ebean.update(original);
            filledForm.get().flags = null;
        } else {
            if (original != null && flagValues == null) {
                original.flags.clear();
                Ebean.update(original);
            }
        }

        List<Subject> newSubjects = new ArrayList<Subject>();
        String subjectSelect = requestData.get("subjectSelect").replace("\"", "");
        Logger.debug("subjectSelect: " + subjectSelect);
        if (StringUtils.isNotEmpty(subjectSelect)) {
            String[] subjects = subjectSelect.split(", ");
            for (String sId : subjects) {
                Long subjectId = Long.valueOf(sId);
                Subject subject = Subject.findById(subjectId);
                if (subject.parent != null) {
                    newSubjects = processParentsSubjects(newSubjects, subject.parent.id);
                }
                if (!newSubjects.contains(subject)) {
                    newSubjects.add(subject);
                }
            }
        }
        filledForm.get().subjects = newSubjects;

        List<Collection> newCollections = new ArrayList<Collection>();
        String collectionSelect = requestData.get("collectionSelect").replace("\"", "");
        Logger.debug("collectionSelect: " + collectionSelect);
        if (StringUtils.isNotEmpty(collectionSelect)) {
            String[] collections = collectionSelect.split(", ");
            for (String cId : collections) {
                Long collectionId = Long.valueOf(cId);
                Collection collection = Collection.findById(collectionId);
                if (collection.parent != null) {
                    newCollections = processParentsCollections(newCollections, collection.parent.id);
                }
                if (!newCollections.contains(collection)) {
                    newCollections.add(collection);
                }
            }
        }
        filledForm.get().collections = newCollections;

        String dateOfPublication = requestData.get("dateOfPublicationText");
        if (StringUtils.isNotEmpty(dateOfPublication)) {
            DateFormat formatter = new SimpleDateFormat("dd-MM-yyyy");
            formatter.setTimeZone(TimeZone.getTimeZone("UTC"));
            try {
                Date date = formatter.parse(dateOfPublication);
                filledForm.get().dateOfPublication = date;
            } catch (ParseException e) {
                e.printStackTrace();
                return info(filledForm, id);
            }
        }

        // Perform NPLD field consistency checks...
        // ---
        // UK Postal Address
        if (StringUtils.isEmpty(filledForm.get().ukPostalAddressUrl)) {
            if (Boolean.TRUE.equals(filledForm.get().ukPostalAddress)) {
                ValidationError ve = new ValidationError("ukPostalAddressUrl",
                        "A URL indicating the UK Postal Address must be provided!");
                filledForm.reject(ve);
                return info(filledForm, id);
            } else {
                filledForm.get().ukPostalAddress = false;
            }
        } else {
            // Auto-tick the checkbox
            filledForm.get().ukPostalAddress = true;

        }
        // via correspondence (which is apparently stored in the 'value' field!)
        if (StringUtils.isEmpty(filledForm.get().value)) {
            if (Boolean.TRUE.equals(filledForm.get().viaCorrespondence)) {
                ValidationError ve = new ValidationError("value",
                        "A description of the correspondence must be provided!");
                filledForm.reject(ve);
                return info(filledForm, id);
            } else {
                filledForm.get().viaCorrespondence = false;
            }
        } else {
            // Auto-tick the checkbox
            filledForm.get().viaCorrespondence = true;
        }
        // professional judgement
        if (StringUtils.isEmpty(filledForm.get().professionalJudgementExp)) {
            if (Boolean.TRUE.equals(filledForm.get().professionalJudgement)) {
                ValidationError ve = new ValidationError("professionalJudgementExp",
                        "An outline of your professional judgement must be provided!");
                filledForm.reject(ve);
                return info(filledForm, id);
            } else {
                filledForm.get().professionalJudgement = false;
            }
        } else {
            // Auto-tick the checkbox
            filledForm.get().professionalJudgement = true;
        }

        //Hidden flag
        if (filledForm.get().hidden == null) {
            filledForm.get().hidden = Boolean.FALSE;
        }
        //Key Site flag
        if (filledForm.get().keySite == null) {
            filledForm.get().keySite = Boolean.FALSE;
        }

        // Ensure items NOT edited herein are re-attached:
        if (original != null) {
            filledForm.get().crawlPermissions = original.crawlPermissions;
            Logger.warn("Attempting to repair " + original.crawlPermissions + "(" + original.crawlPermissions.size()
                    + ")");
            filledForm.get().instances = original.instances;
            filledForm.get().lookupEntries = original.lookupEntries;
        }

        // Run scoping checks:
        filledForm.get().runChecks();

        // noLdCriteriaMet
        Logger.debug("noLdCriteriaMet: " + filledForm.get().noLdCriteriaMet);
        if (!Boolean.TRUE.equals(filledForm.get().noLdCriteriaMet)) {//Check form noLdCriteriaMet field value
            //if FALSE or NULL
            filledForm.get().noLdCriteriaMet = Boolean.FALSE;
        }
        /*
        // Check if those checks invalidate the noLDmet:
        else if((Boolean.TRUE.equals(filledForm.get().isUkHosting) ||
            Boolean.TRUE.equals(filledForm.get().isTopLevelDomain) ||
            Boolean.TRUE.equals(filledForm.get().isUkRegistration) ||
            Boolean.TRUE.equals(filledForm.get().ukPostalAddress) ||
            Boolean.TRUE.equals(filledForm.get().viaCorrespondence) ||
            Boolean.TRUE.equals(filledForm.get().professionalJudgement))
            && Boolean.TRUE.equals(filledForm.get().noLdCriteriaMet) //Check form value
            ) {
        ValidationError ve = new ValidationError("noLdCriteriaMet", "One of the checks for NPLD permission has been passed. Please unselect the 'No LD Criteria Met' field and click Save again");
        filledForm.reject(ve);
        return info(filledForm, id);
        }
        else if(filledForm.get().noLdCriteriaMet == null) {
        filledForm.get().noLdCriteriaMet = Boolean.FALSE;
        //}
        */

        //Updating licence status
        // ANJ: Note that manual changes at this top-level should not modify the existing CrawlPermissions.
        // Commenting out until the needs are more clearly defined.
        /*
        if (filledForm.get().getLatestCrawlPermission() != null) {
           filledForm.get().getLatestCrawlPermission().status = filledForm.get().licenseStatus;
           if( Const.CrawlPermissionStatus.GRANTED.getValue().equalsIgnoreCase(filledForm.get().getLatestCrawlPermission().status)){
          filledForm.get().getLatestCrawlPermission().grantedAt = Utils.INSTANCE.getCurrentTimeStamp();
           }
           if( Const.CrawlPermissionStatus.PENDING.getValue().equalsIgnoreCase(filledForm.get().getLatestCrawlPermission().status)){
          filledForm.get().getLatestCrawlPermission().requestedAt = Utils.INSTANCE.getCurrentTimeStamp();
           }
           filledForm.get().getLatestCrawlPermission().update();
        }
        */

        Result checkUrlsResult = validateUrls(requestData, id, filledForm);

        if (checkUrlsResult != null) {
            return checkUrlsResult;
        }

        filledForm.get().runChecks();

        // Transaction start
        Ebean.beginTransaction();
        try {
            // Lock the child field_url table to guard against a race condition that can result in duplicate URLs being created
            Ebean.createSqlUpdate("LOCK TABLE field_url;").execute();
            Result checkUniqueUrlResult = validateCheckForExistingUrl(id, filledForm);

            if (checkUniqueUrlResult != null) {
                return checkUniqueUrlResult;
            }

            if (id != null) {
                filledForm.get().update(id);
            } else {
                filledForm.get().url = "act-" + Utils.INSTANCE.createId();
                filledForm.get().active = Boolean.TRUE;
                filledForm.get().save();
            }

            Ebean.commitTransaction();
        } finally {
            Ebean.endTransaction();
        }
        // Transaction end

        // The watchedTarget is not cascaded automatically, so we handle it here:
        String watchedString = getFormParam("watched");
        boolean watched = false;
        if (watchedString != null) {
            watched = Boolean.parseBoolean(watchedString);
        }
        Logger.info("WATCHED status is " + watched + " from " + watchedString);
        if (original != null) {
            if (!watched && original.isWatched()) {
                Ebean.delete(original.watchedTarget.documents);
                Ebean.delete(original.watchedTarget.journalTitles);
                Ebean.delete(original.watchedTarget);
            } else {
                if (watched && !original.isWatched()) {
                    filledForm.get().watchedTarget.target = original;
                    filledForm.get().watchedTarget.fastSubjects = FastSubjects.getFastSubjects(filledForm);
                    Ebean.save(filledForm.get().watchedTarget);
                } else {
                    if (watched && original.isWatched()) {
                        original.watchedTarget.documentUrlScheme = filledForm.get().watchedTarget.documentUrlScheme;
                        original.watchedTarget.archivistNotesWT = filledForm.get().watchedTarget.archivistNotesWT;
                        original.watchedTarget.fastSubjects = FastSubjects.getFastSubjects(filledForm);
                        Ebean.update(original.watchedTarget);
                    }
                }
            }
        }

        if (id != null) {
            flash("message", "Target " + filledForm.get().title + " has been updated");
        } else {
            flash("success", "Target " + filledForm.get().title + " has been created");
        }
        return redirect(routes.TargetController.view(filledForm.get().id));
    }

    private static Result validateUrls(DynamicForm requestData, Long id, Form<Target> filledForm)
            throws ActException {
        String fieldUrl = requestData.get("formUrl");

        Logger.debug("\n\nfieldUrl: " + fieldUrl);
        if (StringUtils.isNotEmpty(fieldUrl)) {
            String[] urls = fieldUrl.split(",");
            List<FieldUrl> fieldUrls = new ArrayList<FieldUrl>();

            long position = 0;

            for (String url : urls) {

                String trimmed = url.trim();

                Logger.debug("trimmed " + trimmed);
                URL uri;
                try {
                    uri = new URI(trimmed).normalize().toURL();
                } catch (MalformedURLException | URISyntaxException | IllegalArgumentException e) {
                    ValidationError ve = new ValidationError("formUrl",
                            "The URL entered is not valid. Please check and correct it, and click Save again");
                    filledForm.reject(ve);
                    return info(filledForm, id);
                }

                UrlValidator urlValidator = new UrlValidator();
                if (!urlValidator.isValid(trimmed)) {
                    ValidationError ve = new ValidationError("formUrl",
                            "The URL entered is not valid. Please check and correct it, and click Save again");
                    filledForm.reject(ve);
                    return info(filledForm, id);
                }

                String extFormUrl = uri.toExternalForm();
                FieldUrl fu = new FieldUrl(extFormUrl.trim());
                boolean isValidUrl = Utils.INSTANCE.validUrl(trimmed);
                Logger.debug("valid? " + isValidUrl);
                if (!isValidUrl) {
                    ValidationError ve = new ValidationError("formUrl",
                            "The URL entered is not valid. Please check and correct it, and click Save again 5");
                    filledForm.reject(ve);
                    flash("message", "Invalid URL.");
                    return redirect(routes.TargetController.edit(id));
                }
                Logger.debug("Adding url: " + trimmed + " at position " + position);
                fu.position = position;
                position++;
                Logger.debug("extFormUrl: " + extFormUrl);
                fieldUrls.add(fu);
            }
            filledForm.get().fieldUrls = fieldUrls;
            Logger.debug("fieldUrls: " + filledForm.get().fieldUrls);
        }
        return null;
    }

    private static Result validateCheckForExistingUrl(Long id, Form<Target> filledForm) throws ActException {
        List<FieldUrl> fieldUrls = filledForm.get().fieldUrls;

        for (FieldUrl fieldUrl : fieldUrls) {

            FieldUrl isExistingFieldUrl = FieldUrl.hasDuplicate(fieldUrl.url);

            Logger.debug("For url " + fieldUrl.url);
            Logger.debug("Found existing FieldUrl " + isExistingFieldUrl);
            Logger.debug("Got filledForm.get().id: " + filledForm.get().id);

            if (isExistingFieldUrl != null) {
                if (!isExistingFieldUrl.target.id.equals(id)) {
                    Logger.debug("Found existing FieldUrl.target " + isExistingFieldUrl.target);
                    String duplicateUrl = Play.application().configuration().getString("server_name")
                            + Play.application().configuration().getString("application.context") + "/targets/"
                            + isExistingFieldUrl.target.id;
                    ValidationError ve = new ValidationError("formUrl",
                            "Seed URL already associated with another Target <a href=\"" + duplicateUrl + "\">"
                                    + duplicateUrl + "</a>");
                    filledForm.reject(ve);
                    return info(filledForm, id);
                }
            }
        }

        return null;
    }

    public static Result update(Long id) throws ActException {
        DynamicForm requestData = form().bindFromRequest();

        String action = requestData.get("action");

        if (StringUtils.isNotEmpty(action)) {
            if (action.equals("request")) {
                return redirect(routes.CrawlPermissionController.licenceRequestForTarget(id));
            } else {
                if (action.equals("save")) {
                    return validateForm(requestData, id);
                } else {
                    if (action.equals("archive")) {
                        return redirect(routes.TargetController.archive(id));
                    } else {
                        if (action.equals("delete")) {
                            return delete(id);
                        }
                    }
                }
            }
        }
        return null;
    }

    /**
     * This method saves changes on given target in a new target object
     * completed by revision comment. The "version" field in the Target object
     * contains the timestamp of the change and the last version is marked by
     * flag "active". Remaining Target objects with the same URL are not active.
     *
     * @return
     * @throws ActException
     */
    public static Result save() throws ActException {

        DynamicForm requestData = form().bindFromRequest();
        Map<String, String[]> formParams = request().body().asFormUrlEncoded();

        Form<Target> filledForm = form(Target.class).bindFromRequest();
        if (filledForm.hasErrors()) {
            Logger.debug("errors: " + filledForm.errors());
            return info(filledForm, null);
        }

        return validateForm(requestData, null);
    }

    private static List<Subject> processParentsSubjects(List<Subject> subjects, Long parentId) {
        Subject parent = Subject.findById(parentId);
        if (!subjects.contains(parent)) {
            subjects.add(parent);
        }
        if (parent.parent != null) {
            processParentsSubjects(subjects, parent.parent.id);
        }
        return subjects;
    }

    private static List<Collection> processParentsCollections(List<Collection> collections, Long parentId) {
        Collection parent = Collection.findById(parentId);
        if (!collections.contains(parent)) {
            collections.add(parent);
        }
        if (parent.parent != null) {
            processParentsCollections(collections, parent.parent.id);
        }
        return collections;
    }

    /**
     * This method pushes a message onto a RabbitMQ queue for given target
     * using global settings from project configuration file.
     *
     * @param target The field URL of the target
     * @return
     */
    public static Result archive(Long id) {

        Target target = Target.findById(id);
        Logger.debug("archiveTarget() " + target);
        if (target != null) {
            if (!target.isInScopeAllOrInheritedWithoutLicense()) {
                return ok(infomessage.render("On-demand archiving is only supported for NPLD targets."));
            }

            // Send the message:
            try {
                String queueHost = Play.application().configuration().getString(Const.QUEUE_HOST);
                String queuePort = Play.application().configuration().getString(Const.QUEUE_PORT);
                String queueName = Play.application().configuration().getString(Const.QUEUE_NAME);
                String routingKey = Play.application().configuration().getString(Const.ROUTING_KEY);
                String exchangeName = Play.application().configuration().getString(Const.EXCHANGE_NAME);

                Logger.debug("archiveTarget() queue host: " + queueHost);
                Logger.debug("archiveTarget() queue port: " + queuePort);
                Logger.debug("archiveTarget() queue name: " + queueName);
                Logger.debug("archiveTarget() routing key: " + routingKey);
                Logger.debug("archiveTarget() exchange name: " + exchangeName);

                JsonNode jsonData = Json.toJson(target);
                String message = jsonData.toString();
                Logger.debug("Crawl Now message: " + message);

                ConnectionFactory factory = new ConnectionFactory();
                if (queueHost != null) {
                    factory.setHost(queueHost);
                }
                if (queuePort != null) {
                    factory.setPort(Integer.parseInt(queuePort));
                }
                Connection connection = factory.newConnection();
                Channel channel = connection.createChannel();

                channel.exchangeDeclare(exchangeName, "direct", true);
                channel.queueDeclare(queueName, true, false, false, null);
                channel.queueBind(queueName, exchangeName, routingKey);

                BasicProperties.Builder propsBuilder = new BasicProperties.Builder();
                propsBuilder.deliveryMode(2);
                channel.basicPublish(exchangeName, routingKey, propsBuilder.build(), message.getBytes());

                channel.close();
                connection.close();

            } catch (IOException e) {
                String msg = "There was a problem queuing this crawl instruction. Please refer to the system administrator.";
                User currentUser = User.findByEmail(request().username());
                StringWriter sw = new StringWriter();
                e.printStackTrace(new PrintWriter(sw));
                String msgFullTrace = sw.toString();
                Logger.error(msgFullTrace);
                if (currentUser.hasRole("sys_admin")) {
                    msg = msgFullTrace;
                }
                return ok(infomessage.render(msg));
            } catch (Exception e) {
                String msg = "There was a problem queuing this crawl instruction. Please refer to the system administrator.";
                User currentUser = User.findByEmail(request().username());
                StringWriter sw = new StringWriter();
                e.printStackTrace(new PrintWriter(sw));
                String msgFullTrace = sw.toString();
                Logger.error(msgFullTrace);
                if (currentUser.hasRole("sys_admin")) {
                    msg = msgFullTrace;
                }
                return ok(infomessage.render(msg));
            }
        } else {
            Logger.debug("There was a problem sending the message. Target field for archiving is empty");
            return ok(infomessage
                    .render("There was a problem sending the message. Target field for archiving is empty"));
        }
        return ok(ukwalicenceresult.render());
    }

    /**
     * This method is checking scope for given URL and returns result in JSON format.
     *
     * @param url
     * @return JSON result
     * @throws WhoisException
     */
    public static Result isInScope(String url) throws WhoisException {
        Logger.debug("isInScope controller: " + url);
        boolean res = Scope.INSTANCE.check(url, null, false);

        return ok(Json.toJson(res));
    }

    /**
     * This method calculates collection children - objects that have parents.
     *
     * @param url       The identifier for parent
     * @param targetUrl This is an identifier for current target object
     * @return child collection in JSON form
     */
    public static String getChildren(String url, String targetUrl) {
        //       Logger.debug("getChildren() target URL: " + targetUrl);
        String res = "";
        final StringBuffer sb = new StringBuffer();
        sb.append(", \"children\":");
        List<Collection> childSuggestedCollections = Collection.getChildLevelCollections(url);
        if (childSuggestedCollections.size() > 0) {
            sb.append(getTreeElements(childSuggestedCollections, targetUrl, false));
            res = sb.toString();
            //          Logger.debug("getChildren() res: " + res);
        }
        return res;
    }

    /**
     * Mark collections that are stored in target object as selected
     *
     * @param collectionUrl The collection identifier
     * @param targetUrl     This is an identifier for current target object
     * @return
     */
    public static String checkSelection(String collectionUrl, String targetUrl) {
        String res = "";
        if (targetUrl != null && targetUrl.length() > 0) {
            Target target = Target.findByUrl(targetUrl);
            if (target.collections != null && target.collections.contains(collectionUrl)) {
                res = "\"select\": true ,";
            }
        }
        return res;
    }

    /**
     * Mark preselected collections as selected in filter
     *
     * @param collectionUrl The collection identifier
     * @param targetUrl     This is an identifier for current target object
     * @return
     */
    public static String checkSelectionFilter(String collectionUrl, String targetUrl) {
        String res = "";
        if (targetUrl != null && targetUrl.length() > 0) {
            if (collectionUrl != null && targetUrl.equals(collectionUrl)) {
                res = "\"select\": true ,";
            }
        }
        return res;
    }

    /**
     * This method calculates first order collections for filtering.
     *
     * @param collectionList The list of all collections
     * @param targetUrl      This is an identifier for current target object
     * @param parent         This parameter is used to differentiate between root and children nodes
     * @return collection object in JSON form
     */
    public static String getTreeElementsFilter(List<Collection> collectionList, String targetUrl, boolean parent) {
        //       Logger.debug("getTreeElements() target URL: " + targetUrl);
        String res = "";
        if (collectionList.size() > 0) {
            final StringBuffer sb = new StringBuffer();
            sb.append("[");
            Iterator<Collection> itr = collectionList.iterator();
            boolean firstTime = true;
            while (itr.hasNext()) {
                Collection collection = itr.next();
                //             Logger.debug("add collection: " + collection.name + ", with url: " + collection.url +
                //                   ", parent:" + collection.parent + ", parent size: " + collection.parent.length());
                if ((parent && collection.parent == null) || !parent || collection.parent == null) {
                    if (firstTime) {
                        firstTime = false;
                    } else {
                        sb.append(", ");
                    }
                    //                Logger.debug("added");
                    sb.append("{\"title\": \"" + collection.name + "\","
                            + checkSelectionFilter(collection.url, targetUrl) + " \"key\": \"" + collection.url
                            + "\"" + getChildren(collection.url, targetUrl) + "}");
                }
            }
            //          Logger.debug("collectionList level size: " + collectionList.size());
            sb.append("]");
            res = sb.toString();
            //          Logger.debug("getTreeElements() res: " + res);
        }
        return res;
    }

    //    [{"title":"100 Best Sites (95)","url":"/actdev/collections/act-170","key":"\"act-170\""},
    //     {"title":"19th Century English Literature (0)","url":"/actdev/collections/act-153","key":"\"act-153\""},
    //     {"title":"Commonwealth Games Glasgow 2014 (625)","url":"/actdev/collections/act-252","key":"\"act-252\"","children":[
    //        {"title":"Competitors (350)","url":"/actdev/collections/act-266","key":"\"act-266\""},{"title":"Cultural Programme (44)","url":"/actdev/collections/act-295","key":"\"act-295\""},{"title":"Organisational bodies/venues (57)","url":"/actdev/collections/act-267","key":"\"act-267\""},{"title":"Press & Media Comment (41)","url":"/actdev/collections/act-269","key":"\"act-269\""},{"title":"Sponsors (15)","url":"/actdev/collections/act-270","key":"\"act-270\""},{"title":"Sports (405)","url":"/actdev/collections/act-268","key":"\"act-268\"","children":[{"title":"Aquatics (74)","url":"/actdev/collections/act-287","key":"\"act-287\""},{"title":"Athletics (107)","url":"/actdev/collections/act-286","key":"\"act-286\""},{"title":"Badminton (18)","url":"/actdev/collections/act-285","key":"\"act-285\""},{"title":"Boxing (26)","url":"/actdev/collections/act-284","key":"\"act-284\""},{"title":"Cycling (34)","url":"/actdev/collections/act-283","key":"\"act-283\""},{"title":"Gymnastics (29)","url":"/actdev/collections/act-282","key":"\"act-282\""},{"title":"Hockey (17)","url":"/actdev/collections/act-281","key":"\"act-281\""},{"title":"Judo (9)","url":"/actdev/collections/act-280","key":"\"act-280\""},{"title":"Lawn bowls (11)","url":"/actdev/collections/act-279","key":"\"act-279\""},{"title":"Netball (6)","url":"/actdev/collections/act-278","key":"\"act-278\""},{"title":"Rugby Sevens (16)","url":"/actdev/collections/act-277","key":"\"act-277\""},{"title":"Shooting (14)","url":"/actdev/collections/act-276","key":"\"act-276\""},{"title":"Squash (16)","url":"/actdev/collections/act-275","key":"\"act-275\""},{"title":"Table tennis (6)","url":"/actdev/collections/act-274","key":"\"act-274\""},{"title":"Triathlon (7)","url":"/actdev/collections/act-273","key":"\"act-273\""},{"title":"Weightlifting (7)","url":"/actdev/collections/act-272","key":"\"act-272\""},{"title":"Wrestling (7)","url":"/actdev/collections/act-271","key":"\"act-271\""}]}]},{"title":"Conservative Party Website deletions - Press articles November 2013 (10)","url":"/actdev/collections/act-175","key":"\"act-175\""},{"title":"Ebola (192)","url":"/actdev/collections/act-296","key":"\"act-296\""},{"title":"European Parliament Elections 2014 (1743)","url":"/actdev/collections/act-250","key":"\"act-250\"","children":[{"title":"Academia & think tanks (15)","url":"/actdev/collections/act-257","key":"\"act-257\""},{"title":"Blogs (215)","url":"/actdev/collections/act-263","key":"\"act-263\""},{"title":"Candidates (520)","url":"/actdev/collections/act-262","key":"\"act-262\""},{"title":"EU Institutions (1)","url":"/actdev/collections/act-260","key":"\"act-260\""},{"title":"Interest groups (63)","url":"/actdev/collections/act-254","key":"\"act-254\""},{"title":"Opinion Polls (7)","url":"/actdev/collections/act-261","key":"\"act-261\""},{"title":"Political Parties: National (99)","url":"/actdev/collections/act-256","key":"\"act-256\""},{"title":"Political Parties: Regional & Local (65)","url":"/actdev/collections/act-258","key":"\"act-258\""},{"title":"Press & Media Comment (744)","url":"/actdev/collections/act-255","key":"\"act-255\""},{"title":"Regulation and Guidance (13)","url":"/actdev/collections/act-253","key":"\"act-253\""},{"title":"Social Media (1)","url":"/actdev/collections/act-259","key":"\"act-259\""}]},{"title":"Evolving role of libraries in the UK (0)","url":"/actdev/collections/act-154","key":"\"act-154\""},{"title":"First World War Centenary, 2014-18 (208)","url":"/actdev/collections/act-174","key":"\"act-174\"","children":[{"title":"Heritage Lottery Fund (66)","url":"/actdev/collections/act-265","key":"\"act-265\""}]},

    //        {"title":"Health and Social Care Act 2012 - NHS Reforms (752)","url":"/actdev/collections/act-24","key":"\"act-24\"","children":[

    //            {"title":"NHS (720)","url":"/actdev/collections/act-25","key":"\"act-25\"","children":[
    //                {"title":"Acute Trusts (161)","url":"/actdev/collections/act-27","key":"\"act-27\""},{"title":"Ambulance Trusts (12)","url":"/actdev/collections/act-136","key":"\"act-136\""},{"title":"Campaigning and Advocacy Groups (18)","url":"/actdev/collections/act-148","key":"\"act-148\""},{"title":"Cancer Networks (28)","url":"/actdev/collections/act-137","key":"\"act-137\""},{"title":"Care Trust (29)","url":"/actdev/collections/act-139","key":"\"act-139\""},{"title":"Clinical Commissioning Groups (191)","url":"/actdev/collections/act-28","key":"\"act-28\""},{"title":"Gateways (11)","url":"/actdev/collections/act-141","key":"\"act-141\""},{"title":"Health and Wellbeing Boards (108)","url":"/actdev/collections/act-29","key":"\"act-29\""},{"title":"Healthwatch (127)","url":"/actdev/collections/act-144","key":"\"act-144\""},{"title":"Legislation (17)","url":"/actdev/collections/act-143","key":"\"act-143\""},{"title":"Local Authorities (142)","url":"/actdev/collections/act-166","key":"\"act-166\""},{"title":"Local Involvement Networks (LINks) (159)","url":"/actdev/collections/act-30","key":"\"act-30\""},{"title":"Mental Health Trusts (50)","url":"/actdev/collections/act-138","key":"\"act-138\""},{"title":"NHS programmes (3)","url":"/actdev/collections/act-142","key":"\"act-142\""},{"title":"Press Comment (248)","url":"/actdev/collections/act-150","key":"\"act-150\""},{"title":"Primary Care Trusts (15)","url":"/actdev/collections/act-26","key":"\"act-26\""},{"title":"Private and voluntary sector providers (8)","url":"/actdev/collections/act-169","key":"\"act-169\""},{"title":"Professional Bodies Trade Union (49)","url":"/actdev/collections/act-147","key":"\"act-147\""},{"title":"Public Health Agencies (1)","url":"/actdev/collections/act-134","key":"\"act-134\""},{"title":"Public Health England (138)","url":"/actdev/collections/act-149","key":"\"act-149\""},{"title":"Regulators & Central Government (1)","url":"/actdev/collections/act-135","key":"\"act-135\""},{"title":"Social Media (Facebook, Twitter etc) (42)","url":"/actdev/collections/act-167","key":"\"act-167\""},{"title":"Special Health Authorities (13)","url":"/actdev/collections/act-140","key":"\"act-140\""},{"title":"Specialised Commissioning Group (12)","url":"/actdev/collections/act-145","key":"\"act-145\""},{"title":"Strategic Health Authorities (20)","url":"/actdev/collections/act-82","key":"\"act-82\"","children":[{"title":"London SHA Cluster (1)","url":"/actdev/collections/act-83","key":"\"act-83\"","children":[{"title":"London SHA (1)","url":"/actdev/collections/act-87","key":"\"act-87\"","children":[{"title":"London (1)","url":"/actdev/collections/act-31","key":"\"act-31\""},{"title":"North Central London (6)","url":"/actdev/collections/act-33","key":"\"act-33\""},{"title":"North East London and City (9)","url":"/actdev/collections/act-32","key":"\"act-32\""},{"title":"North West London (8)","url":"/actdev/collections/act-34","key":"\"act-34\""},{"title":"South East London (6)","url":"/actdev/collections/act-36","key":"\"act-36\""},{"title":"South West London (6)","url":"/actdev/collections/act-35","key":"\"act-35\""}]}]},{"title":"Midlands and East SHA Cluster (4)","url":"/actdev/collections/act-85","key":"\"act-85\"","children":[{"title":"East Midlands (1)","url":"/actdev/collections/act-91","key":"\"act-91\"","children":[{"title":"Derby City and Derbyshire (2)","url":"/actdev/collections/act-53","key":"\"act-53\""},{"title":"East Midlands (2)","url":"/actdev/collections/act-52","key":"\"act-52\""},{"title":"Leicestershire County & Rutland and Leicestershire City (2)","url":"/actdev/collections/act-54","key":"\"act-54\""},{"title":"Lincolnshire (1)","url":"/actdev/collections/act-55","key":"\"act-55\""},{"title":"Milton Keynes and Northamptonshire (2)","url":"/actdev/collections/act-56","key":"\"act-56\""},{"title":"Nottinghamshhire County and Nottingham City (2)","url":"/actdev/collections/act-57","key":"\"act-57\""}]},{"title":"East of England (1)","url":"/actdev/collections/act-92","key":"\"act-92\"","children":[{"title":"Bedfordshire and Luton (1)","url":"/actdev/collections/act-59","key":"\"act-59\""},{"title":"Cambridgeshire and Peterborough (3)","url":"/actdev/collections/act-62","key":"\"act-62\""},{"title":"Hertfordshire (1)","url":"/actdev/collections/act-58","key":"\"act-58\""},{"title":"Norfolk and Waveney (2)","url":"/actdev/collections/act-63","key":"\"act-63\""},{"title":"North, Mid and East Essex (1)","url":"/actdev/collections/act-60","key":"\"act-60\""},{"title":"South Essex (2)","url":"/actdev/collections/act-61","key":"\"act-61\""},{"title":"Suffolk (1)","url":"/actdev/collections/act-64","key":"\"act-64\""}]},{"title":"West Midlands (1)","url":"/actdev/collections/act-93","key":"\"act-93\"","children":[{"title":"Arden (2)","url":"/actdev/collections/act-65","key":"\"act-65\""},{"title":"Birmingham and Solihull (5)","url":"/actdev/collections/act-66","key":"\"act-66\""},{"title":"Black Country (4)","url":"/actdev/collections/act-67","key":"\"act-67\""},{"title":"Staffordshire (3)","url":"/actdev/collections/act-69","key":"\"act-69\""},{"title":"West Mercia (5)","url":"/actdev/collections/act-70","key":"\"act-70\""},{"title":"West Midlands (1)","url":"/actdev/collections/act-68","key":"\"act-68\""}]}]},{"title":"North of England SHA Cluster (7)","url":"/actdev/collections/act-84","key":"\"act-84\"","children":[{"title":"North East (3)","url":"/actdev/collections/act-88","key":"\"act-88\"","children":[{"title":"County Durham and Darlington (3)","url":"/actdev/collections/act-37","key":"\"act-37\""},{"title":"North of Tyne (5)","url":"/actdev/collections/act-38","key":"\"act-38\""},{"title":"South of Tyne (4)","url":"/actdev/collections/act-39","key":"\"act-39\""},{"title":"Tees (5)","url":"/actdev/collections/act-40","key":"\"act-40\""}]},{"title":"North West (1)","url":"/actdev/collections/act-89","key":"\"act-89\"","children":[{"title":"Cheshire, Warrington, Wirral (4)","url":"/actdev/collections/act-41","key":"\"act-41\""},{"title":"Cumbria (1)","url":"/actdev/collections/act-42","key":"\"act-42\""},{"title":"Greater Manchester (9)","url":"/actdev/collections/act-43","key":"\"act-43\""},{"title":"Lancashire (5)","url":"/actdev/collections/act-44","key":"\"act-44\""},{"title":"Merseyside (3)","url":"/actdev/collections/act-45","key":"\"act-45\""}]},{"title":"Yorkshire and the Humber (3)","url":"/actdev/collections/act-90","key":"\"act-90\"","children":[{"title":"Bradford (2)","url":"/actdev/collections/act-49","key":"\"act-49\""},{"title":"Calderdale, Kirklees and Wakefield (3)","url":"/actdev/collections/act-46","key":"\"act-46\""},{"title":"Humber (5)","url":"/actdev/collections/act-47","key":"\"act-47\""},{"title":"Leeds (1)","url":"/actdev/collections/act-50","key":"\"act-50\""},{"title":"North Yorkshire and York (1)","url":"/actdev/collections/act-51","key":"\"act-51\""},{"title":"South Yorkshire and Bassetlaw (5)","url":"/actdev/collections/act-48","key":"\"act-48\""}]}]},{"title":"South of England SHA Cluster (1)","url":"/actdev/collections/act-86","key":"\"act-86\"","children":[{"title":"South Central (0)","url":"/actdev/collections/act-94","key":"\"act-94\"","children":[{"title":"Berkshire (6)","url":"/actdev/collections/act-71","key":"\"act-71\""},{"title":"Buckinghamshire and Oxfordshire (2)","url":"/actdev/collections/act-73","key":"\"act-73\""},{"title":"Southampton, Hampshire, Isle of Wight & Portsmouth (5)","url":"/actdev/collections/act-72","key":"\"act-72\""}]},{"title":"South East Coast (1)","url":"/actdev/collections/act-95","key":"\"act-95\"","children":[{"title":"Kent and Medway (4)","url":"/actdev/collections/act-74","key":"\"act-74\""},{"title":"Surrey (1)","url":"/actdev/collections/act-75","key":"\"act-75\""},{"title":"Sussex (5)","url":"/actdev/collections/act-76","key":"\"act-76\""}]},{"title":"South West (0)","url":"/actdev/collections/act-96","key":"\"act-96\"","children":[{"title":"Bath, North East Somerset and Wiltshire (2)","url":"/actdev/collections/act-77","key":"\"act-77\""},{"title":"Bournemouth, Poole and Dorset (2)","url":"/actdev/collections/act-78","key":"\"act-78\""},{"title":"Bristol, North Somerset and South Gloucestershire (4)","url":"/actdev/collections/act-79","key":"\"act-79\""},{"title":"Devon, Plymouth and Torbay (3)","url":"/actdev/collections/act-80","key":"\"act-80\""},{"title":"Gloucestershire and Swindon (3)","url":"/actdev/collections/act-81","key":"\"act-81\""}]}]}]},{"title":"Think Tanks (30)","url":"/actdev/collections/act-146","key":"\"act-146\""}]}]},{"title":"History of Libraries Collection (0)","url":"/actdev/collections/act-155","key":"\"act-155\""},{"title":"History of the Book (0)","url":"/actdev/collections/act-156","key":"\"act-156\""},{"title":"Legal Aid Reform (0)","url":"/actdev/collections/act-157","key":"\"act-157\""},{"title":"Margaret Thatcher (77)","url":"/actdev/collections/act-151","key":"\"act-151\""},{"title":"Nelson Mandela (174)","url":"/actdev/collections/act-178","key":"\"act-178\""},{"title":"News Sites (576)","url":"/actdev/collections/act-173","key":"\"act-173\"","children":[{"title":"Hyperlocal (515)","url":"/actdev/collections/act-297","key":"\"act-297\""}]},{"title":"Oral History in the UK (2)","url":"/actdev/collections/act-158","key":"\"act-158\""},{"title":"Political Action and Communication (5)","url":"/actdev/collections/act-159","key":"\"act-159\""},{"title":"Religion, politics and law since 2005 (9)","url":"/actdev/collections/act-160","key":"\"act-160\""},{"title":"Religion/Theology (237)","url":"/actdev/collections/act-172","key":"\"act-172\""},{"title":"Scottish Independence Referendum (1144)","url":"/actdev/collections/act-171","key":"\"act-171\"","children":[{"title":"Charities, Churches and Third Sector (16)","url":"/actdev/collections/act-293","key":"\"act-293\""},{"title":"Commercial Publishers (1)","url":"/actdev/collections/act-294","key":"\"act-294\""},{"title":"Government (UK and Scottish) (83)","url":"/actdev/collections/act-291","key":"\"act-291\""},{"title":"National Campaigning Groups (80)","url":"/actdev/collections/act-288","key":"\"act-288\""},{"title":"Political Parties and Trade Unions (249)","url":"/actdev/collections/act-289","key":"\"act-289\""},{"title":"Press, Media & Comment (263)","url":"/actdev/collections/act-292","key":"\"act-292\""},{"title":"Think Tanks and Research Institutes (47)","url":"/actdev/collections/act-290","key":"\"act-290\""}]},{"title":"Slavery and Abolition in the Caribbean (0)","url":"/actdev/collections/act-161","key":"\"act-161\""},{"title":"Tour de France (Yorkshire) 2014 (60)","url":"/actdev/collections/act-264","key":"\"act-264\""},{"title":"UK relations with the Low Countries (0)","url":"/actdev/collections/act-162","key":"\"act-162\""},{"title":"UK response to Philippines disaster 2013 (501)","url":"/actdev/collections/act-176","key":"\"act-176\""},{"title":"Video Games (0)","url":"/actdev/collections/act-163","key":"\"act-163\""},{"title":"Winter Olympics Sochi 2014 (128)","url":"/actdev/collections/act-177","key":"\"act-177\""}]    

    //    [{"title":"Health and Social Care Act 2012 - NHS Reforms","select":true,"key":"act-24","children":[

    //       {"title":"NHS","key":"act-25","children":[
    //          {"title":"Campaigning and Advocacy Groups","key":"act-148"},{"title":"Acute Trusts","key":"act-27"},{"title":"Clinical Commissioning Groups","key":"act-28"},{"title":"Strategic Health Authorities","key":"act-82","children":[
    //              {"title":"London SHA Cluster","key":"act-83","children":[{"title":"London SHA","key":"act-87","children":[
    //                   {"title":"North West London","key":"act-34"},{"title":"London","key":"act-31"},{"title":"North East London and City","key":"act-32"},{"title":"North Central London","key":"act-33"},{"title":"South West London","key":"act-35"},{"title":"South East London","key":"act-36"}]}]},{"title":"Midlands and East SHA Cluster","key":"act-85","children":[{"title":"East Midlands","key":"act-91","children":[{"title":"Nottinghamshhire County and Nottingham City","key":"act-57"},{"title":"Derby City and Derbyshire","key":"act-53"},{"title":"Leicestershire County & Rutland and Leicestershire City","key":"act-54"},{"title":"Lincolnshire","key":"act-55"},{"title":"Milton Keynes and Northamptonshire","key":"act-56"}]},{"title":"West Midlands","key":"act-93","children":[{"title":"Black Country","key":"act-67"},{"title":"Arden","key":"act-65"},{"title":"Birmingham and Solihull","key":"act-66"},{"title":"Staffordshire","key":"act-69"},{"title":"West Mercia","key":"act-70"}]},{"title":"East of England","key":"act-92","children":[{"title":"Hertfordshire","key":"act-58"},{"title":"Bedfordshire and Luton","key":"act-59"},{"title":"North,  Mid and East Essex","key":"act-60"},{"title":"South Essex","key":"act-61"},{"title":"Cambridgeshire and Peterborough","key":"act-62"},{"title":"Norfolk and Waveney","key":"act-63"},{"title":"Suffolk","key":"act-64"}]}]},{"title":"North of England SHA Cluster","key":"act-84","children":[{"title":"North West","key":"act-89","children":[{"title":"Greater Manchester","key":"act-43"},{"title":"Cheshire,  Warrington,  Wirral","key":"act-41"},{"title":"Cumbria","key":"act-42"},{"title":"Lancashire","key":"act-44"},{"title":"Merseyside","key":"act-45"}]},{"title":"North East","key":"act-88","children":[{"title":"County Durham and Darlington","key":"act-37"},{"title":"North of Tyne","key":"act-38"},{"title":"South of Tyne","key":"act-39"},{"title":"Tees","key":"act-40"}]},{"title":"Yorkshire and the Humber","key":"act-90","children":[{"title":"Calderdale,  Kirklees and Wakefield","key":"act-46"},{"title":"Humber","key":"act-47"},{"title":"South Yorkshire and Bassetlaw","key":"act-48"},{"title":"Bradford","key":"act-49"},{"title":"Leeds","key":"act-50"},{"title":"North Yorkshire and York","key":"act-51"}]}]},{"title":"South of England SHA Cluster","key":"act-86","children":[{"title":"South Central","key":"act-94","children":[{"title":"Berkshire","key":"act-71"},{"title":"Southampton,  Hampshire,  Isle of Wight & Portsmouth","key":"act-72"},{"title":"Buckinghamshire and Oxfordshire","key":"act-73"}]},{"title":"South East Coast","key":"act-95","children":[{"title":"Kent and Medway","key":"act-74"},{"title":"Surrey","key":"act-75"},{"title":"Sussex","key":"act-76"}]},{"title":"South West","key":"act-96","children":[{"title":"Bath,  North East Somerset and Wiltshire","key":"act-77"},{"title":"Bournemouth,  Poole and Dorset","key":"act-78"},{"title":"Bristol,  North Somerset and South Gloucestershire","key":"act-79"},{"title":"Devon,  Plymouth and Torbay","key":"act-80"},{"title":"Gloucestershire and Swindon","key":"act-81"}]}]}]},{"title":"Health and Wellbeing Boards","key":"act-29"},{"title":"Local Involvement Networks (LINks)","key":"act-30"},{"title":"Care Trust","key":"act-139"},{"title":"Public Health England","key":"act-149"},{"title":"Special Health Authorities","key":"act-140"},{"title":"Public Health Agencies","key":"act-134"},{"title":"Ambulance Trusts","key":"act-136"},{"title":"Mental Health Trusts","key":"act-138"},{"title":"Gateways","key":"act-141"},{"title":"NHS programmes","key":"act-142"},{"title":"Professional Bodies Trade Union","key":"act-147"},{"title":"Legislation","key":"act-143"},{"title":"Healthwatch","key":"act-144"},{"title":"Specialised Commissioning Group","key":"act-145"},{"title":"Think Tanks","key":"act-146"},{"title":"Press Comment","key":"act-150"},{"title":"Cancer Networks","key":"act-137"},{"title":"Regulators & Central Government","key":"act-135"},{"title":"Local Authorities","key":"act-166"},{"title":"Social Media (Facebook,  Twitter etc)","key":"act-167"},{"title":"Primary Care Trusts","key":"act-26"},{"title":"Private and voluntary sector providers","key":"act-169"}]}]},{"title":"Scottish Independence Referendum","key":"act-171","children":[{"title":"Press,  Media & Comment","key":"act-292"},{"title":"Political Parties and Trade Unions","key":"act-289"},{"title":"Think Tanks and Research Institutes","key":"act-290"},{"title":"National Campaigning Groups","key":"act-288"},{"title":"Government (UK and Scottish)","key":"act-291"},{"title":"Charities,  Churches and Third Sector","key":"act-293"}]},{"title":"Science,  Technology & Medicine","key":"act-99","children":[{"title":"Physics","key":"act-108"},{"title":"Astronomy","key":"act-107"}]},{"title":"Religion,  politics and law since 2005","key":"act-160"},{"title":"News Sites","key":"act-173"},{"title":"100 Best Sites","key":"act-170"},{"title":"Margaret Thatcher","key":"act-151"},{"title":"First World War Centenary,  2014-18","key":"act-174","children":[{"title":"Heritage Lottery Fund","key":"act-265"}]},{"title":"Religion/Theology","key":"act-172"},{"title":"European Parliament Elections 2014","key":"act-250","children":[{"title":"Political Parties: National","key":"act-256"},{"title":"Academia & think tanks","key":"act-257"},{"title":"Interest groups","key":"act-254"},{"title":"Regulation and Guidance","key":"act-253"},{"title":"Blogs","key":"act-263"},{"title":"Political Parties: Regional & Local","key":"act-258"},{"title":"Social Media","key":"act-259"},{"title":"EU Institutions","key":"act-260"},{"title":"Opinion Polls","key":"act-261"},{"title":"Candidates","key":"act-262"}]},{"title":"Nelson Mandela","key":"act-178"},{"title":"UK response to Philippines disaster 2013","key":"act-176"},{"title":"Commonwealth Games Glasgow 2014","key":"act-252","children":[{"title":"Organisational bodies/venues","key":"act-267"},{"title":"Sports","key":"act-268","children":[{"title":"Rugby Sevens","key":"act-277"},{"title":"Badminton","key":"act-285"},{"title":"Cycling","key":"act-283"},{"title":"Wrestling","key":"act-271"},{"title":"Hockey","key":"act-281"},{"title":"Boxing","key":"act-284"},{"title":"Aquatics","key":"act-287"},{"title":"Athletics","key":"act-286"},{"title":"Gymnastics","key":"act-282"},{"title":"Judo","key":"act-280"},{"title":"Lawn bowls","key":"act-279"},{"title":"Netball","key":"act-278"},{"title":"Shooting","key":"act-276"},{"title":"Squash","key":"act-275"},{"title":"Table tennis","key":"act-274"},{"title":"Triathlon","key":"act-273"},{"title":"Weightlifting","key":"act-272"}]},{"title":"Press & Media Comment","key":"act-269"},{"title":"Sponsors","key":"act-270"},{"title":"Competitors","key":"act-266"},{"title":"Cultural Programme","key":"act-295"}]},{"title":"Winter Olympics Sochi 2014","key":"act-177"},{"title":"Oral History in the UK","key":"act-158"},{"title":"Conservative Party Website deletions - Press articles November 2013","key":"act-175"},{"title":"Political Action and Communication","key":"act-159"},{"title":"Tour de France (Yorkshire) 2014","key":"act-264"}]

    /**
     * This method computes a tree of collections in JSON format.
     *
     * @param targetUrl This is an identifier for current target object
     * @return tree structure
     */
    @BodyParser.Of(BodyParser.Json.class)
    public static Result getSuggestedCollections(String targetUrl) {
        //       Logger.debug("getSuggestedCollections() target URL: " + targetUrl);
        JsonNode jsonData = null;
        final StringBuffer sb = new StringBuffer();
        List<Collection> suggestedCollections = Collection.getFirstLevelCollections();
        sb.append(getTreeElements(suggestedCollections, targetUrl, true));
        //       Logger.debug("collections main level size: " + suggestedCollections.size());
        jsonData = Json.toJson(Json.parse(sb.toString()));
        //       Logger.debug("getCollections() json: " + jsonData.toString());
        return ok(jsonData);
    }

    /**
     * This method calculates first order collections.
     *
     * @param collectionList The list of all collections
     * @param targetUrl      This is an identifier for current target object
     * @param parent         This parameter is used to differentiate between root and children nodes
     * @return collection object in JSON form
     */
    public static String getTreeElements(List<Collection> collectionList, String targetUrl, boolean parent) {
        //       Logger.debug("getTreeElements() target URL: " + targetUrl);
        String res = "";
        if (collectionList.size() > 0) {
            final StringBuffer sb = new StringBuffer();
            sb.append("[");
            Iterator<Collection> itr = collectionList.iterator();
            boolean firstTime = true;
            while (itr.hasNext()) {
                Collection collection = itr.next();
                //             Logger.debug("add collection: " + collection.name + ", with url: " + collection.url +
                //                   ", parent:" + collection.parent + ", parent size: " + collection.parent.length());
                if ((parent && collection.parent == null) || !parent || collection.parent == null) {
                    if (firstTime) {
                        firstTime = false;
                    } else {
                        sb.append(", ");
                    }
                    //                Logger.debug("added");
                    sb.append("{\"title\": \"" + collection.name + "\"," + checkSelection(collection.url, targetUrl)
                            + " \"key\": \"" + collection.url + "\"" + getChildren(collection.url, targetUrl)
                            + "}");
                }
            }
            //          Logger.debug("collectionList level size: " + collectionList.size());
            sb.append("]");
            res = sb.toString();
            //          Logger.debug("getTreeElements() res: " + res);
        }
        return res;
    }

    /**
     * This method computes a tree of collections in JSON format for filtering.
     *
     * @param targetUrl This is an identifier for current target object
     * @return tree structure
     */
    @BodyParser.Of(BodyParser.Json.class)
    public static Result getSuggestedCollectionsFilter(String targetUrl) {
        //       Logger.debug("getSuggestedCollectionsFilter() target URL: " + targetUrl);
        JsonNode jsonData = null;
        final StringBuffer sb = new StringBuffer();
        List<Collection> suggestedCollections = Collection.getFirstLevelCollections();
        sb.append(getTreeElementsFilter(suggestedCollections, targetUrl, true));
        //       Logger.debug("collections main level size: " + suggestedCollections.size());
        jsonData = Json.toJson(Json.parse(sb.toString()));
        //       Logger.debug("getCollections() json: " + jsonData.toString());
        return ok(jsonData);
    }

    /**
     * This method filters targets by crawl frequency.
     *
     * @return crawl frequency list
     */
    public static List<Target> getCrawlFrequency() {
        List<Target> res = new ArrayList<Target>();
        List<String> subjects = new ArrayList<String>();
        List<Target> allTargets = Target.find.all();
        Iterator<Target> itr = allTargets.iterator();
        while (itr.hasNext()) {
            Target target = itr.next();
            if (target.crawlFrequency != null && target.crawlFrequency.length() > 0
                    && !subjects.contains(target.crawlFrequency)) {
                ExpressionList<Target> ll = Target.find.where().contains("crawlFrequency", target.crawlFrequency);
                if (ll.findRowCount() > 0) {
                    res.add(target);
                    subjects.add(target.crawlFrequency);
                }
            }
        }
        return res;
    }

    public static void raiseFlag(Target target, String flagName) {
        for (Flag flag : target.flags) {
            if (flag.name.equals(flagName)) {
                return;
            }
        }
        target.flags.add(Flag.findByName(flagName));
        Ebean.update(target);
    }

    private static ObjectNode createCollectionNode(Collection collection) {
        ObjectNode collectionNode = JsonNodeFactory.instance.objectNode();
        Logger.debug("collection: " + collection.name);
        collectionNode.put("id", collection.id);
        collectionNode.put("name", collection.name);
        //      collectionNode.put("description", collection.description);
        collectionNode.put("createdAt", collection.createdAt.toString());
        Long milliseconds = collection.updatedAt.getTime();
        Long seconds = (Long) (milliseconds / 1000);
        collectionNode.put("updatedAt", seconds);
        collectionNode.put("publish", collection.publish);
        if (collection.parent != null) {
            collectionNode.put("parent", createCollectionNode(collection.parent));
        }
        return collectionNode;
    }

    @Security.Authenticated(SecuredController.class)
    public static Result getTargetCategories(Long id) {
        Target target = Target.findById(id);
        //      List<Collection> categories = target.getCollectionCategories();
        List<Collection> categories = target.collections;

        ArrayNode arrayNode = JsonNodeFactory.instance.arrayNode();

        for (Collection collection : categories) {
            ObjectNode collectionNode = createCollectionNode(collection);
            arrayNode.add(collectionNode);
        }

        //      JsonNode jsonNode = Json.toJson(categories);
        //      Iterator<JsonNode> iterator = jsonNode.iterator();
        //      if (categories != null) {
        //         Logger.debug("collections: " + categories.size());
        //      }
        //      while(iterator.hasNext()) {
        //         JsonNode node = (JsonNode) iterator.next();
        //         ObjectNode objectNode = (ObjectNode)node;
        //         objectNode.remove("children");
        //         JsonNode updatedAt = objectNode.get("updatedAt");
        //         Long milliseconds = updatedAt.asLong();
        //         Long seconds = (Long)(milliseconds/1000);
        //         Logger.debug("milliseconds: " + milliseconds);
        //         Logger.debug("seconds: " + seconds);
        //         objectNode.put("updatedAt", seconds);
        //      }

        return ok(arrayNode);
    }

    @Security.Authenticated(SecuredController.class)
    public static Result upload() {
        User user = User.findByEmail(request().username());
        return ok(views.html.targets.upload.render(user));
    }

    @Security.Authenticated(SecuredController.class)
    public static Result uploadExcel() {
        MultipartFormData body = request().body().asMultipartFormData();
        FilePart picture = body.getFile("excelFile");
        if (picture != null) {
            String fileName = picture.getFilename();
            String contentType = picture.getContentType();
            File file = picture.getFile();
            Logger.info("Uploaded " + fileName);
            try {
                excelParser(file);
                flash("success", "File " + fileName + " uploaded");
            } catch (Throwable e) {
                flash("error", "File " + fileName + " parse errors:");
                flash("error_log", e.getMessage());
                e.printStackTrace();
            }
            return redirect(routes.TargetController.upload());
        } else {
            Logger.info("Upload failed ");
            flash("error", "Missing file");
            return redirect(routes.TargetController.upload());
        }
    }

    private static void excelParser(File inputFile) throws Throwable {

        FileInputStream file = new FileInputStream(inputFile);

        //Create Workbook instance holding reference to .xls[x] file
        Workbook workbook = WorkbookFactory.create(file);

        //Get first/desired sheet from the workbook
        Sheet sheet = workbook.getSheetAt(0);

        // Check total row:
        if (sheet.getPhysicalNumberOfRows() <= 1) {
            throw new Exception("Sheet should have at least one row.");
        }
        Logger.debug("Sheet has " + sheet.getPhysicalNumberOfRows() + " rows.");

        //Iterate through each rows one by one
        Iterator<Row> rowIterator = sheet.iterator();

        // Header row:
        Row header = rowIterator.next();
        Logger.debug("HEADER: " + header);
        // TODO Check header row is right.

        // And the rest:
        StringBuilder sb = new StringBuilder();
        while (rowIterator.hasNext()) {
            Row row = rowIterator.next();

            // Get
            Target target = new Target();
            target.title = row.getCell(0).getStringCellValue();
            target.fieldUrls = new ArrayList<FieldUrl>();
            // Check URL
            FieldUrl url = new FieldUrl(row.getCell(1).getStringCellValue());
            target.fieldUrls.add(url);
            FieldUrl existingFieldUrl = FieldUrl.findByUrl(url.url);
            if (existingFieldUrl != null) {
                String error = "Row # " + row.getRowNum() + ": CONFLICT - URL " + existingFieldUrl.url
                        + " is already part of target " + existingFieldUrl.target.id + "\n";
                Logger.debug(error);
                sb.append(error);
                continue;
            }
            //Collection c = new Collection();
            //c.name = 

            // 
            System.out.println(target);

            // TODO Merge with controllers.ApplicationController.bulkImport() code to avoid repetition.
            target.revision = Const.INITIAL_REVISION;
            target.active = true;

            target.selectionType = Const.SelectionType.SELECTION.name();

            if (target.noLdCriteriaMet == null) {
                target.noLdCriteriaMet = Boolean.FALSE;
            }

            if (target.keySite == null) {
                target.keySite = Boolean.FALSE;
            }

            if (target.ignoreRobotsTxt == null) {
                target.ignoreRobotsTxt = Boolean.FALSE;
            }

            // Save - disabled right now, as we do not want this live as yet.
            /*
            target.runChecks();
            target.save();
            */

            //
            System.out.println(target);
        }
        workbook.close();
        file.close();

        // And report errors
        if (sb.length() > 0) {
            throw (new Exception(sb.toString()));
        }
    }

    public static void main(String args[]) {
        try {
            excelParser(new File("test/upload/w3act-targets-upload.xlsx"));
        } catch (Throwable e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        }
    }

}