List of usage examples for java.util Map getOrDefault
default V getOrDefault(Object key, V defaultValue)
From source file:org.languagetool.server.TextChecker.java
void checkText(AnnotatedText aText, HttpExchange httpExchange, Map<String, String> parameters, ErrorRequestLimiter errorRequestLimiter, String remoteAddress) throws Exception { checkParams(parameters);/*from ww w. j av a 2 s. co m*/ long timeStart = System.currentTimeMillis(); UserLimits limits = ServerTools.getUserLimits(parameters, config); // logging information String agent = parameters.get("useragent") != null ? parameters.get("useragent") : "-"; Long agentId = null, userId = null; if (logger.isLogging()) { DatabaseAccess db = DatabaseAccess.getInstance(); agentId = db.getOrCreateClientId(parameters.get("useragent")); userId = limits.getPremiumUid(); } String referrer = httpExchange.getRequestHeaders().getFirst("Referer"); String userAgent = httpExchange.getRequestHeaders().getFirst("User-Agent"); if (aText.getPlainText().length() > limits.getMaxTextLength()) { String msg = "limit: " + limits.getMaxTextLength() + ", size: " + aText.getPlainText().length(); logger.log(new DatabaseAccessLimitLogEntry("MaxCharacterSizeExceeded", logServerId, agentId, userId, msg, referrer, userAgent)); ServerMetricsCollector.getInstance() .logRequestError(ServerMetricsCollector.RequestErrorType.MAX_TEXT_SIZE); throw new TextTooLongException( "Your text exceeds the limit of " + limits.getMaxTextLength() + " characters (it's " + aText.getPlainText().length() + " characters). Please submit a shorter text."); } UserConfig userConfig = new UserConfig( limits.getPremiumUid() != null ? getUserDictWords(limits.getPremiumUid()) : Collections.emptyList(), new HashMap<>(), config.getMaxSpellingSuggestions()); // NOTE: at the moment, feedback for A/B-Tests is only delivered from this client, so only run tests there if (agent != null && agent.equals("ltorg")) { userConfig.setAbTest(config.getAbTest()); } //print("Check start: " + text.length() + " chars, " + langParam); boolean autoDetectLanguage = getLanguageAutoDetect(parameters); List<String> preferredVariants = getPreferredVariants(parameters); if (parameters.get("noopLanguages") != null && !autoDetectLanguage) { ServerMetricsCollector.getInstance() .logRequestError(ServerMetricsCollector.RequestErrorType.INVALID_REQUEST); throw new IllegalArgumentException( "You can specify 'noopLanguages' only when also using 'language=auto'"); } List<String> noopLangs = parameters.get("noopLanguages") != null ? Arrays.asList(parameters.get("noopLanguages").split(",")) : Collections.emptyList(); List<String> preferredLangs = parameters.get("preferredLanguages") != null ? Arrays.asList(parameters.get("preferredLanguages").split(",")) : Collections.emptyList(); DetectedLanguage detLang = getLanguage(aText.getPlainText(), parameters, preferredVariants, noopLangs, preferredLangs); Language lang = detLang.getGivenLanguage(); Integer count = languageCheckCounts.get(lang.getShortCodeWithCountryAndVariant()); if (count == null) { count = 1; } else { count++; } //print("Starting check: " + aText.getPlainText().length() + " chars, #" + count); String motherTongueParam = parameters.get("motherTongue"); Language motherTongue = motherTongueParam != null ? Languages.getLanguageForShortCode(motherTongueParam) : null; boolean useEnabledOnly = "yes".equals(parameters.get("enabledOnly")) || "true".equals(parameters.get("enabledOnly")); List<Language> altLanguages = new ArrayList<>(); if (parameters.get("altLanguages") != null) { String[] altLangParams = parameters.get("altLanguages").split(",\\s*"); for (String langCode : altLangParams) { Language altLang = Languages.getLanguageForShortCode(langCode); altLanguages.add(altLang); if (altLang.hasVariant() && !altLang.isVariant()) { ServerMetricsCollector.getInstance() .logRequestError(ServerMetricsCollector.RequestErrorType.INVALID_REQUEST); throw new IllegalArgumentException("You specified altLanguage '" + langCode + "', but for this language you need to specify a variant, e.g. 'en-GB' instead of just 'en'"); } } } List<String> enabledRules = getEnabledRuleIds(parameters); List<String> disabledRules = getDisabledRuleIds(parameters); List<CategoryId> enabledCategories = getCategoryIds("enabledCategories", parameters); List<CategoryId> disabledCategories = getCategoryIds("disabledCategories", parameters); if ((disabledRules.size() > 0 || disabledCategories.size() > 0) && useEnabledOnly) { ServerMetricsCollector.getInstance() .logRequestError(ServerMetricsCollector.RequestErrorType.INVALID_REQUEST); throw new IllegalArgumentException( "You cannot specify disabled rules or categories using enabledOnly=true"); } if (enabledRules.isEmpty() && enabledCategories.isEmpty() && useEnabledOnly) { ServerMetricsCollector.getInstance() .logRequestError(ServerMetricsCollector.RequestErrorType.INVALID_REQUEST); throw new IllegalArgumentException( "You must specify enabled rules or categories when using enabledOnly=true"); } boolean useQuerySettings = enabledRules.size() > 0 || disabledRules.size() > 0 || enabledCategories.size() > 0 || disabledCategories.size() > 0; boolean allowIncompleteResults = "true".equals(parameters.get("allowIncompleteResults")); boolean enableHiddenRules = "true".equals(parameters.get("enableHiddenRules")); JLanguageTool.Mode mode = ServerTools.getMode(parameters); String callback = parameters.get("callback"); QueryParams params = new QueryParams(altLanguages, enabledRules, disabledRules, enabledCategories, disabledCategories, useEnabledOnly, useQuerySettings, allowIncompleteResults, enableHiddenRules, mode, callback); Long textSessionId = null; try { if (parameters.containsKey("textSessionId")) { String textSessionIdStr = parameters.get("textSessionId"); if (textSessionIdStr.contains(":")) { // transitioning to new format used in chrome addon // format: "{random number in 0..99999}:{unix time}" long random, timestamp; int sepPos = textSessionIdStr.indexOf(':'); random = Long.valueOf(textSessionIdStr.substring(0, sepPos)); timestamp = Long.valueOf(textSessionIdStr.substring(sepPos + 1)); // use random number to choose a slice in possible range of values // then choose position in slice by timestamp long maxRandom = 100000; long randomSegmentSize = (Long.MAX_VALUE - maxRandom) / maxRandom; long segmentOffset = random * randomSegmentSize; if (timestamp > randomSegmentSize) { print(String.format("Could not transform textSessionId '%s'", textSessionIdStr)); } textSessionId = segmentOffset + timestamp; } else { textSessionId = Long.valueOf(textSessionIdStr); } userConfig.setTextSessionId(textSessionId); } } catch (NumberFormatException ex) { print("Could not parse textSessionId '" + parameters.get("textSessionId") + "' as long: " + ex.getMessage()); } int textSize = aText.getPlainText().length(); List<RuleMatch> ruleMatchesSoFar = Collections.synchronizedList(new ArrayList<>()); Future<List<RuleMatch>> future = executorService.submit(new Callable<List<RuleMatch>>() { @Override public List<RuleMatch> call() throws Exception { // use to fake OOM in thread for testing: /*if (Math.random() < 0.1) { throw new OutOfMemoryError(); }*/ return getRuleMatches(aText, lang, motherTongue, parameters, params, userConfig, f -> ruleMatchesSoFar.add(f)); } }); String incompleteResultReason = null; List<RuleMatch> matches; try { if (limits.getMaxCheckTimeMillis() < 0) { matches = future.get(); } else { matches = future.get(limits.getMaxCheckTimeMillis(), TimeUnit.MILLISECONDS); } } catch (ExecutionException e) { future.cancel(true); if (ExceptionUtils.getRootCause(e) instanceof ErrorRateTooHighException) { ServerMetricsCollector.getInstance() .logRequestError(ServerMetricsCollector.RequestErrorType.TOO_MANY_ERRORS); logger.log(new DatabaseCheckErrorLogEntry("ErrorRateTooHigh", logServerId, agentId, userId, lang, detLang.getDetectedLanguage(), textSize, "matches: " + ruleMatchesSoFar.size())); } if (params.allowIncompleteResults && ExceptionUtils.getRootCause(e) instanceof ErrorRateTooHighException) { print(e.getMessage() + " - returning " + ruleMatchesSoFar.size() + " matches found so far. Detected language: " + detLang); matches = new ArrayList<>(ruleMatchesSoFar); // threads might still be running, so make a copy incompleteResultReason = "Results are incomplete: " + ExceptionUtils.getRootCause(e).getMessage(); } else if (e.getCause() != null && e.getCause() instanceof OutOfMemoryError) { throw (OutOfMemoryError) e.getCause(); } else { throw new RuntimeException(e.getMessage() + ", detected: " + detLang, e); } } catch (TimeoutException e) { boolean cancelled = future.cancel(true); Path loadFile = Paths.get("/proc/loadavg"); // works in Linux only(?) String loadInfo = loadFile.toFile().exists() ? Files.readAllLines(loadFile).toString() : "(unknown)"; if (errorRequestLimiter != null) { errorRequestLimiter.logAccess(remoteAddress, httpExchange.getRequestHeaders(), parameters); } String message = "Text checking took longer than allowed maximum of " + limits.getMaxCheckTimeMillis() + " milliseconds (cancelled: " + cancelled + ", lang: " + lang.getShortCodeWithCountryAndVariant() + ", detected: " + detLang + ", #" + count + ", " + aText.getPlainText().length() + " characters of text" + ", mode: " + mode.toString().toLowerCase() + ", h: " + reqCounter.getHandleCount() + ", r: " + reqCounter.getRequestCount() + ", system load: " + loadInfo + ")"; if (params.allowIncompleteResults) { print(message + " - returning " + ruleMatchesSoFar.size() + " matches found so far"); matches = new ArrayList<>(ruleMatchesSoFar); // threads might still be running, so make a copy incompleteResultReason = "Results are incomplete: text checking took longer than allowed maximum of " + String.format(Locale.ENGLISH, "%.2f", limits.getMaxCheckTimeMillis() / 1000.0) + " seconds"; } else { ServerMetricsCollector.getInstance() .logRequestError(ServerMetricsCollector.RequestErrorType.MAX_CHECK_TIME); logger.log(new DatabaseCheckErrorLogEntry("MaxCheckTimeExceeded", logServerId, agentId, limits.getPremiumUid(), lang, detLang.getDetectedLanguage(), textSize, "load: " + loadInfo)); throw new RuntimeException(message, e); } } setHeaders(httpExchange); List<RuleMatch> hiddenMatches = new ArrayList<>(); if (config.getHiddenMatchesServer() != null && params.enableHiddenRules && config.getHiddenMatchesLanguages().contains(lang)) { if (config.getHiddenMatchesServerFailTimeout() > 0 && lastHiddenMatchesServerTimeout != -1 && System.currentTimeMillis() - lastHiddenMatchesServerTimeout < config .getHiddenMatchesServerFailTimeout()) { ServerMetricsCollector.getInstance().logHiddenServerStatus(false); print("Warn: Skipped querying hidden matches server at " + config.getHiddenMatchesServer() + " because of recent error/timeout (timeout=" + config.getHiddenMatchesServerFailTimeout() + "ms)."); } else { ResultExtender resultExtender = new ResultExtender(config.getHiddenMatchesServer(), config.getHiddenMatchesServerTimeout()); try { long start = System.currentTimeMillis(); List<RemoteRuleMatch> extensionMatches = resultExtender .getExtensionMatches(aText.getPlainText(), parameters); hiddenMatches = resultExtender.getFilteredExtensionMatches(matches, extensionMatches); long end = System.currentTimeMillis(); print("Hidden matches: " + extensionMatches.size() + " -> " + hiddenMatches.size() + " in " + (end - start) + "ms for " + lang.getShortCodeWithCountryAndVariant()); ServerMetricsCollector.getInstance().logHiddenServerStatus(true); lastHiddenMatchesServerTimeout = -1; } catch (Exception e) { ServerMetricsCollector.getInstance().logHiddenServerStatus(false); print("Warn: Failed to query hidden matches server at " + config.getHiddenMatchesServer() + ": " + e.getClass() + ": " + e.getMessage()); lastHiddenMatchesServerTimeout = System.currentTimeMillis(); } } } int compactMode = Integer.parseInt(parameters.getOrDefault("c", "0")); String response = getResponse(aText, detLang, motherTongue, matches, hiddenMatches, incompleteResultReason, compactMode); if (params.callback != null) { // JSONP - still needed today for the special case of hosting your own on-premise LT without SSL // and using it from a local MS Word (not Online Word) - issue #89 in the add-in repo: response = params.callback + "(" + response + ");"; } String messageSent = "sent"; String languageMessage = lang.getShortCodeWithCountryAndVariant(); try { httpExchange.sendResponseHeaders(HttpURLConnection.HTTP_OK, response.getBytes(ENCODING).length); httpExchange.getResponseBody().write(response.getBytes(ENCODING)); ServerMetricsCollector.getInstance().logResponse(HttpURLConnection.HTTP_OK); } catch (IOException exception) { // the client is disconnected messageSent = "notSent: " + exception.getMessage(); } if (motherTongue != null) { languageMessage += " (mother tongue: " + motherTongue.getShortCodeWithCountryAndVariant() + ")"; } if (autoDetectLanguage) { languageMessage += "[auto]"; } languageCheckCounts.put(lang.getShortCodeWithCountryAndVariant(), count); int computationTime = (int) (System.currentTimeMillis() - timeStart); String version = parameters.get("v") != null ? ", v:" + parameters.get("v") : ""; print("Check done: " + aText.getPlainText().length() + " chars, " + languageMessage + ", #" + count + ", " + referrer + ", " + matches.size() + " matches, " + computationTime + "ms, agent:" + agent + version + ", " + messageSent + ", q:" + (workQueue != null ? workQueue.size() : "?") + ", h:" + reqCounter.getHandleCount() + ", dH:" + reqCounter.getDistinctIps() + ", m:" + mode.toString().toLowerCase()); int matchCount = matches.size(); Map<String, Integer> ruleMatchCount = new HashMap<>(); for (RuleMatch match : matches) { String ruleId = match.getRule().getId(); ruleMatchCount.put(ruleId, ruleMatchCount.getOrDefault(ruleId, 0) + 1); } ServerMetricsCollector.getInstance().logCheck(lang, computationTime, textSize, matchCount, mode, agent, ruleMatchCount); if (!config.isSkipLoggingChecks()) { DatabaseCheckLogEntry logEntry = new DatabaseCheckLogEntry(userId, agentId, logServerId, textSize, matchCount, lang, detLang.getDetectedLanguage(), computationTime, textSessionId, mode.toString()); logEntry.setRuleMatches(new DatabaseRuleMatchLogEntry( config.isSkipLoggingRuleMatches() ? Collections.emptyMap() : ruleMatchCount)); logger.log(logEntry); } }