Java tutorial
/* * See the NOTICE file distributed with this work for additional * information regarding copyright ownership. * * This is free software; you can redistribute it and/or modify it * under the terms of the GNU Lesser General Public License as * published by the Free Software Foundation; either version 2.1 of * the License, or (at your option) any later version. * * This software is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this software; if not, write to the Free * Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA * 02110-1301 USA, or see the FSF site: http://www.fsf.org. */ package org.xwiki.test.escaping.framework; import java.io.BufferedReader; import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStreamReader; import java.io.Reader; import java.io.UnsupportedEncodingException; import java.net.URLEncoder; import java.util.Arrays; import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; import java.util.regex.Matcher; import java.util.regex.Pattern; import org.apache.commons.codec.binary.Base64; import org.apache.commons.httpclient.Credentials; import org.apache.commons.httpclient.HttpClient; import org.apache.commons.httpclient.HttpStatus; import org.apache.commons.httpclient.InvalidRedirectLocationException; import org.apache.commons.httpclient.UsernamePasswordCredentials; import org.apache.commons.httpclient.auth.AuthScope; import org.apache.commons.httpclient.methods.GetMethod; import org.apache.commons.httpclient.params.HttpClientParams; import org.apache.commons.httpclient.params.HttpConnectionManagerParams; import org.junit.AfterClass; import org.junit.Assert; import org.junit.BeforeClass; import org.xwiki.test.escaping.suite.FileTest; import org.xwiki.validator.ValidationError; /** * Abstract base class for escaping tests. Implements common initialization pattern and some utility methods like URL * escaping, retrieving page content by URL etc. Subclasses need to implement parsing and custom tests. * <p> * Note: JUnit4 requires tests to have one public default constructor, subclasses will need to implement it and pass * pattern matcher to match file names they can handle. * <p> * Starting and stopping XWiki server is handled transparently for all subclasses, tests can be run alone using * -Dtest=ClassName, a parent test suite should start XWiki server before running all tests for efficiency using * {@link SingleXWikiExecutor}. * <p> * The following configuration properties are supported (set in maven): * <ul> * <li>pattern (optional): Additional pattern to select files to be tested (use -Dpattern="substring-regex"). Matches * all files if empty.</li> * </ul> * <p> * Automatic tests (see {@link AbstractAutomaticTest}) additionally support: * <ul> * <li>patternExcludeFiles (optional): List of RegEx patterns to exclude files from the tests</li> * </ul> * * @version $Id: 37857c9a69ce897c14c9d9418e6e55dcc39d01af $ * @since 2.5M1 */ public abstract class AbstractEscapingTest implements FileTest { /** Static part of the test URL. */ private static final String URL_START = "http://127.0.0.1:8080/xwiki/bin/"; /** Language parameter name. */ private static final String LANGUAGE = "language"; /** Secret token parameter name. */ private static final String SECRET_TOKEN = "form_token"; /** HTTP client shared between all subclasses. */ private static HttpClient client; /** A flag controlling login. If true, administrator credentials are used. */ private static boolean loggedIn = true; /** Stores two cached tokens, one for each value of loggedIn (false -> 0, true -> 1). */ private static String[] secretTokens = new String[2]; private static Set<String> XML_MIMETYPES = new HashSet<>( Arrays.asList("text/html", "text/xml", "application/xml")); /** File name of the template to use. */ protected String name; /** User provided data found in the file. */ protected Set<String> userInput; /** Pattern used to match files by name. */ private Pattern namePattern; /** * Create new AbstractEscapingTest. * * @param fileNameMatcher regex pattern used to filter files by name */ protected AbstractEscapingTest(Pattern fileNameMatcher) { this.namePattern = fileNameMatcher; } /** * Start XWiki server if run alone. * * @throws Exception on errors */ @BeforeClass public static void startExecutor() throws Exception { SingleXWikiExecutor.getExecutor().start(); } /** * Stop XWiki server if run alone. * * @throws Exception on errors */ @AfterClass public static void stopExecutor() throws Exception { SingleXWikiExecutor.getExecutor().stop(); } /** * Change multi-language mode. Note: XWiki server must already be started. * * @param enabled enable the multi-language mode if true, disable otherwise */ protected static void setMultiLanguageMode(boolean enabled) { String url = AbstractEscapingTest.URL_START + "save/XWiki/XWikiPreferences?"; url += SECRET_TOKEN + "=" + getSecretToken(); url += "&XWiki.XWikiPreferences_0_languages=&XWiki.XWikiPreferences_0_multilingual="; AbstractEscapingTest.getUrlContent(url + (enabled ? 1 : 0)); // set language=en to prevent false positives coming from the cookies String langUrl = AbstractEscapingTest.URL_START + "view/Main/?" + LANGUAGE + "=en"; AbstractEscapingTest.getUrlContent(langUrl); } /** * {@inheritDoc} * <p> * The implementation for escaping tests checks if the given file name matches the supported name pattern and parses * the file. * * @see org.xwiki.test.escaping.suite.FileTest#initialize(java.lang.String, java.io.Reader) */ @Override public boolean initialize(String name, final Reader reader) { this.name = name; if (!fileNameMatches(name) || !patternMatches(name) || isExcludedFile(name)) { // TODO debug log the reason why the test was skipped return false; } this.userInput = parse(reader); return true; } /** * Check if the internal file name pattern matches the given file name. * * @param fileName file name to check * @return true if the name matches, false otherwise */ protected boolean fileNameMatches(String fileName) { return this.namePattern != null && this.namePattern.matcher(fileName).matches(); } /** * Check if the system property "pattern" matches (substring regular expression) the file name. Empty pattern * matches everything. * * @param fileName file name to check * @return true if the pattern matches, false otherwise */ protected boolean patternMatches(String fileName) { String pattern = System.getProperty("pattern", ""); if (pattern == null || pattern.equals("")) { return true; } return Pattern.matches(".*" + pattern + ".*", fileName); } /** * Check if the given file should be excluded from the tests. * * @param fileName file name to check * @return true if the file should be excluded, false otherwise */ protected abstract boolean isExcludedFile(String fileName); /** * Parse the file and collect parameters controlled by the user. * * @param reader the reader associated with the file * @return collection of user-controlled input parameters */ protected abstract Set<String> parse(Reader reader); /** * Check if the authentication status. * * @return true if the requests will be sent authenticated as admin, false otherwise */ protected static boolean isLoggedIn() { return loggedIn; } /** * Set authentication status. * * @param value the value to set */ protected static void setLoggedIn(boolean value) { loggedIn = value; } /** * Download a page from the server and return its content. Throws a {@link RuntimeException} on connection problems * etc. * * @param url URL of the page * @return content of the page */ protected static URLContent getUrlContent(String url) { GetMethod get = new GetMethod(url); get.setFollowRedirects(true); if (isLoggedIn()) { get.setDoAuthentication(true); get.addRequestHeader("Authorization", "Basic " + new String(Base64.encodeBase64("Admin:admin".getBytes()))); } try { int statusCode = AbstractEscapingTest.getClient().executeMethod(get); switch (statusCode) { case HttpStatus.SC_OK: // everything is fine break; case HttpStatus.SC_UNAUTHORIZED: // do not fail on 401 (unauthorized), used in some tests System.out.println("WARNING, Ignoring status 401 (unauthorized) for URL: " + url); break; case HttpStatus.SC_CONFLICT: // do not fail on 409 (conflict), used in some templates System.out.println("WARNING, Ignoring status 409 (conflict) for URL: " + url); break; case HttpStatus.SC_NOT_FOUND: // ignore 404 (the page is still rendered) break; case HttpStatus.SC_INTERNAL_SERVER_ERROR: // ignore 500 (internal server error), which is used by the standard exception.vm error display break; default: throw new RuntimeException("HTTP GET request returned status " + statusCode + " (" + get.getStatusText() + ") for URL: " + url); } return new URLContent(get.getResponseHeader("Content-Type").getValue(), get.getResponseBody()); } catch (IOException exception) { throw new RuntimeException("Error retrieving URL: " + url, exception); } finally { get.releaseConnection(); } } /** * URL-escape given string. * * @param str string to escape, "" is used if null * @return URL-escaped {@code str} */ protected static String escapeUrl(String str) { try { return URLEncoder.encode(str == null ? "" : str, "UTF-8"); } catch (UnsupportedEncodingException exception) { // should not happen throw new RuntimeException("Should not happen: ", exception); } } /** * Get an instance of the HTTP client to use. * * @return HTTP client initialized with admin credentials */ protected static HttpClient getClient() { if (AbstractEscapingTest.client == null) { HttpClient adminClient = new HttpClient(); // set up admin credentials Credentials defaultcreds = new UsernamePasswordCredentials("Admin", "admin"); adminClient.getState().setCredentials(AuthScope.ANY, defaultcreds); // set up client parameters HttpClientParams clientParams = new HttpClientParams(); clientParams.setSoTimeout(20000); // We need to allow circular redirects, because some templates redirect to the same location with different // query parameters and the check for circular redirect in HttpClient only checks the URI path without the // parameters. // Note that actual circular redirects are still aborted after following them for some fixed number of times clientParams.setBooleanParameter(HttpClientParams.ALLOW_CIRCULAR_REDIRECTS, true); adminClient.setParams(clientParams); // set up connections parameters HttpConnectionManagerParams connectionParams = new HttpConnectionManagerParams(); connectionParams.setConnectionTimeout(30000); adminClient.getHttpConnectionManager().setParams(connectionParams); AbstractEscapingTest.client = adminClient; } return AbstractEscapingTest.client; } @Override public String toString() { return this.name + ' ' + this.userInput; } /** * Check for unescaped data in the given {@code content}. Throws {@link RuntimeException} on errors. * * @param url URL used in the test * @return list of found validation errors */ protected List<ValidationError> getUnderEscapingErrors(String url) { // TODO better use XWiki logging System.out.println("Testing URL: " + url); URLContent content = null; try { content = AbstractEscapingTest.getUrlContent(url); } catch (RuntimeException e) { if (e.getCause() instanceof InvalidRedirectLocationException) { // Don't fail the test if we can't follow a redirect because the redirect location can be taken from the // request parameters which are controlled by the test and most of the tests use values that are not // valid URLs. The code that performs the redirect always assumes the redirect URL is valid. System.out.println(e.getCause().getMessage()); return Collections.emptyList(); } else { throw e; } } // TODO: add support for other types than XML if (content.getType() == null || XML_MIMETYPES.contains(content.getType().getMimeType()) || content.getType().getMimeType().endsWith("+xml")) { String where = " Template: " + this.name + "\n URL: " + url; Assert.assertNotNull("Response is null\n" + where, content); XMLEscapingValidator validator = new XMLEscapingValidator(); validator.setDocument(new ByteArrayInputStream(content.getContent())); try { return validator.validate(); } catch (EscapingError error) { // most probably false positive, generate an error instead of failing the test throw new RuntimeException(EscapingError.formatMessage(error.getMessage(), this.name, url, null)); } } else { System.err.println("WARN: Unsupported content type [" + content.getType() + "] for URL [" + url + "]"); return Collections.emptyList(); } } /** * A convenience method that throws an {@link EscapingError} on failure. * * @param url URL used in the test * @param description description of the test */ protected void checkUnderEscaping(String url, String description) { List<ValidationError> errors = getUnderEscapingErrors(url); if (!errors.isEmpty()) { throw new EscapingError("Escaping test for " + description + " failed.", this.name, url, errors); } } /** * Create the target URL from the given parameters. URL-escapes everything. Adds language=en if the parameter map * does not contain language parameter. * * @param action action to use, "view" is used if null * @param space space name to use, "Main" is used if null * @param page page name to use, "WebHome" is used if null * @param parameters list of parameters with values, parameters are omitted if null, "" is used is a value is null * @return the resulting absolute URL */ protected static String createUrl(String action, String space, String page, Map<String, String> parameters) { return createUrl(action, space, page, parameters, true); } /** * Create the target URL from the given parameters. URL-escapes everything. Adds secret token if needed. * * @param action action to use, "view" is used if null * @param space space name to use, "Main" is used if null * @param page page name to use, "WebHome" is used if null * @param parameters list of parameters with values, parameters are omitted if null, "" is used is a value is null * @param addLanguage add language=en if it is not set in the parameter map * @return the resulting absolute URL */ protected static String createUrl(String action, String space, String page, Map<String, String> parameters, boolean addLanguage) { String url = URL_START + escapeUrl(action == null ? "view" : action) + "/"; url += escapeUrl(space == null ? "Main" : space) + "/"; url += escapeUrl(page == null ? "WebHome" : page); String delimiter = "?"; if (parameters != null) { for (String parameter : parameters.keySet()) { if (parameter != null && !parameter.equals("")) { String value = parameters.get(parameter); url += delimiter + escapeUrl(parameter) + "=" + escapeUrl(value); } delimiter = "&"; } } // special handling for language parameter to exclude false positives (language setting is saved in cookies and // sent on subsequent requests) if (addLanguage && (parameters == null || !parameters.containsKey(LANGUAGE))) { url += delimiter + LANGUAGE + "=en"; } // some tests need to create or delete pages, we add secret token to avoid CSRF protection failures if ((action == null || !action.equals("edit")) && (parameters == null || !parameters.containsKey(SECRET_TOKEN))) { url += delimiter + SECRET_TOKEN + "=" + getSecretToken(); } return url; } /** * Get the secret token used for CSRF protection. Caches 2 tokens (for logged in and logged out) on the first call. * * @return anti-CSRF secret token, or empty string on error * @since 3.2M1 */ protected static String getSecretToken() { int index = isLoggedIn() ? 1 : 0; if (secretTokens[index] == null) { secretTokens[index] = getSecretTokenFromPage(); } return secretTokens[index]; } /** * Parse a wiki page to get the current secret token. * * @return secret token */ private static String getSecretTokenFromPage() { Pattern pattern = Pattern.compile("<input[^>]+" + SECRET_TOKEN + "[^>]+value=('|\")([^'\"]+)"); try { String url = createUrl("edit", "Main", "WebHome", null); BufferedReader reader = new BufferedReader(new InputStreamReader( new ByteArrayInputStream(AbstractEscapingTest.getUrlContent(url).getContent()))); String line; while ((line = reader.readLine()) != null) { Matcher matcher = pattern.matcher(line); if (matcher.find() && matcher.groupCount() == 2) { return matcher.group(2); } } } catch (IOException exception) { exception.printStackTrace(); } // something went really wrong System.out.println("WARNING, Failed to cache anti-CSRF secret token, some tests might fail!"); return ""; } }