Java tutorial
/* * Overchan Android (Meta Imageboard Client) * Copyright (C) 2014-2016 miku-nyan <https://github.com/miku-nyan> * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation, either version 3 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program. If not, see <http://www.gnu.org/licenses/>. */ package nya.miku.wishmaster.chans.krautchan; import java.io.ByteArrayOutputStream; import java.io.Closeable; import java.util.ArrayList; import java.util.Arrays; import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Locale; import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; import org.apache.commons.lang3.StringEscapeUtils; import cz.msebera.android.httpclient.Header; import cz.msebera.android.httpclient.HttpHeaders; import cz.msebera.android.httpclient.NameValuePair; import cz.msebera.android.httpclient.client.entity.UrlEncodedFormEntity; import cz.msebera.android.httpclient.cookie.Cookie; import cz.msebera.android.httpclient.impl.cookie.BasicClientCookie; import cz.msebera.android.httpclient.message.BasicNameValuePair; import android.content.Context; import android.content.SharedPreferences; import android.content.res.Resources; import android.graphics.drawable.Drawable; import android.preference.EditTextPreference; import android.preference.Preference; import android.preference.PreferenceGroup; import android.support.v4.content.res.ResourcesCompat; import nya.miku.wishmaster.R; import nya.miku.wishmaster.api.CloudflareChanModule; import nya.miku.wishmaster.api.interfaces.CancellableTask; import nya.miku.wishmaster.api.interfaces.ProgressListener; import nya.miku.wishmaster.api.models.BoardModel; import nya.miku.wishmaster.api.models.CaptchaModel; import nya.miku.wishmaster.api.models.DeletePostModel; import nya.miku.wishmaster.api.models.PostModel; import nya.miku.wishmaster.api.models.SendPostModel; import nya.miku.wishmaster.api.models.SimpleBoardModel; import nya.miku.wishmaster.api.models.ThreadModel; import nya.miku.wishmaster.api.models.UrlPageModel; import nya.miku.wishmaster.api.util.ChanModels; import nya.miku.wishmaster.api.util.RegexUtils; import nya.miku.wishmaster.common.IOUtils; import nya.miku.wishmaster.common.Logger; import nya.miku.wishmaster.http.ExtendedMultipartBuilder; import nya.miku.wishmaster.http.streamer.HttpRequestModel; import nya.miku.wishmaster.http.streamer.HttpResponseModel; import nya.miku.wishmaster.http.streamer.HttpStreamer; import nya.miku.wishmaster.http.streamer.HttpWrongStatusCodeException; import nya.miku.wishmaster.lib.org_json.JSONObject; public class KrautModule extends CloudflareChanModule { private static final String TAG = "KrautModule"; static final String CHAN_NAME = "krautchan.net"; private static final String CHAN_DOMAIN = "krautchan.net"; private static final String PREF_KEY_KOMPTURCODE_COOKIE = "PREF_KEY_KOMPTURCODE_COOKIE"; private static final String KOMTURCODE_COOKIE_NAME = "desuchan.komturcode"; private Map<String, BoardModel> boardsMap = null; private String lastCaptchaId = null; public KrautModule(SharedPreferences preferences, Resources resources) { super(preferences, resources); } @Override public String getChanName() { return CHAN_NAME; } @Override public String getDisplayingName() { return "Krautchan"; } @Override public Drawable getChanFavicon() { return ResourcesCompat.getDrawable(resources, R.drawable.favicon_krautchan, null); } @Override protected void initHttpClient() { super.initHttpClient(); setKompturcodeCookie(preferences.getString(getSharedKey(PREF_KEY_KOMPTURCODE_COOKIE), null)); } @Override protected String getCloudflareCookieDomain() { return CHAN_DOMAIN; } private void setKompturcodeCookie(String kompturcodeCookie) { if (kompturcodeCookie != null && kompturcodeCookie.length() > 0) { BasicClientCookie c = new BasicClientCookie(KOMTURCODE_COOKIE_NAME, kompturcodeCookie); c.setDomain(CHAN_DOMAIN); httpClient.getCookieStore().addCookie(c); } } public void addKompturcodePreference(PreferenceGroup preferenceGroup) { Context context = preferenceGroup.getContext(); EditTextPreference kompturcodePreference = new EditTextPreference(context); kompturcodePreference.setTitle(R.string.kraut_prefs_kompturcode); kompturcodePreference.setDialogTitle(R.string.kraut_prefs_kompturcode); kompturcodePreference.setSummary(R.string.kraut_prefs_kompturcode_summary); kompturcodePreference.setKey(getSharedKey(PREF_KEY_KOMPTURCODE_COOKIE)); kompturcodePreference.setOnPreferenceChangeListener(new Preference.OnPreferenceChangeListener() { @Override public boolean onPreferenceChange(Preference preference, Object newValue) { setKompturcodeCookie((String) newValue); return true; } }); preferenceGroup.addPreference(kompturcodePreference); } @Override public void addPreferencesOnScreen(PreferenceGroup preferenceGroup) { addKompturcodePreference(preferenceGroup); addPasswordPreference(preferenceGroup); addHttpsPreference(preferenceGroup, true); addCloudflareRecaptchaFallbackPreference(preferenceGroup); addProxyPreferences(preferenceGroup); } private boolean useHttps() { return useHttps(true); } /** * If (url == null) returns boards list (SimpleBoardModel[]), thread/threads page (ThreadModel[]) otherwise */ private Object readPage(String url, ProgressListener listener, CancellableTask task, boolean checkIfModified) throws Exception { boolean boardsList = url == null; if (boardsList) url = (useHttps() ? "https://" : "http://") + CHAN_DOMAIN + "/nav"; boolean catalog = boardsList ? false : url.contains("/catalog/"); HttpResponseModel responseModel = null; Closeable in = null; HttpRequestModel rqModel = HttpRequestModel.builder().setGET().setCheckIfModified(checkIfModified).build(); try { responseModel = HttpStreamer.getInstance().getFromUrl(url, rqModel, httpClient, listener, task); if (responseModel.statusCode == 200) { in = boardsList ? new KrautBoardsListReader(responseModel.stream) : (catalog ? new KrautCatalogReader(responseModel.stream) : new KrautReader(responseModel.stream)); if (task != null && task.isCancelled()) throw new Exception("interrupted"); return boardsList ? ((KrautBoardsListReader) in).readBoardsList() : (catalog ? ((KrautCatalogReader) in).readPage() : ((KrautReader) in).readPage()); } else { if (responseModel.notModified()) return null; byte[] html = null; try { ByteArrayOutputStream byteStream = new ByteArrayOutputStream(1024); IOUtils.copyStream(responseModel.stream, byteStream); html = byteStream.toByteArray(); } catch (Exception e) { } if (html != null) { checkCloudflareError(new HttpWrongStatusCodeException(responseModel.statusCode, responseModel.statusReason, html), url); } throw new HttpWrongStatusCodeException(responseModel.statusCode, responseModel.statusCode + " - " + responseModel.statusReason); } } catch (Exception e) { if (responseModel != null) HttpStreamer.getInstance().removeFromModifiedMap(url); throw e; } finally { IOUtils.closeQuietly(in); if (responseModel != null) responseModel.release(); } } @Override public SimpleBoardModel[] getBoardsList(ProgressListener listener, CancellableTask task, SimpleBoardModel[] oldBoardsList) throws Exception { SimpleBoardModel[] boardsList = (SimpleBoardModel[]) readPage(null, listener, task, oldBoardsList != null); if (boardsList == null) return oldBoardsList; Map<String, BoardModel> newMap = new HashMap<>(); for (SimpleBoardModel board : boardsList) { newMap.put(board.boardName, KrautBoardsListReader.getDefaultBoardModel(board.boardName, board.boardDescription, board.boardCategory)); } boardsMap = newMap; return boardsList; } @Override public BoardModel getBoard(String shortName, ProgressListener listener, CancellableTask task) throws Exception { if (boardsMap == null) { try { getBoardsList(listener, task, null); } catch (Exception e) { Logger.e(TAG, "cannot get boards list", e); } } if (boardsMap != null && boardsMap.containsKey(shortName)) return boardsMap.get(shortName); return KrautBoardsListReader.getDefaultBoardModel(shortName, shortName, null); } @Override public ThreadModel[] getThreadsList(String boardName, int page, ProgressListener listener, CancellableTask task, ThreadModel[] oldList) throws Exception { UrlPageModel urlModel = new UrlPageModel(); urlModel.chanName = CHAN_NAME; urlModel.type = UrlPageModel.TYPE_BOARDPAGE; urlModel.boardName = boardName; urlModel.boardPage = page; String url = buildUrl(urlModel); ThreadModel[] threads = (ThreadModel[]) readPage(url, listener, task, oldList != null); if (threads == null) { return oldList; } else { return threads; } } @Override public ThreadModel[] getCatalog(String boardName, int catalogType, ProgressListener listener, CancellableTask task, ThreadModel[] oldList) throws Exception { UrlPageModel urlModel = new UrlPageModel(); urlModel.chanName = CHAN_NAME; urlModel.type = UrlPageModel.TYPE_CATALOGPAGE; urlModel.boardName = boardName; String url = buildUrl(urlModel); ThreadModel[] threads = (ThreadModel[]) readPage(url, listener, task, oldList != null); if (threads == null) { return oldList; } else { return threads; } } @Override public PostModel[] getPostsList(String boardName, String threadNumber, ProgressListener listener, CancellableTask task, PostModel[] oldList) throws Exception { UrlPageModel urlModel = new UrlPageModel(); urlModel.chanName = CHAN_NAME; urlModel.type = UrlPageModel.TYPE_THREADPAGE; urlModel.boardName = boardName; urlModel.threadNumber = threadNumber; String url = buildUrl(urlModel); ThreadModel[] threads = (ThreadModel[]) readPage(url, listener, task, oldList != null); if (threads == null) { return oldList; } else { if (threads.length == 0) throw new Exception("Unable to parse response"); return oldList == null ? threads[0].posts : ChanModels.mergePostsLists(Arrays.asList(oldList), Arrays.asList(threads[0].posts)); } } @Override public CaptchaModel getNewCaptcha(String boardName, String threadNumber, ProgressListener listener, CancellableTask task) throws Exception { String url = (useHttps() ? "https://" : "http://") + CHAN_DOMAIN + "/ajax/checkpost?board=" + boardName; try { JSONObject data = HttpStreamer.getInstance() .getJSONObjectFromUrl(url, HttpRequestModel.DEFAULT_GET, httpClient, listener, task, true) .getJSONObject("data"); if (data.optString("captchas", "").equals("always")) { StringBuilder captchaUrlBuilder = new StringBuilder(); captchaUrlBuilder.append(useHttps() ? "https://" : "http://").append(CHAN_DOMAIN) .append("/captcha?id="); StringBuilder captchaIdBuilder = new StringBuilder(); captchaIdBuilder.append(boardName); if (threadNumber != null) captchaIdBuilder.append(threadNumber); for (Cookie cookie : httpClient.getCookieStore().getCookies()) { if (cookie.getName().equalsIgnoreCase("desuchan.session")) { captchaIdBuilder.append('-').append(cookie.getValue()); break; } } captchaIdBuilder.append('-').append(new Date().getTime()).append('-') .append(Math.round(100000000 * Math.random())); String captchaId = captchaIdBuilder.toString(); captchaUrlBuilder.append(captchaId); String captchaUrl = captchaUrlBuilder.toString(); CaptchaModel captchaModel = downloadCaptcha(captchaUrl, listener, task); lastCaptchaId = captchaId; return captchaModel; } } catch (HttpWrongStatusCodeException e) { checkCloudflareError(e, url); } catch (Exception e) { Logger.e(TAG, "exception while getting captcha", e); } return null; } @Override public String sendPost(SendPostModel model, ProgressListener listener, CancellableTask task) throws Exception { String url = (useHttps() ? "https://" : "http://") + CHAN_DOMAIN + "/post"; ExtendedMultipartBuilder postEntityBuilder = ExtendedMultipartBuilder.create().setDelegates(listener, task) .addString("internal_n", model.name).addString("internal_s", model.subject); if (model.sage) postEntityBuilder.addString("sage", "1"); postEntityBuilder.addString("internal_t", model.comment); if (lastCaptchaId != null) { postEntityBuilder.addString("captcha_name", lastCaptchaId).addString("captcha_secret", model.captchaAnswer); lastCaptchaId = null; } if (model.attachments != null) { String[] images = new String[] { "file_0", "file_1", "file_2", "file_3" }; for (int i = 0; i < model.attachments.length; ++i) { postEntityBuilder.addFile(images[i], model.attachments[i], model.randomHash); } } postEntityBuilder.addString("forward", "thread").addString("password", model.password).addString("board", model.boardName); if (model.threadNumber != null) postEntityBuilder.addString("parent", model.threadNumber); HttpRequestModel request = HttpRequestModel.builder().setPOST(postEntityBuilder.build()).setNoRedirect(true) .build(); HttpResponseModel response = null; try { response = HttpStreamer.getInstance().getFromUrl(url, request, httpClient, null, task); if (response.statusCode == 302) { for (Header header : response.headers) { if (header != null && HttpHeaders.LOCATION.equalsIgnoreCase(header.getName())) { String location = header.getValue(); if (location.contains("banned")) throw new Exception("You are banned"); return fixRelativeUrl(header.getValue()); } } } else if (response.statusCode == 200) { ByteArrayOutputStream output = new ByteArrayOutputStream(1024); IOUtils.copyStream(response.stream, output); String htmlResponse = output.toString("UTF-8"); int messageErrorPos = htmlResponse.indexOf("class=\"message_error"); if (messageErrorPos == -1) return null; //assume success int p2 = htmlResponse.indexOf('>', messageErrorPos); if (p2 != -1) { String errorMessage = htmlResponse.substring(p2 + 1); int p3 = errorMessage.indexOf("</tr>"); if (p3 != -1) errorMessage = errorMessage.substring(0, p3); errorMessage = RegexUtils.trimToSpace( StringEscapeUtils.unescapeHtml4(RegexUtils.removeHtmlTags(errorMessage)).trim()); throw new Exception(errorMessage); } } throw new HttpWrongStatusCodeException(response.statusCode, response.statusCode + " - " + response.statusReason); } finally { if (response != null) response.release(); } } @Override public String deletePost(DeletePostModel model, ProgressListener listener, CancellableTask task) throws Exception { String url = (useHttps() ? "https://" : "http://") + CHAN_DOMAIN + "/delete"; List<NameValuePair> pairs = new ArrayList<NameValuePair>(); pairs.add(new BasicNameValuePair("post_" + model.postNumber, "delete")); pairs.add(new BasicNameValuePair("password", model.password)); pairs.add(new BasicNameValuePair("board", model.boardName)); HttpRequestModel request = HttpRequestModel.builder().setPOST(new UrlEncodedFormEntity(pairs, "UTF-8")) .setNoRedirect(true).build(); HttpResponseModel response = null; try { response = HttpStreamer.getInstance().getFromUrl(url, request, httpClient, null, task); if (response.statusCode == 302) { for (Header header : response.headers) { if (header != null && HttpHeaders.LOCATION.equalsIgnoreCase(header.getName())) { String location = header.getValue(); if (location.contains("banned")) throw new Exception("You are banned"); break; } } return null; } else if (response.statusCode == 200) { ByteArrayOutputStream output = new ByteArrayOutputStream(1024); IOUtils.copyStream(response.stream, output); String htmlResponse = output.toString("UTF-8"); int messageNoticePos = htmlResponse.indexOf("class=\"message_notice"); if (messageNoticePos == -1) return null; int p2 = htmlResponse.indexOf('>', messageNoticePos); if (p2 != -1) { String errorMessage = htmlResponse.substring(p2 + 1); int p3 = errorMessage.indexOf("</tr>"); if (p3 != -1) errorMessage = errorMessage.substring(0, p3); errorMessage = RegexUtils.trimToSpace( StringEscapeUtils.unescapeHtml4(RegexUtils.removeHtmlTags(errorMessage)).trim()); throw new Exception(errorMessage); } } throw new HttpWrongStatusCodeException(response.statusCode, response.statusCode + " - " + response.statusReason); } finally { if (response != null) response.release(); } } @Override public String buildUrl(UrlPageModel model) throws IllegalArgumentException { if (!model.chanName.equals(CHAN_NAME)) throw new IllegalArgumentException("wrong chan"); if (model.boardName != null && !model.boardName.matches("\\w*")) throw new IllegalArgumentException("wrong board name"); StringBuilder url = new StringBuilder(); url.append(useHttps() ? "https://" : "http://").append(CHAN_DOMAIN).append('/'); switch (model.type) { case UrlPageModel.TYPE_INDEXPAGE: return url.toString(); case UrlPageModel.TYPE_BOARDPAGE: return url.append(model.boardName).append('/') .append(model.boardPage != UrlPageModel.DEFAULT_FIRST_PAGE && model.boardPage > 1 ? (String.valueOf(model.boardPage - 1) + ".html") : "") .toString(); case UrlPageModel.TYPE_THREADPAGE: return url.append(model.boardName).append("/thread-").append(model.threadNumber).append(".html").append( model.postNumber == null || model.postNumber.length() == 0 ? "" : ("#" + model.postNumber)) .toString(); case UrlPageModel.TYPE_CATALOGPAGE: return url.append("catalog/").append(model.boardName).toString(); case UrlPageModel.TYPE_OTHERPAGE: return url.append(model.otherPath.startsWith("/") ? model.otherPath.substring(1) : model.otherPath) .toString(); } throw new IllegalArgumentException("wrong page type"); } @Override public UrlPageModel parseUrl(String url) throws IllegalArgumentException { String domain; String path = ""; Matcher parseUrl = Pattern.compile("https?://(?:www\\.)?(.+)", Pattern.CASE_INSENSITIVE).matcher(url); if (!parseUrl.find()) throw new IllegalArgumentException("incorrect url"); Matcher parsePath = Pattern.compile("(.+?)(?:/(.*))").matcher(parseUrl.group(1)); if (parsePath.find()) { domain = parsePath.group(1).toLowerCase(Locale.US); path = parsePath.group(2); } else { domain = parseUrl.group(1).toLowerCase(Locale.US); } if (!domain.equals(CHAN_DOMAIN)) throw new IllegalArgumentException("wrong chan"); UrlPageModel model = new UrlPageModel(); model.chanName = CHAN_NAME; if (path.length() == 0) { model.type = UrlPageModel.TYPE_INDEXPAGE; return model; } Matcher threadPage = Pattern.compile("([^/]+)/thread-(\\d+)\\.html[^#]*(?:#(\\d+))?").matcher(path); if (threadPage.find()) { model.type = UrlPageModel.TYPE_THREADPAGE; model.boardName = threadPage.group(1); model.threadNumber = threadPage.group(2); model.postNumber = threadPage.group(3); return model; } Matcher catalogPage = Pattern.compile("catalog/(\\w+)").matcher(path); if (catalogPage.find()) { model.boardName = catalogPage.group(1); model.type = UrlPageModel.TYPE_CATALOGPAGE; model.catalogType = 0; return model; } Matcher boardPage = Pattern.compile("([^/]+)(?:/(\\d+)\\.html?)?").matcher(path); if (boardPage.find()) { model.type = UrlPageModel.TYPE_BOARDPAGE; model.boardName = boardPage.group(1); String page = boardPage.group(2); model.boardPage = page == null ? 1 : (Integer.parseInt(page) + 1); return model; } throw new IllegalArgumentException("fail to parse"); } }