Example usage for org.jsoup.nodes Document getElementsByClass

List of usage examples for org.jsoup.nodes Document getElementsByClass

Introduction

In this page you can find the example usage for org.jsoup.nodes Document getElementsByClass.

Prototype

public Elements getElementsByClass(String className) 

Source Link

Document

Find elements that have this class, including or under this element.

Usage

From source file:accountgen.controller.Controller.java

private void setAddress(Document doc, Person p) {
    Elements e = doc.getElementsByClass("address");
    Element ad = e.select(".adr").first();
    Address address = new Address();
    String streetnumber = StringEscapeUtils
            .unescapeHtml4(/*w w  w  .  j a v a 2s  . c o  m*/
                    ad.html().split("<br />")[0].split(" ")[ad.html().split("<br />")[0].split(" ").length - 1])
            .trim();
    String state = StringEscapeUtils
            .unescapeHtml4(
                    ad.html().split("<br />")[1].split(" ")[ad.html().split("<br />")[1].split(" ").length - 1])
            .trim();
    address.setStreetnumber(streetnumber);
    address.setStreetname(StringEscapeUtils.unescapeHtml4(ad.html().split(streetnumber)[0]).trim());
    address.setState(state);
    address.setPostcode(
            StringEscapeUtils.unescapeHtml4(ad.html().split("<br />")[1].split(state)[0]).trim().split(" ")[0]);
    address.setCountry(Consts.COUNTRY);
    p.setAdress(address);
}

From source file:org.manalith.ircbot.plugin.uriinfo.UriInfoPlugin.java

private String getSiteSpecificTitle(String uri, Document document) {
    if (uri.startsWith("http://mlbpark.donga.com/bbs/view.php?")
            || uri.startsWith("http://mlbpark.donga.com/mbs/articleV.php?")) {
        // MLB Park article
        Element element = document.getElementsByClass("D14").first();
        if (element != null) {
            try {
                return element.child(0).text();
            } catch (IndexOutOfBoundsException e) {
                return null;
            }//from   w w w . ja  v  a 2 s  .  com
        }
    } else if (uri.startsWith("http://www.slrclub.com/bbs/vx2.php?")) {
        Element element = document.getElementsByClass("sbj").first();
        if (element != null) {
            return element.text();
        }
    }

    return null;
}

From source file:jobhunter.infoempleo.Client.java

public Job execute() throws IOException, URISyntaxException {
    l.debug("Connecting to {}", url);

    update("Connecting", 1L);
    final Document doc = Jsoup.connect(url).get();

    update("Parsing HTML", 2L);
    final Job job = Job.of();

    job.setPortal(InfoEmpleoWebPlugin.portal);
    job.setLink(url);/*from   w ww .  j  a v  a 2 s . co  m*/
    job.setExtId("");

    final StringBuilder description = new StringBuilder();

    doc.getElementsByClass("linea").forEach(td -> {
        td.getElementsByTag("p").forEach(p -> {
            description.append(StringEscapeUtils.unescapeHtml4(p.html())).append(System.lineSeparator());
        });
    });

    job.setDescription(description.toString());

    job.setPosition(doc.getElementById("ctl00_CPH_Body_Link_Subtitulo").attr("title"));

    job.getCompany().setName(getCompany(doc));

    final String href = doc.getElementById("ctl00_CPH_Body_lnkEnviarAmigoI").attr("href");

    final String extId = URLEncodedUtils.parse(new URI(href), "UTF-8").stream()
            .filter(nvp -> nvp.getName().equalsIgnoreCase("Id_Oferta")).findFirst().get().getValue();

    job.setExtId(extId);

    update("Done", 3L);
    return job;
}

From source file:com.blackducksoftware.tools.nrt.generator.NRTReportGenerator.java

/**
 * Copies the HTML template into the finalHtmlOutput then injects the
 * generates JSON data into the specific div location and writes it out.
 * //from w  ww .j  a va  2 s  .c om
 * @param expectedFile
 */
public void generateHTMLFromTemplate(File finalHtmlOutput) {

    log.info("Writing to report: " + finalHtmlOutput);
    String jsonComponentList = generateJSONFromObject(componentMap);
    String jsonPropertyList = generateJSONFromObject(nrtConfig.getOptionsForExport());
    // Construct a variable out of it
    jsonComponentList = "var compList=[" + jsonComponentList + "]";
    jsonPropertyList = "var propList=[" + jsonPropertyList + "]";

    PrintWriter writer = null;
    try {
        // Read the template
        Document doc = Jsoup.parse(finalHtmlOutput, "UTF-8");

        // Inject the JSON
        Elements jsonElementDivBlock = doc.getElementsByClass(NRTConstants.HTML_JSON_DATA_BLOCK);

        // This will be empty, but it should exist
        Element jsonDivElement = jsonElementDivBlock.get(0);

        if (jsonDivElement != null) {
            // Remove any script tags from it, in case the user populated
            // the template incorrectly with data
            if (jsonDivElement.children().size() > 0) {
                Elements children = jsonDivElement.children();
                for (int i = 0; i < children.size(); i++) {
                    Element el = children.get(i);
                    el.remove();
                }
            }

            addNewScriptElementWithJson(jsonDivElement, jsonComponentList);
            addNewScriptElementWithJson(jsonDivElement, jsonPropertyList);
        } else {
            log.error("Unable to find a valid critical DIV inside HTML template: "
                    + NRTConstants.HTML_JSON_DATA_BLOCK);
        }
        writer = new PrintWriter(finalHtmlOutput, "UTF-8");
        // Write out the file
        writer.write(doc.html());
        writer.flush();
        writer.close();

    } catch (Exception e) {
        log.error("Unable to write out final report file!", e);
    } finally {
        writer.close();
    }

}

From source file:model.ParseInfoFromSite.java

/**
 * Parse info about a bank from NBU site and return it
 * // w  w w.  j  av a 2 s  . c  om
 * @param URL
 *            on page with information about one bank
 * @return Map <String,String> with information about bank
 */
public Map<String, String> getBankInfoMap(String URL) {

    logger.info("run");
    Map<String, String> map = new HashMap<String, String>();
    Document doc;

    try {

        doc = Jsoup.connect(URL).userAgent("Mozilla").timeout(10 * 1000).get();
        List<Comment> comments = findAllComments(doc);
        for (Comment comment : comments) {
            String data = comment.getData();
            comment.after(data);
            comment.remove();
        }

        Elements cells = doc.getElementsByClass("cell");

        for (int i = 0; i < cells.size(); i = i + 2) {

            map.put(cells.get(i).text(), cells.get(i + 1).text());
        }
    } catch (IOException e) {
        e.printStackTrace();
    }
    return map;
}

From source file:am.roadpolice.roadpolice.downloaders.Submitter.java

/**
 * This function process URL and collect needed information about
 * violation and add it to Violation List (mViolationInfoList).
 *
 * @param url URL from which data will be processed.
 * @return null if no error occurs; otherwise ERROR1/ERROR2 if something
 * was changed in server behaviour,  ERROR3 if error occurs while trying
 * to get JSOUP document, or server error text.
 *//*from w ww .  j a  v a 2  s  .  co  m*/
private String processUrl(final String url) {
    Logger.debugLine();
    Logger.debug(TAG, "Processing URL: " + url);

    HttpGet httpGet = new HttpGet(url);
    HttpParams httpParameters = new BasicHttpParams();
    // Set the timeout in milliseconds until a connection is established.
    // The default value is zero, that means the timeout is not used.
    HttpConnectionParams.setConnectionTimeout(httpParameters, CONNECT_TIMEOUT);
    // Set the default socket timeout (SO_TIMEOUT)
    // in milliseconds which is the timeout for waiting for data.
    HttpConnectionParams.setSoTimeout(httpParameters, SOCKET_TIMEOUT);
    DefaultHttpClient httpClient = new DefaultHttpClient(httpParameters);

    try {

        HttpResponse response = httpClient.execute(httpGet);
        java.util.Scanner s = new java.util.Scanner(response.getEntity().getContent()).useDelimiter("\\A");

        Document document = Jsoup.parse(s.hasNext() ? s.next() : null);
        // In the case if  some data  was provided not correct in
        // the url server generates page with red text, we handle
        // this situation and return text in the red block.
        Elements errorElement = document.getElementsByClass("red");
        if (errorElement != null && errorElement.first() != null) {
            final String errorText = errorElement.first().text();
            Logger.debug(TAG, "Found Error Element (RED): " + errorText);
            return errorText;
        }

        Elements tableElements = document.getElementsByClass("dahk_yes");
        tableElements.addAll(document.getElementsByClass("dahk_no"));
        for (Element element : tableElements) {
            Elements tdElements = element.getElementsByTag("td");

            int tdElementsCount = ViolationInfo.COL_OWNER_FULL_NAME;
            ViolationInfo violationInfo = null;
            for (Element tdElement : tdElements) {
                final String text = tdElement.text().trim();

                // We found vehicle registration number.
                if (text.equalsIgnoreCase(mRegNum)) {
                    // Create new class object to store data.
                    violationInfo = new ViolationInfo(mRegNum);
                    violationInfo.setCertificateNumber(mCerNum);
                    continue;
                }

                // Violation Info object was not created, reason can be
                // that something is changed on the server side.
                if (violationInfo == null) {
                    return ERROR_ON_SERVER_SIDE;
                }

                switch (tdElementsCount) {
                case ViolationInfo.COL_OWNER_FULL_NAME:
                    violationInfo.setOwnerFullName(text);
                    break;

                case ViolationInfo.COL_OWNER_ADDRESS:
                    violationInfo.setOwnerAddress(text);
                    break;

                case ViolationInfo.COL_TO_PAY:
                    violationInfo.setToPay(text);
                    break;

                case ViolationInfo.COL_PAYED:
                    violationInfo.setPayed(text);
                    break;

                case ViolationInfo.COL_CAR_MODEL:
                    violationInfo.setCarModel(text);
                    break;

                case ViolationInfo.COL_THE_DECISION:
                    // Do Nothing ...
                    break;

                case ViolationInfo.COL_DATE:
                    violationInfo.setDate(text);
                    break;

                case ViolationInfo.COL_PIN:
                    violationInfo.setPin(text);
                    break;

                default:
                    return ERROR_WHILE_PARSING_DATA;
                }

                tdElementsCount++;
            }

            // Add items to the list.
            mViolationInfoList.add(violationInfo);
        }
    } catch (IOException e) {
        Logger.error(TAG, "----> Exception occurs while trying to get JSOUP document.");
        Logger.error(TAG, "----> Message: " + e.getMessage());
        return ERROR_WHILE_CREATING_JSOUP;
    }

    return null;
}

From source file:jp.mau.twappremover.MainActivity.java

private boolean login() {
    HttpPost request = new HttpPost(LOGIN_PAGE);
    request.addHeader("User-Agent", USER_AGENT);
    request.addHeader("Referer", TOP_PAGE);
    request.addHeader("Content-Type", "application/x-www-form-urlencoded");
    request.addHeader("Cookie", "_twitter_sess=" + _session_id + "; guest_id=" + _guest_id);

    List<NameValuePair> pairs = new ArrayList<NameValuePair>();
    pairs.add(new BasicNameValuePair("session[username_or_email]", _id.getText().toString()));
    pairs.add(new BasicNameValuePair("session[password]", _pass.getText().toString()));
    pairs.add(new BasicNameValuePair("remember_me", "1"));
    pairs.add(new BasicNameValuePair("return_to_ssl", "true"));
    pairs.add(new BasicNameValuePair("redirect_after_login", " "));
    pairs.add(new BasicNameValuePair(KEY_AUTH_TOKEN_TAG, _auth_token));

    try {/*from   www.  jav a 2s  .c o m*/
        request.setEntity(new UrlEncodedFormEntity(pairs, "UTF-8"));
        String result = _client.execute(request, new ResponseHandler<String>() {
            @Override
            public String handleResponse(HttpResponse response) throws ClientProtocolException, IOException {
                switch (response.getStatusLine().getStatusCode()) {
                case HttpStatus.SC_OK:
                    return EntityUtils.toString(response.getEntity(), "UTF-8");
                case HttpStatus.SC_NOT_FOUND:
                    throw new RuntimeException("not found");
                default:
                    throw new RuntimeException("error");
                }
            }
        });
        // ??(signin-wrapper???????)
        Document doc = Jsoup.parse(result);
        Elements elm = doc.getElementsByClass("signin-wrapper");
        if (elm.size() > 0)
            return false;

        // auth_token????
        List<Cookie> cookies = _client.getCookieStore().getCookies();
        for (Cookie c : cookies) {
            if (c.getName().equals(KEY_AUTH_TOKEN)) {
                _cookie_auth = c.getValue();
            }
        }

    } catch (Exception ex) {
        ex.printStackTrace();
    }
    return true;
}

From source file:com.qkj.qkjmanage.action.OilManageAction.java

public void getOilPrice() throws Exception {
    List<String> prices = new ArrayList<>();
    //?/*from  ww  w.j a  v  a  2 s.c om*/
    try {
        Document doc = null;
        doc = Jsoup.connect("http://ny.gold600.com/qinghai.html").get();
        Elements element1 = doc.getElementsByClass("JO_330q63");
        Elements element2 = doc.getElementsByClass("JO_331q63");
        Elements element3 = doc.getElementsByClass("JO_332q63");

        prices.add(element1.text());
        prices.add(element2.text());
        prices.add(element3.text());

        HttpServletRequest request = ServletActionContext.getRequest();
        JSONArray jsonArray = JSONArray.fromObject(prices);
        HttpServletResponse response = ServletActionContext.getResponse();
        response.setContentType("text/html;charset=UTF-8");
        response.getWriter().print(jsonArray);
    } catch (Exception e) {
        log.error(this.getClass().getName() + "!getOilPrice ??:", e);
        throw new Exception(this.getClass().getName() + "!getOilPrice ??:", e);
    }
}

From source file:com.geecko.QuickLyric.tasks.IdDecoder.java

@Override
protected Lyrics doInBackground(String... strings) {
    String url = strings[0];//from w  ww  . ja va2s .c  o m
    String artist;
    String track;
    if (url.contains("//www.soundhound.com/")) {
        try { // todo switch to Jsoup
            String html = getUrlAsString(url);
            int preceding = html.indexOf("root.App.trackDa") + 19;
            int following = html.substring(preceding).indexOf(";");
            String data = html.substring(preceding, preceding + following);
            JSONObject jsonData = new JSONObject(data);
            artist = jsonData.getString("artist_display_name");
            track = jsonData.getString("track_name");
        } catch (IOException | JSONException e) {
            e.printStackTrace();
            return new Lyrics(ERROR);
        }

    } else if (url.contains("//shz.am/")) {
        try {
            Document doc = Jsoup.connect(url.trim()).get();
            track = doc.getElementsByAttribute("data-track-title").text();
            artist = doc.getElementsByAttribute("data-track-artist").text();
        } catch (IOException e) {
            e.printStackTrace();
            return new Lyrics(ERROR);
        }
    } else if (url.contains("//play.google.com/store/music/")) {
        String docID = url.substring(url.indexOf("&tid=") + 5);
        try {
            Document doc = Jsoup.connect(url).get();
            Element playCell = doc.getElementsByAttributeValue("data-track-docid", docID).get(0);
            artist = doc.getElementsByClass("primary").text();
            track = playCell.parent().parent().child(1).getElementsByClass("title").text();
        } catch (IOException e) {
            e.printStackTrace();
            return new Lyrics(ERROR);
        }
    } else
        return new Lyrics(ERROR);
    Lyrics res = new Lyrics(Lyrics.SEARCH_ITEM);
    res.setArtist(artist);
    res.setTitle(track);
    return res;
}

From source file:biz.shadowservices.DegreesToolbox.DataFetcher.java

public FetchResult updateData(Context context, boolean force) {
    //Open database
    DBOpenHelper dbhelper = new DBOpenHelper(context);
    SQLiteDatabase db = dbhelper.getWritableDatabase();

    // check for internet connectivity
    try {//from  ww w  .j av a2s. co m
        if (!isOnline(context)) {
            Log.d(TAG, "We do not seem to be online. Skipping Update.");
            return FetchResult.NOTONLINE;
        }
    } catch (Exception e) {
        exceptionReporter.reportException(Thread.currentThread(), e, "Exception during isOnline()");
    }
    SharedPreferences sp = PreferenceManager.getDefaultSharedPreferences(context);
    if (!force) {
        try {
            if (sp.getBoolean("loginFailed", false) == true) {
                Log.d(TAG, "Previous login failed. Skipping Update.");
                DBLog.insertMessage(context, "i", TAG, "Previous login failed. Skipping Update.");
                return FetchResult.LOGINFAILED;
            }
            if (sp.getBoolean("autoupdates", true) == false) {
                Log.d(TAG, "Automatic updates not enabled. Skipping Update.");
                DBLog.insertMessage(context, "i", TAG, "Automatic updates not enabled. Skipping Update.");
                return FetchResult.NOTALLOWED;
            }
            if (!isBackgroundDataEnabled(context) && sp.getBoolean("obeyBackgroundData", true)) {
                Log.d(TAG, "Background data not enabled. Skipping Update.");
                DBLog.insertMessage(context, "i", TAG, "Background data not enabled. Skipping Update.");
                return FetchResult.NOTALLOWED;
            }
            if (!isAutoSyncEnabled() && sp.getBoolean("obeyAutoSync", true)
                    && sp.getBoolean("obeyBackgroundData", true)) {
                Log.d(TAG, "Auto sync not enabled. Skipping Update.");
                DBLog.insertMessage(context, "i", TAG, "Auto sync not enabled. Skipping Update.");
                return FetchResult.NOTALLOWED;
            }
            if (isWifi(context) && !sp.getBoolean("wifiUpdates", true)) {
                Log.d(TAG, "On wifi, and wifi auto updates not allowed. Skipping Update");
                DBLog.insertMessage(context, "i", TAG,
                        "On wifi, and wifi auto updates not allowed. Skipping Update");
                return FetchResult.NOTALLOWED;
            } else if (!isWifi(context)) {
                Log.d(TAG, "We are not on wifi.");
                if (!isRoaming(context) && !sp.getBoolean("2DData", true)) {
                    Log.d(TAG, "Automatic updates on 2Degrees data not enabled. Skipping Update.");
                    DBLog.insertMessage(context, "i", TAG,
                            "Automatic updates on 2Degrees data not enabled. Skipping Update.");
                    return FetchResult.NOTALLOWED;
                } else if (isRoaming(context) && !sp.getBoolean("roamingData", false)) {
                    Log.d(TAG, "Automatic updates on roaming mobile data not enabled. Skipping Update.");
                    DBLog.insertMessage(context, "i", TAG,
                            "Automatic updates on roaming mobile data not enabled. Skipping Update.");
                    return FetchResult.NOTALLOWED;
                }

            }
        } catch (Exception e) {
            exceptionReporter.reportException(Thread.currentThread(), e,
                    "Exception while finding if to update.");
        }

    } else {
        Log.d(TAG, "Update Forced");
    }

    try {
        String username = sp.getString("username", null);
        String password = sp.getString("password", null);
        if (username == null || password == null) {
            DBLog.insertMessage(context, "i", TAG, "Username or password not set.");
            return FetchResult.USERNAMEPASSWORDNOTSET;
        }

        // Find the URL of the page to send login data to.
        Log.d(TAG, "Finding Action. ");
        HttpGetter loginPageGet = new HttpGetter("https://secure.2degreesmobile.co.nz/web/ip/login");
        String loginPageString = loginPageGet.execute();
        if (loginPageString != null) {
            Document loginPage = Jsoup.parse(loginPageString,
                    "https://secure.2degreesmobile.co.nz/web/ip/login");
            Element loginForm = loginPage.getElementsByAttributeValue("name", "loginFrm").first();
            String loginAction = loginForm.attr("action");
            // Send login form
            List<NameValuePair> loginValues = new ArrayList<NameValuePair>();
            loginValues.add(new BasicNameValuePair("externalURLRedirect", ""));
            loginValues.add(new BasicNameValuePair("hdnAction", "login_userlogin"));
            loginValues.add(new BasicNameValuePair("hdnAuthenticationType", "M"));
            loginValues.add(new BasicNameValuePair("hdnlocale", ""));

            loginValues.add(new BasicNameValuePair("userid", username));
            loginValues.add(new BasicNameValuePair("password", password));
            Log.d(TAG, "Sending Login ");
            HttpPoster sendLoginPoster = new HttpPoster(loginAction, loginValues);
            // Parse result

            String loginResponse = sendLoginPoster.execute();
            Document loginResponseParsed = Jsoup.parse(loginResponse);
            // Determine if this is a pre-pay or post-paid account.
            boolean postPaid;
            if (loginResponseParsed
                    .getElementById("p_CustomerPortalPostPaidHomePage_WAR_customerportalhomepage") == null) {
                Log.d(TAG, "Pre-pay account or no account.");
                postPaid = false;
            } else {
                Log.d(TAG, "Post-paid account.");
                postPaid = true;
            }

            String homepageUrl = "https://secure.2degreesmobile.co.nz/group/ip/home";
            if (postPaid) {
                homepageUrl = "https://secure.2degreesmobile.co.nz/group/ip/postpaid";
            }
            HttpGetter homepageGetter = new HttpGetter(homepageUrl);
            String homepageHTML = homepageGetter.execute();
            Document homePage = Jsoup.parse(homepageHTML);

            Element accountSummary = homePage.getElementById("accountSummary");
            if (accountSummary == null) {
                Log.d(TAG, "Login failed.");
                return FetchResult.LOGINFAILED;
            }
            db.delete("cache", "", null);
            /* This code fetched some extra details for postpaid users, but on reflection they aren't that useful.
             * Might reconsider this.
             *
             if (postPaid) {
                     
               Element accountBalanceSummaryTable = accountSummary.getElementsByClass("tableBillSummary").first();
               Elements rows = accountBalanceSummaryTable.getElementsByTag("tr");
               int rowno = 0;
               for (Element row : rows) {
                  if (rowno > 1) {
             break;
                  }
                  //Log.d(TAG, "Starting row");
                  //Log.d(TAG, row.html());
                  Double value;
                  try {
             Element amount = row.getElementsByClass("tableBillamount").first();
             String amountHTML = amount.html();
             Log.d(TAG, amountHTML.substring(1));
             value = Double.parseDouble(amountHTML.substring(1));
                  } catch (Exception e) {
             Log.d(TAG, "Failed to parse amount from row.");
             value = null;
                  }
                  String expiresDetails = "";
                  String expiresDate = null;
                  String name = null;
                  try {
             Element details = row.getElementsByClass("tableBilldetail").first();
             name = details.ownText();
             Element expires = details.getElementsByTag("em").first();
             if (expires != null) {
                 expiresDetails = expires.text();
             } 
             Log.d(TAG, expiresDetails);
             Pattern pattern;
             pattern = Pattern.compile("\\(payment is due (.*)\\)");
             Matcher matcher = pattern.matcher(expiresDetails);
             if (matcher.find()) {
                /*Log.d(TAG, "matched expires");
                Log.d(TAG, "group 0:" + matcher.group(0));
                Log.d(TAG, "group 1:" + matcher.group(1));
                Log.d(TAG, "group 2:" + matcher.group(2)); *
                String expiresDateString = matcher.group(1);
                Date expiresDateObj;
                if (expiresDateString != null) {
                   if (expiresDateString.length() > 0) {
                      try {
                         expiresDateObj = DateFormatters.EXPIRESDATE.parse(expiresDateString);
                         expiresDate = DateFormatters.ISO8601DATEONLYFORMAT.format(expiresDateObj);
                      } catch (java.text.ParseException e) {
                         Log.d(TAG, "Could not parse date: " + expiresDateString);
                      }
                   }   
                }
             }
                  } catch (Exception e) {
             Log.d(TAG, "Failed to parse details from row.");
                  }
                  String expirev = null;
                  ContentValues values = new ContentValues();
                  values.put("name", name);
                  values.put("value", value);
                  values.put("units", "$NZ");
                  values.put("expires_value", expirev );
                  values.put("expires_date", expiresDate);
                  db.insert("cache", "value", values );
                  rowno++;
               }
            } */
            Element accountSummaryTable = accountSummary.getElementsByClass("tableAccountSummary").first();
            Elements rows = accountSummaryTable.getElementsByTag("tr");
            for (Element row : rows) {
                // We are now looking at each of the rows in the data table.
                //Log.d(TAG, "Starting row");
                //Log.d(TAG, row.html());
                Double value;
                String units;
                try {
                    Element amount = row.getElementsByClass("tableBillamount").first();
                    String amountHTML = amount.html();
                    //Log.d(TAG, amountHTML);
                    String[] amountParts = amountHTML.split("&nbsp;", 2);
                    //Log.d(TAG, amountParts[0]);
                    //Log.d(TAG, amountParts[1]);
                    if (amountParts[0].contains("Included") || amountParts[0].equals("All You Need")
                            || amountParts[0].equals("Unlimited Text*")) {
                        value = Values.INCLUDED;
                    } else {
                        try {
                            value = Double.parseDouble(amountParts[0]);
                        } catch (NumberFormatException e) {
                            exceptionReporter.reportException(Thread.currentThread(), e, "Decoding value.");
                            value = 0.0;
                        }
                    }
                    units = amountParts[1];
                } catch (NullPointerException e) {
                    //Log.d(TAG, "Failed to parse amount from row.");
                    value = null;
                    units = null;
                }
                Element details = row.getElementsByClass("tableBilldetail").first();
                String name = details.getElementsByTag("strong").first().text();
                Element expires = details.getElementsByTag("em").first();
                String expiresDetails = "";
                if (expires != null) {
                    expiresDetails = expires.text();
                }
                Log.d(TAG, expiresDetails);
                Pattern pattern;
                if (postPaid == false) {
                    pattern = Pattern.compile("\\(([\\d\\.]*) ?\\w*? ?expiring on (.*)\\)");
                } else {
                    pattern = Pattern.compile("\\(([\\d\\.]*) ?\\w*? ?will expire on (.*)\\)");
                }
                Matcher matcher = pattern.matcher(expiresDetails);
                Double expiresValue = null;
                String expiresDate = null;
                if (matcher.find()) {
                    /*Log.d(TAG, "matched expires");
                    Log.d(TAG, "group 0:" + matcher.group(0));
                    Log.d(TAG, "group 1:" + matcher.group(1));
                    Log.d(TAG, "group 2:" + matcher.group(2)); */
                    try {
                        expiresValue = Double.parseDouble(matcher.group(1));
                    } catch (NumberFormatException e) {
                        expiresValue = null;
                    }
                    String expiresDateString = matcher.group(2);
                    Date expiresDateObj;
                    if (expiresDateString != null) {
                        if (expiresDateString.length() > 0) {
                            try {
                                expiresDateObj = DateFormatters.EXPIRESDATE.parse(expiresDateString);
                                expiresDate = DateFormatters.ISO8601DATEONLYFORMAT.format(expiresDateObj);
                            } catch (java.text.ParseException e) {
                                Log.d(TAG, "Could not parse date: " + expiresDateString);
                            }
                        }
                    }
                }
                ContentValues values = new ContentValues();
                values.put("name", name);
                values.put("value", value);
                values.put("units", units);
                values.put("expires_value", expiresValue);
                values.put("expires_date", expiresDate);
                db.insert("cache", "value", values);
            }

            if (postPaid == false) {
                Log.d(TAG, "Getting Value packs...");
                // Find value packs
                HttpGetter valuePacksPageGet = new HttpGetter(
                        "https://secure.2degreesmobile.co.nz/group/ip/prevaluepack");
                String valuePacksPageString = valuePacksPageGet.execute();
                //DBLog.insertMessage(context, "d", "",  valuePacksPageString);
                if (valuePacksPageString != null) {
                    Document valuePacksPage = Jsoup.parse(valuePacksPageString);
                    Elements enabledPacks = valuePacksPage.getElementsByClass("yellow");
                    for (Element enabledPack : enabledPacks) {
                        Element offerNameElemt = enabledPack
                                .getElementsByAttributeValueStarting("name", "offername").first();
                        if (offerNameElemt != null) {
                            String offerName = offerNameElemt.val();
                            DBLog.insertMessage(context, "d", "", "Got element: " + offerName);
                            ValuePack[] packs = Values.valuePacks.get(offerName);
                            if (packs == null) {
                                DBLog.insertMessage(context, "d", "",
                                        "Offer name: " + offerName + " not matched.");
                            } else {
                                for (ValuePack pack : packs) {
                                    ContentValues values = new ContentValues();
                                    values.put("plan_startamount", pack.value);
                                    values.put("plan_name", offerName);
                                    DBLog.insertMessage(context, "d", "",
                                            "Pack " + pack.type.id + " start value set to " + pack.value);
                                    db.update("cache", values, "name = '" + pack.type.id + "'", null);
                                }
                            }
                        }
                    }
                }
            }

            SharedPreferences.Editor prefedit = sp.edit();
            Date now = new Date();
            prefedit.putString("updateDate", DateFormatters.ISO8601FORMAT.format(now));
            prefedit.putBoolean("loginFailed", false);
            prefedit.putBoolean("networkError", false);
            prefedit.commit();
            DBLog.insertMessage(context, "i", TAG, "Update Successful");
            return FetchResult.SUCCESS;

        }
    } catch (ClientProtocolException e) {
        DBLog.insertMessage(context, "w", TAG, "Network error: " + e.getMessage());
        return FetchResult.NETWORKERROR;
    } catch (IOException e) {
        DBLog.insertMessage(context, "w", TAG, "Network error: " + e.getMessage());
        return FetchResult.NETWORKERROR;
    } finally {
        db.close();
    }
    return null;
}