Java tutorial
/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.nutch.crawl; import java.io.IOException; import java.lang.invoke.MethodHandles; import static org.apache.nutch.crawl.CrawlDatum.*; import org.apache.hadoop.conf.Configuration; import org.apache.nutch.util.TimingUtil; import org.apache.hadoop.mapreduce.Reducer.Context; import static org.junit.Assert.*; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; public class TODOTestCrawlDbStates extends TestCrawlDbStates { private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); /** * NUTCH-578: a fetch_retry should result in a db_gone if db.fetch.retry.max * is reached. Retry counter has to be reset appropriately. */ @Test public void testCrawlDbReducerPageRetrySchedule() { LOG.info("NUTCH-578: test long running continuous crawl with fetch_retry"); ContinuousCrawlTestUtil crawlUtil = new ContinuousCrawlTestFetchRetry(); // keep going for long, to "provoke" a retry counter overflow try { if (!crawlUtil.run(150)) { fail("fetch_retry did not result in a db_gone if retry counter > maxRetries (NUTCH-578)"); } } catch (IOException e) { e.printStackTrace(); } } private class ContinuousCrawlTestFetchRetry extends ContinuousCrawlTestUtil { private int retryMax = 3; private int totalRetries = 0; ContinuousCrawlTestFetchRetry() { super(); fetchStatus = STATUS_FETCH_RETRY; retryMax = context.getConfiguration().getInt("db.fetch.retry.max", retryMax); } @Override protected CrawlDatum fetch(CrawlDatum datum, long currentTime) { datum.setStatus(fetchStatus); datum.setFetchTime(currentTime); totalRetries++; return datum; } @Override protected boolean check(CrawlDatum result) { if (result.getRetriesSinceFetch() > retryMax) { LOG.warn("Retry counter > db.fetch.retry.max: " + result); } else if (result.getRetriesSinceFetch() == Byte.MAX_VALUE) { LOG.warn("Retry counter max. value reached (overflow imminent): " + result); } else if (result.getRetriesSinceFetch() < 0) { LOG.error("Retry counter overflow: " + result); return false; } // use retry counter bound to this class (totalRetries) // instead of result.getRetriesSinceFetch() because the retry counter // in CrawlDatum could be reset (eg. NUTCH-578_v5.patch) if (totalRetries < retryMax) { if (result.getStatus() == STATUS_DB_UNFETCHED) { LOG.info("ok: " + result); result.getRetriesSinceFetch(); return true; } } else { if (result.getStatus() == STATUS_DB_GONE) { LOG.info("ok: " + result); return true; } } LOG.warn("wrong: " + result); return false; } } /** * NUTCH-1564 AdaptiveFetchSchedule: sync_delta forces immediate re-fetch for * documents not modified * <p> * Problem: documents not modified for a longer time are fetched in every * cycle because of an error in the SYNC_DELTA calculation of * {@link AdaptiveFetchSchedule}. <br> * The next fetch time should always be in the future, never in the past. * </p> */ @Test public void testAdaptiveFetchScheduleSyncDelta() { LOG.info("NUTCH-1564 test SYNC_DELTA calculation of AdaptiveFetchSchedule"); Context context = CrawlDBTestUtil.createContext(); Configuration conf = context.getConfiguration(); conf.setLong("db.fetch.interval.default", 172800); // 2 days conf.setLong("db.fetch.schedule.adaptive.min_interval", 86400); // 1 day conf.setLong("db.fetch.schedule.adaptive.max_interval", 604800); // 7 days conf.setLong("db.fetch.interval.max", 604800); // 7 days conf.set("db.fetch.schedule.class", "org.apache.nutch.crawl.AdaptiveFetchSchedule"); ContinuousCrawlTestUtil crawlUtil = new CrawlTestFetchScheduleNotModifiedFetchTime(context); crawlUtil.setInterval(FetchSchedule.SECONDS_PER_DAY / 3); try { if (!crawlUtil.run(100)) { fail("failed: sync_delta calculation with AdaptiveFetchSchedule"); } } catch (IOException e) { e.printStackTrace(); } } private class CrawlTestFetchScheduleNotModifiedFetchTime extends CrawlTestFetchNotModified { // time of current fetch private long fetchTime; private long minInterval; private long maxInterval; CrawlTestFetchScheduleNotModifiedFetchTime(Context context) { super(context); Configuration conf = context.getConfiguration(); minInterval = conf.getLong("db.fetch.schedule.adaptive.min_interval", 86400); // 1 day maxInterval = conf.getLong("db.fetch.schedule.adaptive.max_interval", 604800); // 7 days if (conf.getLong("db.fetch.interval.max", 604800) < maxInterval) { maxInterval = conf.getLong("db.fetch.interval.max", 604800); } } @Override protected CrawlDatum fetch(CrawlDatum datum, long currentTime) { // remember time of fetching fetchTime = currentTime; return super.fetch(datum, currentTime); } @Override protected boolean check(CrawlDatum result) { if (result.getStatus() == STATUS_DB_NOTMODIFIED) { // check only status notmodified here long secondsUntilNextFetch = (result.getFetchTime() - fetchTime) / 1000L; if (secondsUntilNextFetch < -1) { // next fetch time is in the past (more than one second) LOG.error("Next fetch time is in the past: " + result); return false; } if (secondsUntilNextFetch < 60) { // next fetch time is in less than one minute // (critical: Nutch can hardly be so fast) LOG.error("Less then one minute until next fetch: " + result); } // Next fetch time should be within min. and max. (tolerance: 60 sec.) if (secondsUntilNextFetch + 60 < minInterval || secondsUntilNextFetch - 60 > maxInterval) { LOG.error("Interval until next fetch time (" + TimingUtil.elapsedTime(fetchTime, result.getFetchTime()) + ") is not within min. and max. interval: " + result); // TODO: is this a failure? } } return true; } } }