INSIDE THE HEATON RESEARCH SPIDER - HTTP Programming Recipes for Java Bots - page 487

/**
* Get the source page that contains the specified URL.
*
* @param url
* The URL to seek the source for.
* @return The source of the specified URL.
* @throws WorkloadException
* Thrown if the source of the specified URL
* could not be found.
*/
public URL getSource(URL url) {
URLStatus s = this.workload.get(url);
if (s == null) {
return null;
} else {
return s.getSource();
}
}
/**
* Get a new URL to work on. Wait if there are no URL's
* currently available. Return null if done with the
* current host. The URL being returned will be marked as
* in progress.
*
* @return The next URL to work on,
* @throws WorkloadException
* Thrown if the next URL could not be obtained.
*/
public URL getWork() {
URL url;
try {
url = this.waiting.poll(5, TimeUnit.SECONDS);
if (url != null) {
setStatus(url, null, URLStatus.Status.WORKING, -1);
this.workingCount++;
}
return url;
} catch (InterruptedException e) {
return null;
}
}