INSIDE THE HEATON RESEARCH SPIDER - HTTP Programming Recipes for Java Bots - page 471

* @throws IOException
* Thrown if an I/O error occurs.
*/
private void addURL(String u, SpiderReportable.URLType type)
throws IOException {
if (u == null) {
return;
}
try {
URL url = URLUtility.constructURL(this.base, u, true);
url = this.spider.getWorkloadManager().convertURL(
url.toString());
if (url.getProtocol().equalsIgnoreCase("http")
|| url.getProtocol().equalsIgnoreCase("https")) {
if (this.spider.getReport().spiderFoundURL(url,
this.base, type)) {
try {
this.spider.addURL(url, this.base, this.depth + 1);
} catch (WorkloadException e) {
throw new IOException(e.getMessage());
}
}
}
}
catch (MalformedURLException e) {
logger.log(Level.INFO, "Malformed URL found:" + u);
} catch (WorkloadException e) {
logger.log(Level.INFO, "Invalid URL found:" + u);
}
}
/**
* This method is called when an anchor(A) tag is found.
*
* @param href
* The link found.
* @throws IOException
* I/O error.
*/
private void handleA(String href) throws IOException {
if (href != null) {
href = href.trim();