INSIDE THE HEATON RESEARCH SPIDER - HTTP Programming Recipes for Java Bots - page 463

Listing 14.3: The Worker Class (SpiderWorker.java)
package com.heatonresearch.httprecipes.spider;
import java.io.*;
import java.net.*;
import java.util.logging.*;
import com.heatonresearch.httprecipes.spider.workload.*;
public class SpiderWorker implements Runnable {
/**
* The logger.
*/
private static Logger logger = Logger.getLogger(
"com.heatonresearch.httprecipes.spider.SpiderWorker");
/**
* The URL being processed.
*/
private URL url;
/**
* The Spider object that this worker belongs to.
*/
private Spider spider;
/**
* Construct a SpiderWorker object.
*
* @param spider
* The spider this worker will work with.
* @param url
* The URL to be processed.
*/
public SpiderWorker(Spider spider, URL url) {
this.spider = spider;
this.url = url;
}
/*
* This method is called by the thread pool to process one
* single URL.
*/
public void run() {
URLConnection connection = null;
InputStream is = null;