USING A SPIDER - HTTP Programming Recipes for Java Bots - page 401

/**
* Called when the spider encounters a URL.
*
* @param url
* The URL that the spider found.
* @param source
* The page that the URL was found on.
* @param type
* The type of link this URL is.
* @return True if the spider should scan for links on
* this page.
*/
public boolean spiderFoundURL(URL url, URL source,
URLType type);
/**
* Called when the spider is about to process a NON-HTML
* URL.
*
* @param url
* The URL that the spider found.
* @param stream
* An InputStream to read the page contents from.
* @throws IOException
* Thrown if an IO error occurs while processing
* the page.
*/
public void spiderProcessURL(URL url, InputStream stream)
throws IOException;
/**
* Called when the spider is ready to process an HTML
* URL.
*
* @param url
* The URL that the spider is about to process.
* @param parse
* An object that will allow you you to parse the
* HTML on this page.
* @throws IOException
* Thrown if an IO error occurs while processing
* the page.
*/
public void spiderProcessURL(URL url, SpiderParseHTML parse)
throws IOException;