Java Reference
In-Depth Information
}
}
return false;
}
/**
* Called when a new host is to be processed. Hosts
* are processed one at a time. SpiderFilter classes
* can not be shared among hosts.
*
* @param host
* The new host.
* @param userAgent
* The user agent being used by the spider. Leave
* null for default.
* @throws IOException
* Thrown if an I/O error occurs.
*/
public void newHost(String host, String userAgent)
throws IOException
{
String str;
this.active = false;
this.userAgent = userAgent;
this.robotURL = new URL("http", host, 80, "/robots.txt");
URLConnection http = this.robotURL.openConnection();
if (userAgent != null)
{
http.setRequestProperty("User-Agent", userAgent);
}
InputStream is = http.getInputStream();
InputStreamReader isr = new InputStreamReader(is);
BufferedReader r = new BufferedReader(isr);
exclude.clear();
try
{
while ((str = r.readLine()) != null)
{
loadLine(str);
