Java Reference
In-Depth Information
/*
* Is the parser active? It can become inactive when
* parsing sections of the file for other user agents.
*/
private boolean active;
/*
* The user agent string we are to use, null for default.
*/
private String userAgent;
/**
* Returns a list of URL's to be excluded.
*
* @return A vector of URL's to be excluded.
*/
public List<String> getExclude()
{
return this.exclude;
}
/**
* Returns the full URL of the robots.txt file.
*
* @return The full URL of the robots.txt file.
*/
public URL getRobotFile()
{
return this.robotURL;
}
/**
* Check to see if the specified URL is to be excluded.
*
* @param url
* The URL to be checked.
* @return Returns true if the URL should be excluded.
*/
public boolean isExcluded(URL url)
{
for (String str : this.exclude)
{
if (url.getFile().startsWith(str))
{
return true;
