public class RobotsTxtEntry
extends java.lang.Object
Modifier and Type | Field and Description |
---|---|
private static java.lang.String |
AGENT_NAME |
private java.lang.String |
agentName |
private static java.lang.String |
ALLOW_PATH_LIST |
private java.util.List<java.lang.String> |
allowPathList |
private static java.lang.String |
CRAWL_DELAY |
private static java.lang.String |
CRAWL_DELAY_MILLIS |
private java.util.List<java.lang.String> |
denyPathList |
private static java.lang.String |
DISALLOW_PATH_LIST |
private static java.lang.String |
ETAG |
private static java.lang.String |
HOST_NAME |
private java.lang.String |
hostName |
private java.lang.String |
info |
private static java.lang.String |
LOADED_DATE |
private java.util.Map<java.lang.String,byte[]> |
mem |
private static java.lang.String |
MOD_DATE |
private static java.lang.String |
SITEMAP_LIST |
private java.util.List<java.lang.String> |
sitemapList |
Modifier | Constructor and Description |
---|---|
protected |
RobotsTxtEntry(MultiProtocolURL theURL,
java.util.List<java.lang.String> allowPathList,
java.util.List<java.lang.String> disallowPathList,
java.util.Date loadedDate,
java.util.Date modDate,
java.lang.String eTag,
java.util.List<java.lang.String> sitemapList,
long crawlDelayMillis,
java.lang.String agentName) |
protected |
RobotsTxtEntry(java.lang.String hostName,
java.util.Map<java.lang.String,byte[]> mem) |
Modifier and Type | Method and Description |
---|---|
private void |
fillMultiValue(java.util.List<java.lang.String> list,
java.lang.String listName) |
java.lang.String |
getAgentName() |
int |
getCrawlDelayMillis() |
protected java.lang.String |
getETag() |
protected java.lang.String |
getHostName() |
java.lang.String |
getInfo() |
protected java.util.Date |
getLoadedDate() |
protected java.util.Map<java.lang.String,byte[]> |
getMem() |
protected java.util.Date |
getModDate() |
java.util.List<java.lang.String> |
getSitemaps()
get the sitemap urls
|
boolean |
isDisallowed(MultiProtocolURL subpathURL) |
private void |
readMultiValue(java.util.List<java.lang.String> externallist,
java.util.List<java.lang.String> internallist,
java.lang.String listName) |
protected void |
setLoadedDate(java.util.Date newLoadedDate) |
java.lang.String |
toString() |
private static final java.lang.String HOST_NAME
private static final java.lang.String ALLOW_PATH_LIST
private static final java.lang.String DISALLOW_PATH_LIST
private static final java.lang.String SITEMAP_LIST
private static final java.lang.String LOADED_DATE
private static final java.lang.String MOD_DATE
private static final java.lang.String ETAG
private static final java.lang.String CRAWL_DELAY
private static final java.lang.String CRAWL_DELAY_MILLIS
private static final java.lang.String AGENT_NAME
private final java.util.Map<java.lang.String,byte[]> mem
private final java.util.List<java.lang.String> allowPathList
private final java.util.List<java.lang.String> denyPathList
private final java.util.List<java.lang.String> sitemapList
private final java.lang.String hostName
private final java.lang.String agentName
private java.lang.String info
protected RobotsTxtEntry(java.lang.String hostName, java.util.Map<java.lang.String,byte[]> mem)
protected RobotsTxtEntry(MultiProtocolURL theURL, java.util.List<java.lang.String> allowPathList, java.util.List<java.lang.String> disallowPathList, java.util.Date loadedDate, java.util.Date modDate, java.lang.String eTag, java.util.List<java.lang.String> sitemapList, long crawlDelayMillis, java.lang.String agentName)
private void fillMultiValue(java.util.List<java.lang.String> list, java.lang.String listName)
private void readMultiValue(java.util.List<java.lang.String> externallist, java.util.List<java.lang.String> internallist, java.lang.String listName)
protected java.lang.String getHostName()
public java.lang.String getAgentName()
protected java.util.Map<java.lang.String,byte[]> getMem()
public java.lang.String toString()
toString
in class java.lang.Object
public java.util.List<java.lang.String> getSitemaps()
protected java.util.Date getLoadedDate()
protected void setLoadedDate(java.util.Date newLoadedDate)
protected java.util.Date getModDate()
protected java.lang.String getETag()
public int getCrawlDelayMillis()
public boolean isDisallowed(MultiProtocolURL subpathURL)
public java.lang.String getInfo()