public final class CrawlSwitchboard
extends java.lang.Object
Constructor and Description |
---|
CrawlSwitchboard(java.lang.String networkName,
Switchboard switchboard) |
Modifier and Type | Method and Description |
---|---|
boolean |
allCrawlsFinished(CrawlQueues crawlQueues) |
void |
cleanProfiles(java.util.Set<java.lang.String> deletionCandidate) |
boolean |
clear() |
void |
close() |
CrawlProfile |
get(byte[] profileKey)
Get a profile from active or passive stack.
|
java.util.Set<byte[]> |
getActive() |
CrawlProfile |
getActive(byte[] profileKey) |
java.util.Set<java.lang.String> |
getActiveProfiles() |
java.util.Set<java.lang.String> |
getFinishedProfiles(CrawlQueues crawlQueues) |
java.util.Set<byte[]> |
getPassive() |
CrawlProfile |
getPassive(byte[] profileKey) |
CrawlProfile |
getPushCrawlProfile(java.lang.String collection) |
RowHandleSet |
getURLHashes(byte[] profileKey) |
private void |
initActiveCrawlProfiles() |
private static MapHeap |
loadFromDB(java.io.File file)
Loads crawl profiles from a DB file.
|
void |
putActive(byte[] profileKey,
CrawlProfile profile) |
void |
putPassive(byte[] profileKey,
CrawlProfile profile) |
void |
removeActive(byte[] profileKey) |
void |
removePassive(byte[] profileKey) |
private void |
resetProfiles() |
public static final java.lang.String CRAWL_PROFILE_AUTOCRAWL_DEEP
public static final java.lang.String CRAWL_PROFILE_AUTOCRAWL_SHALLOW
public static final java.lang.String CRAWL_PROFILE_PROXY
public static final java.lang.String CRAWL_PROFILE_REMOTE
public static final java.lang.String CRAWL_PROFILE_SNIPPET_LOCAL_TEXT
public static final java.lang.String CRAWL_PROFILE_SNIPPET_GLOBAL_TEXT
public static final java.lang.String CRAWL_PROFILE_GREEDY_LEARNING_TEXT
public static final java.lang.String CRAWL_PROFILE_SNIPPET_LOCAL_MEDIA
public static final java.lang.String CRAWL_PROFILE_SNIPPET_GLOBAL_MEDIA
public static final java.lang.String CRAWL_PROFILE_SURROGATE
public static java.util.Set<java.lang.String> DEFAULT_PROFILES
public static final java.lang.String DBFILE_ACTIVE_CRAWL_PROFILES
public static final java.lang.String DBFILE_PASSIVE_CRAWL_PROFILES
public static final long CRAWL_PROFILE_PROXY_RECRAWL_CYCLE
public static final long CRAWL_PROFILE_SNIPPET_LOCAL_TEXT_RECRAWL_CYCLE
public static final long CRAWL_PROFILE_SNIPPET_GLOBAL_TEXT_RECRAWL_CYCLE
public static final long CRAWL_PROFILE_GREEDY_LEARNING_TEXT_RECRAWL_CYCLE
public static final long CRAWL_PROFILE_SNIPPET_LOCAL_MEDIA_RECRAWL_CYCLE
public static final long CRAWL_PROFILE_SNIPPET_GLOBAL_MEDIA_RECRAWL_CYCLE
public static final long CRAWL_PROFILE_SURROGATE_RECRAWL_CYCLE
private final ConcurrentLog log
private MapHeap profilesActiveCrawls
private final MapHeap profilesPassiveCrawls
private final java.util.Map<byte[],CrawlProfile> profilesActiveCrawlsCache
private final java.util.Map<java.lang.String,RowHandleSet> profilesActiveCrawlsCounter
public CrawlProfile defaultProxyProfile
public CrawlProfile defaultRemoteProfile
public CrawlProfile defaultTextSnippetLocalProfile
public CrawlProfile defaultTextSnippetGlobalProfile
public CrawlProfile defaultTextGreedyLearningProfile
public CrawlProfile defaultMediaSnippetLocalProfile
public CrawlProfile defaultMediaSnippetGlobalProfile
public CrawlProfile defaultSurrogateProfile
public CrawlProfile defaultAutocrawlDeepProfile
public CrawlProfile defaultAutocrawlShallowProfile
private java.util.Map<java.lang.String,CrawlProfile> defaultPushProfiles
private final java.io.File queuesRoot
private Switchboard switchboard
public CrawlSwitchboard(java.lang.String networkName, Switchboard switchboard)
public CrawlProfile get(byte[] profileKey)
profileKey
- public CrawlProfile getActive(byte[] profileKey)
public CrawlProfile getPassive(byte[] profileKey)
public java.util.Set<byte[]> getActive()
public java.util.Set<byte[]> getPassive()
public void removeActive(byte[] profileKey)
public void removePassive(byte[] profileKey)
public void putActive(byte[] profileKey, CrawlProfile profile)
public void putPassive(byte[] profileKey, CrawlProfile profile)
public RowHandleSet getURLHashes(byte[] profileKey)
private void initActiveCrawlProfiles()
public CrawlProfile getPushCrawlProfile(java.lang.String collection)
private void resetProfiles()
public boolean clear() throws java.lang.InterruptedException
java.lang.InterruptedException
public java.util.Set<java.lang.String> getActiveProfiles()
public java.util.Set<java.lang.String> getFinishedProfiles(CrawlQueues crawlQueues)
public boolean allCrawlsFinished(CrawlQueues crawlQueues)
public void cleanProfiles(java.util.Set<java.lang.String> deletionCandidate)
public void close()
private static MapHeap loadFromDB(java.io.File file)
file
- DB file