public final class Protocol
extends java.lang.Object
Modifier and Type | Class and Description |
---|---|
static class |
Protocol.Post
wrapper class for multi-post attempts to multiple IPs
|
private static class |
Protocol.SearchResult |
private static class |
Protocol.WriteMetadataNodeToLocalIndexThread
This thread is used to write a collection of URIMetadataNode documents to a segment allowing to be safely stopped.
|
private static class |
Protocol.WriteToLocalIndexThread
This thread is used to write a collection of Solr documents to a segment allowing to be safely stopped.
|
Modifier and Type | Field and Description |
---|---|
static java.util.concurrent.atomic.AtomicInteger |
metadataRetrievalRunning |
private static CollectionSchema[] |
snippetFields |
Constructor and Description |
---|
Protocol() |
Modifier and Type | Method and Description |
---|---|
static boolean |
authentifyRequest(serverObjects post,
serverSwitch env) |
static java.util.LinkedHashMap<java.lang.String,ContentBody> |
basicRequestParts(Switchboard sb,
java.lang.String targetHash,
java.lang.String salt)
put in all the essentials for routing and network authentication
|
protected static boolean |
checkDocumentSize(SolrDocument doc,
long maxSize)
Only when maxSize is greater than zero, check that doc size is lower.
|
static java.util.Map<java.lang.String,java.lang.String> |
crawlReceipt(Seed mySeed,
Seed target,
java.lang.String process,
java.lang.String result,
java.lang.String reason,
URIMetadataNode entry,
java.lang.String wordhashes) |
static java.util.Map<java.lang.String,java.lang.String> |
getProfile(Seed targetSeed) |
static java.util.Map<java.lang.String,java.lang.String> |
hello(Seed mySeed,
PeerActions peerActions,
java.lang.String targetAddress,
java.lang.String targetHash)
this is called to enrich the seed information by - own address (if peer is behind a nat/router) - check
peer type (virgin/junior/senior/principal) to do this, we send a 'Hello' to another peer this carries
the following information: 'iam' - own hash 'youare' - remote hash, to verify that we are correct 'key'
- a session key that the remote peer may use to answer and the own seed string we expect the following
information to be send back: - 'yourip' the ip of the connection peer (we) - 'yourtype' the type of
this peer that the other peer checked by asking for a specific word and the remote seed string one
exceptional failure case is when we know the other's peers hash, the other peers responds correctly but
they appear to be another peer by comparisment of the other peer's hash this works of course only if we
know the other peer's hash.
|
static ReferenceContainerCache<WebStructureGraph.HostReference> |
loadIDXHosts(Seed target) |
static java.util.Map<java.lang.String,java.lang.String> |
permissionMessage(java.lang.String targetAddress,
java.lang.String targetHash) |
protected static int |
primarySearch(SearchEvent event,
java.lang.String wordhashes,
java.lang.String excludehashes,
java.lang.String language,
Classification.ContentDomain contentdom,
int count,
long time,
int maxDistance,
int partitions,
Seed target,
SecondarySearchSuperviser secondarySearchSuperviser,
Blacklist blacklist) |
static RSSFeed |
queryRemoteCrawlURLs(SeedDB seedDB,
Seed target,
int maxCount,
long maxTime) |
static long[] |
queryRWICount(java.lang.String targetAddress,
java.lang.String targetHash,
int timeout) |
static Seed |
querySeed(Seed target,
java.lang.String seedHash) |
private static void |
remoteSearchProcess(SearchEvent event,
int count,
long time,
java.lang.String wordhashes,
Seed target,
Blacklist blacklist,
Protocol.SearchResult result) |
static java.lang.String |
requestPartsToString(java.util.Map<java.lang.String,ContentBody> parts) |
protected static int |
secondarySearch(SearchEvent event,
java.lang.String wordhashes,
java.lang.String urlhashes,
Classification.ContentDomain contentdom,
int count,
long time,
int maxDistance,
int partitions,
Seed target,
Blacklist blacklist) |
protected static int |
solrQuery(SearchEvent event,
SolrQuery solrQuery,
int offset,
int count,
Seed target,
int partitions,
Blacklist blacklist)
Execute solr query against specified target.
|
static java.lang.String |
transferIndex(SeedDB seeds,
Seed targetSeed,
ReferenceContainerCache<WordReference> indexes,
HandleSet urlRefs,
Segment segment,
boolean gzipBody,
int timeout)
transfer the index.
|
private static java.util.Map<java.lang.String,java.lang.String> |
transferRWI(Seed targetSeed,
ReferenceContainerCache<WordReference> indexes,
boolean gzipBody,
int timeout) |
private static java.util.Map<java.lang.String,java.lang.String> |
transferURL(Seed targetSeed,
java.lang.String[] uhs,
HandleSet urlRefs,
Segment segment,
boolean gzipBody,
int timeout) |
private static final CollectionSchema[] snippetFields
public static java.util.concurrent.atomic.AtomicInteger metadataRetrievalRunning
public static java.util.Map<java.lang.String,java.lang.String> hello(Seed mySeed, PeerActions peerActions, java.lang.String targetAddress, java.lang.String targetHash)
public static long[] queryRWICount(java.lang.String targetAddress, java.lang.String targetHash, int timeout)
public static RSSFeed queryRemoteCrawlURLs(SeedDB seedDB, Seed target, int maxCount, long maxTime)
protected static int primarySearch(SearchEvent event, java.lang.String wordhashes, java.lang.String excludehashes, java.lang.String language, Classification.ContentDomain contentdom, int count, long time, int maxDistance, int partitions, Seed target, SecondarySearchSuperviser secondarySearchSuperviser, Blacklist blacklist) throws java.lang.InterruptedException
java.lang.InterruptedException
protected static int secondarySearch(SearchEvent event, java.lang.String wordhashes, java.lang.String urlhashes, Classification.ContentDomain contentdom, int count, long time, int maxDistance, int partitions, Seed target, Blacklist blacklist) throws java.lang.InterruptedException
java.lang.InterruptedException
private static void remoteSearchProcess(SearchEvent event, int count, long time, java.lang.String wordhashes, Seed target, Blacklist blacklist, Protocol.SearchResult result) throws SpaceExceededException, java.lang.InterruptedException
SpaceExceededException
java.lang.InterruptedException
protected static int solrQuery(SearchEvent event, SolrQuery solrQuery, int offset, int count, Seed target, int partitions, Blacklist blacklist) throws java.lang.InterruptedException
event
- search event ot feed with resultssolrQuery
- solr queryoffset
- pagination start indicecount
- expected maximum resultstarget
- target peer to query. May be null : in that case, local peer is queried.partitions
- blacklist
- url list to exclude from resultsjava.lang.InterruptedException
- when interrupt status on calling thread is detected while processingprotected static boolean checkDocumentSize(SolrDocument doc, long maxSize)
doc
- document to verify. Must not be null.maxSize
- maximum allowed size in bytespublic static java.util.Map<java.lang.String,java.lang.String> permissionMessage(java.lang.String targetAddress, java.lang.String targetHash)
public static java.util.Map<java.lang.String,java.lang.String> crawlReceipt(Seed mySeed, Seed target, java.lang.String process, java.lang.String result, java.lang.String reason, URIMetadataNode entry, java.lang.String wordhashes)
public static java.lang.String transferIndex(SeedDB seeds, Seed targetSeed, ReferenceContainerCache<WordReference> indexes, HandleSet urlRefs, Segment segment, boolean gzipBody, int timeout)
targetSeed
- indexes
- urlCache
- gzipBody
- timeout
- private static java.util.Map<java.lang.String,java.lang.String> transferRWI(Seed targetSeed, ReferenceContainerCache<WordReference> indexes, boolean gzipBody, int timeout)
private static java.util.Map<java.lang.String,java.lang.String> transferURL(Seed targetSeed, java.lang.String[] uhs, HandleSet urlRefs, Segment segment, boolean gzipBody, int timeout)
public static java.util.Map<java.lang.String,java.lang.String> getProfile(Seed targetSeed)
public static ReferenceContainerCache<WebStructureGraph.HostReference> loadIDXHosts(Seed target)
public static final boolean authentifyRequest(serverObjects post, serverSwitch env)
public static final java.util.LinkedHashMap<java.lang.String,ContentBody> basicRequestParts(Switchboard sb, java.lang.String targetHash, java.lang.String salt)
sb
- targetHash
- salt
- public static java.lang.String requestPartsToString(java.util.Map<java.lang.String,ContentBody> parts)