public class WebStructureGraph
extends java.lang.Object
Modifier and Type | Class and Description |
---|---|
static class |
WebStructureGraph.HostReference |
static class |
WebStructureGraph.HostReferenceFactory |
private static class |
WebStructureGraph.LearnObject |
private class |
WebStructureGraph.PublicRefDNSResolvingProcess |
static class |
WebStructureGraph.StructureEntry |
private class |
WebStructureGraph.StructureIterator |
Modifier and Type | Field and Description |
---|---|
static WebStructureGraph.HostReferenceFactory |
hostReferenceFactory |
private static ReferenceContainerCache<WebStructureGraph.HostReference> |
hostReferenceIndexCache |
private static long |
hostReferenceIndexCacheTime |
private static long |
hostReferenceIndexCacheTTL |
private static WebStructureGraph.LearnObject |
leanrefObjectPOISON |
private static ConcurrentLog |
log |
static int |
maxhosts |
static int |
maxref |
private java.util.concurrent.BlockingQueue<WebStructureGraph.LearnObject> |
publicRefDNSResolvingQueue |
private WebStructureGraph.PublicRefDNSResolvingProcess |
publicRefDNSResolvingWorker |
private java.util.TreeMap<java.lang.String,byte[]> |
structure_new |
private java.util.TreeMap<java.lang.String,byte[]> |
structure_old |
private java.io.File |
structureFile |
Constructor and Description |
---|
WebStructureGraph(java.io.File structureFile) |
Modifier and Type | Method and Description |
---|---|
void |
clear() |
void |
close() |
boolean |
exists(java.lang.String hosthash) |
void |
generateCitationReference(DigestURL from,
DigestURL to) |
void |
generateCitationReference(DigestURL url,
Document document) |
java.lang.String |
hostHash2hostName(java.lang.String hosthash) |
ReversibleScoreMap<java.lang.String> |
hostReferenceScore() |
java.lang.String |
hostWithMaxReferences() |
ReferenceContainerCache<WebStructureGraph.HostReference> |
incomingReferences() |
WebStructureGraph.StructureEntry |
incomingReferences(java.lang.String hosthash) |
private void |
incomingReferencesEnrich(ReferenceContainerCache<WebStructureGraph.HostReference> idx,
java.util.Iterator<WebStructureGraph.StructureEntry> structureIterator,
long time) |
void |
joinOldNew() |
private static void |
joinStructure(java.util.TreeMap<java.lang.String,byte[]> into,
java.util.TreeMap<java.lang.String,byte[]> from) |
private void |
learnrefs(WebStructureGraph.LearnObject lro) |
private static java.lang.String |
map2refstr(java.util.Map<java.lang.String,java.lang.Integer> map) |
private static java.lang.String |
none2refstr() |
WebStructureGraph.StructureEntry |
outgoingReferences(java.lang.String hosthash) |
int |
referencesCount(java.lang.String hosthash) |
private static int |
refstr2count(java.lang.String refs) |
private static java.util.Map<java.lang.String,java.lang.Integer> |
refstr2map(java.lang.String refs) |
java.util.Iterator<WebStructureGraph.StructureEntry> |
structureEntryIterator(boolean latest) |
public static int maxref
public static int maxhosts
private static final ConcurrentLog log
private final java.io.File structureFile
private final java.util.TreeMap<java.lang.String,byte[]> structure_old
private final java.util.TreeMap<java.lang.String,byte[]> structure_new
private final java.util.concurrent.BlockingQueue<WebStructureGraph.LearnObject> publicRefDNSResolvingQueue
private final WebStructureGraph.PublicRefDNSResolvingProcess publicRefDNSResolvingWorker
private static final WebStructureGraph.LearnObject leanrefObjectPOISON
public static final WebStructureGraph.HostReferenceFactory hostReferenceFactory
private static ReferenceContainerCache<WebStructureGraph.HostReference> hostReferenceIndexCache
private static long hostReferenceIndexCacheTime
private static final long hostReferenceIndexCacheTTL
public void clear()
private static int refstr2count(java.lang.String refs)
private static java.util.Map<java.lang.String,java.lang.Integer> refstr2map(java.lang.String refs)
private static java.lang.String none2refstr()
private static java.lang.String map2refstr(java.util.Map<java.lang.String,java.lang.Integer> map)
public boolean exists(java.lang.String hosthash)
public WebStructureGraph.StructureEntry outgoingReferences(java.lang.String hosthash)
public WebStructureGraph.StructureEntry incomingReferences(java.lang.String hosthash)
public ReferenceContainerCache<WebStructureGraph.HostReference> incomingReferences()
private void incomingReferencesEnrich(ReferenceContainerCache<WebStructureGraph.HostReference> idx, java.util.Iterator<WebStructureGraph.StructureEntry> structureIterator, long time)
public int referencesCount(java.lang.String hosthash)
public java.lang.String hostHash2hostName(java.lang.String hosthash)
private void learnrefs(WebStructureGraph.LearnObject lro)
private static void joinStructure(java.util.TreeMap<java.lang.String,byte[]> into, java.util.TreeMap<java.lang.String,byte[]> from)
public void joinOldNew()
public java.lang.String hostWithMaxReferences()
public ReversibleScoreMap<java.lang.String> hostReferenceScore()
public java.util.Iterator<WebStructureGraph.StructureEntry> structureEntryIterator(boolean latest)
public void close()