public class Domains
extends java.lang.Object
Modifier and Type | Field and Description |
---|---|
static long |
cacheHit_Hit |
static long |
cacheHit_Insert |
static long |
cacheHit_Miss |
static long |
cacheMiss_Hit |
static long |
cacheMiss_Insert |
static long |
cacheMiss_Miss |
private static java.util.Set<java.lang.String> |
ccSLD_TLD |
private static java.lang.String[] |
ccSLD_TLD_list |
private static int |
CONCURRENCY_LEVEL |
private static KeyList |
globalHosts |
private static java.lang.Class<?> |
InetAddressLocatorClass |
private static java.lang.reflect.Method |
InetAddressLocatorGetLocaleInetAddressMethod |
private static java.lang.String |
INTRANET_IPv4_PATTERN |
private static java.lang.String |
INTRANET_IPv6_PATTERN |
private static java.util.regex.Pattern |
INTRANET_PATTERNS |
static java.lang.String |
LOCALHOST |
private static java.lang.String |
LOCALHOST_IPv4_PATTERN |
private static java.lang.String |
LOCALHOST_IPv6_PATTERN |
private static java.lang.String |
LOCALHOST_NAME |
private static java.util.regex.Pattern |
LOCALHOST_PATTERNS |
private static java.util.Set<java.net.InetAddress> |
localHostAddresses |
private static java.util.Set<java.lang.String> |
localHostNames |
private static ConcurrentLog |
log |
private static java.util.concurrent.ConcurrentHashMap<java.lang.String,java.lang.Object> |
LOOKUP_SYNC |
private static int |
MAX_NAME_CACHE_HIT_SIZE |
private static int |
MAX_NAME_CACHE_MISS_SIZE |
private static java.util.Set<java.net.InetAddress> |
myHostAddresses |
private static ARC<java.lang.String,java.net.InetAddress> |
NAME_CACHE_HIT |
private static ARC<java.lang.String,java.lang.String> |
NAME_CACHE_MISS |
private static java.util.List<java.util.regex.Pattern> |
nameCacheNoCachingPatterns |
private static boolean |
noLocalCheck |
private static java.lang.String |
PRESENT |
private static java.util.Set<java.net.InetAddress> |
publicIPv4HostAddresses |
private static java.util.Set<java.lang.String> |
publicIPv4HostNames |
private static java.util.Set<java.net.InetAddress> |
publicIPv6HostAddresses |
private static java.util.Set<java.lang.String> |
publicIPv6HostNames |
private static TimeLimiter |
timeLimiter |
private static java.lang.String[] |
TLD_Africa |
static int |
TLD_Africa_ID |
private static java.lang.String[] |
TLD_EuropeRussia |
static int |
TLD_EuropeRussia_ID |
private static java.lang.String[] |
TLD_Generic |
static int |
TLD_Generic_ID |
static int |
TLD_Local_ID |
private static java.lang.String[] |
TLD_MiddleEastWestAsia |
static int |
TLD_MiddleEastWestAsia_ID |
private static java.lang.String[] |
TLD_MiddleSouthAmerica |
static int |
TLD_MiddleSouthAmerica_ID |
private static java.lang.String[] |
TLD_NorthAmericaOceania
! ! ! A T T E N T I O N A T T E N T I O N A T T E N T I O N ! ! !
Do not move a TLD to another group (if you do not exactly know what you
are doing)! Because it will change the hash of the url!
|
static int |
TLD_NorthAmericaOceania_ID |
private static java.lang.String[] |
TLD_SouthEastAsia |
static int |
TLD_SouthEastAsia_ID |
private static java.util.Map<java.lang.String,java.lang.Integer> |
TLDID |
Constructor and Description |
---|
Domains() |
Modifier and Type | Method and Description |
---|---|
static java.lang.String |
chopZoneID(java.lang.String ip) |
static void |
clear() |
static void |
close() |
static java.net.InetAddress |
dnsResolve(java.lang.String host0)
resolve a host address using a local DNS cache and a DNS lookup if necessary
|
static java.net.InetAddress |
dnsResolveFromCache(java.lang.String host)
Does an DNS-Check to resolve a hostname to an IP.
|
static java.lang.String |
getDNC(java.lang.String host)
compute the Domain Class Name, which is either the top-level-domain or
a combination of the second-level-domain plus top-level-domain if the second-level-domain
is a ccSLD ("country code second-level domain").
|
static int |
getDomainID(java.lang.String host,
java.net.InetAddress hostaddress) |
static java.lang.String |
getHostName(java.net.InetAddress i) |
static java.util.Locale |
getLocale(java.net.InetAddress address)
find the locale for a given Address
This uses the InetAddressLocator.jar library
TODO: integrate http://www.maxmind.com/app/geolitecountry
|
static java.util.Locale |
getLocale(java.lang.String host)
find the locale for a given host.
|
static java.lang.String |
getSmartSLD(java.lang.String host)
Compute the Second Level Domain of a host name excluding a possible use of a ccSLD.
|
static void |
init(java.io.File globalHostsnameCache) |
private static void |
insertTLDProps(java.lang.String[] TLDList,
int id) |
static boolean |
isIntranet(java.lang.String host)
check if a given host is the name for a local host address
this method will return true if noLocalCheck is switched on.
|
private static boolean |
isLocal(java.net.InetAddress a) |
static boolean |
isLocal(java.lang.String host,
java.net.InetAddress hostaddress)
check if the given host is a local address.
|
private static boolean |
isLocal(java.lang.String host,
java.net.InetAddress hostaddress,
boolean recursive) |
static boolean |
isLocalhost(java.lang.String host)
check the host ip string against localhost names
|
static boolean |
isThisHostIP(java.net.InetAddress clientAddress) |
static boolean |
isThisHostIP(java.util.Set<java.lang.String> hostNames)
this method is deprecated in some way because it is not applicable on IPv6
TODO: remove / replace
|
static boolean |
isThisHostIP(java.lang.String hostName)
this method is deprecated in some way because it is not applicable on IPv6
TODO: remove / replace
|
static void |
main(java.lang.String[] args) |
static java.util.List<java.util.regex.Pattern> |
makePatterns(java.lang.String patternList) |
static boolean |
matchesList(java.lang.String obj,
java.util.List<java.util.regex.Pattern> patterns) |
static java.util.Set<java.net.InetAddress> |
myIntranetIPs()
generate a list of intranet InetAddresses
|
static java.util.Set<java.lang.String> |
myPublicIPs() |
static java.util.Set<java.net.InetAddress> |
myPublicIPv4()
Get all IPv4 addresses which are assigned to the local host but are public IP addresses.
|
static java.util.Set<java.net.InetAddress> |
myPublicIPv6()
Get all IPv6 addresses which are assigned to the local host but are public IP addresses.
|
static java.net.InetAddress |
myPublicLocalIP()
Deprecated.
|
static int |
nameCacheHitSize()
Returns the number of entries in the nameCacheHit map
|
static int |
nameCacheMissSize() |
static int |
nameCacheNoCachingPatternsSize() |
static void |
setHostName(java.net.InetAddress i,
java.lang.String host)
in case that the host name was resolved using a time-out request
it can be nice to push that information to the name cache
|
static void |
setNoCachingPatterns(java.lang.String patternList) |
static void |
setNoLocalCheck(boolean v)
the isLocal check can be switched off to gain a better crawling speed.
|
static java.lang.String |
stripToHostName(java.lang.String target)
strip off any parts of an url, address string (containing host/ip:port) or raw IPs/Hosts,
considering that the host may also be an (IPv4) IP or a IPv6 IP in brackets.
|
static int |
stripToPort(java.lang.String target) |
private static final ConcurrentLog log
public static final java.lang.String LOCALHOST
private static java.lang.String LOCALHOST_NAME
private static java.lang.Class<?> InetAddressLocatorClass
private static java.lang.reflect.Method InetAddressLocatorGetLocaleInetAddressMethod
private static final java.util.Set<java.lang.String> ccSLD_TLD
private static final java.lang.String PRESENT
private static final java.lang.String LOCALHOST_IPv4_PATTERN
private static final java.lang.String LOCALHOST_IPv6_PATTERN
private static final java.lang.String INTRANET_IPv4_PATTERN
private static final java.lang.String INTRANET_IPv6_PATTERN
private static final java.util.regex.Pattern LOCALHOST_PATTERNS
private static final java.util.regex.Pattern INTRANET_PATTERNS
private static final int MAX_NAME_CACHE_HIT_SIZE
private static final int MAX_NAME_CACHE_MISS_SIZE
private static final int CONCURRENCY_LEVEL
private static final ARC<java.lang.String,java.net.InetAddress> NAME_CACHE_HIT
private static final ARC<java.lang.String,java.lang.String> NAME_CACHE_MISS
private static final java.util.concurrent.ConcurrentHashMap<java.lang.String,java.lang.Object> LOOKUP_SYNC
private static java.util.List<java.util.regex.Pattern> nameCacheNoCachingPatterns
public static long cacheHit_Hit
public static long cacheHit_Miss
public static long cacheHit_Insert
public static long cacheMiss_Hit
public static long cacheMiss_Miss
public static long cacheMiss_Insert
private static java.util.Set<java.net.InetAddress> myHostAddresses
private static java.util.Set<java.net.InetAddress> localHostAddresses
private static java.util.Set<java.net.InetAddress> publicIPv4HostAddresses
private static java.util.Set<java.net.InetAddress> publicIPv6HostAddresses
private static java.util.Set<java.lang.String> localHostNames
private static java.util.Set<java.lang.String> publicIPv4HostNames
private static java.util.Set<java.lang.String> publicIPv6HostNames
private static final java.lang.String[] TLD_NorthAmericaOceania
private static final java.lang.String[] TLD_MiddleSouthAmerica
private static final java.lang.String[] TLD_EuropeRussia
private static final java.lang.String[] TLD_MiddleEastWestAsia
private static final java.lang.String[] TLD_SouthEastAsia
private static final java.lang.String[] TLD_Africa
private static final java.lang.String[] TLD_Generic
private static final java.lang.String[] ccSLD_TLD_list
private static java.util.Map<java.lang.String,java.lang.Integer> TLDID
public static final int TLD_EuropeRussia_ID
public static final int TLD_MiddleSouthAmerica_ID
public static final int TLD_SouthEastAsia_ID
public static final int TLD_MiddleEastWestAsia_ID
public static final int TLD_NorthAmericaOceania_ID
public static final int TLD_Africa_ID
public static final int TLD_Generic_ID
public static final int TLD_Local_ID
private static KeyList globalHosts
private static boolean noLocalCheck
private static final TimeLimiter timeLimiter
private static void insertTLDProps(java.lang.String[] TLDList, int id)
public static void init(java.io.File globalHostsnameCache)
public static void setNoLocalCheck(boolean v)
v
- public static void close()
public static java.net.InetAddress dnsResolveFromCache(java.lang.String host) throws java.net.UnknownHostException
host
- Hostname of the host in demand.java.net.UnknownHostException
public static void setNoCachingPatterns(java.lang.String patternList) throws java.util.regex.PatternSyntaxException
java.util.regex.PatternSyntaxException
public static java.util.List<java.util.regex.Pattern> makePatterns(java.lang.String patternList) throws java.util.regex.PatternSyntaxException
java.util.regex.PatternSyntaxException
public static boolean matchesList(java.lang.String obj, java.util.List<java.util.regex.Pattern> patterns)
public static java.lang.String getHostName(java.net.InetAddress i)
public static void setHostName(java.net.InetAddress i, java.lang.String host)
i
- the inet addresshost
- the known host namepublic static java.lang.String stripToHostName(java.lang.String target)
target
- public static int stripToPort(java.lang.String target)
public static java.net.InetAddress dnsResolve(java.lang.String host0)
clienthost
- public static void clear()
public static int nameCacheHitSize()
public static int nameCacheMissSize()
public static int nameCacheNoCachingPatternsSize()
@Deprecated public static java.net.InetAddress myPublicLocalIP()
public static java.util.Set<java.lang.String> myPublicIPs()
public static java.util.Set<java.net.InetAddress> myPublicIPv4()
public static java.util.Set<java.net.InetAddress> myPublicIPv6()
public static java.util.Set<java.net.InetAddress> myIntranetIPs()
public static boolean isThisHostIP(java.lang.String hostName)
hostName
- public static boolean isThisHostIP(java.util.Set<java.lang.String> hostNames)
hostName
- public static boolean isThisHostIP(java.net.InetAddress clientAddress)
public static int getDomainID(java.lang.String host, java.net.InetAddress hostaddress)
public static java.lang.String chopZoneID(java.lang.String ip)
public static boolean isLocalhost(java.lang.String host)
host
- public static boolean isIntranet(java.lang.String host)
host
- public static boolean isLocal(java.lang.String host, java.net.InetAddress hostaddress)
host
- hostaddress
- may be null if not known yetprivate static boolean isLocal(java.lang.String host, java.net.InetAddress hostaddress, boolean recursive)
private static boolean isLocal(java.net.InetAddress a)
public static java.util.Locale getLocale(java.lang.String host)
host
- public static java.util.Locale getLocale(java.net.InetAddress address)
address
- public static java.lang.String getDNC(java.lang.String host)
host
- public static java.lang.String getSmartSLD(java.lang.String host)
host
- public static void main(java.lang.String[] args)