<?php

[0] Fix | Delete

require_once(dirname(__FILE__) . '/wfUtils.php');

[1] Fix | Delete

class wfCrawl {

[2] Fix | Delete

const GOOGLE_BOT_VERIFIED = 'verified';

[3] Fix | Delete

const GOOGLE_BOT_FAKE = 'fakeBot';

[4] Fix | Delete

const GOOGLE_BOT_UNDETERMINED = 'undetermined';

[5] Fix | Delete

[6] Fix | Delete

public static function isCrawler($UA){

[7] Fix | Delete

$browscap = new wfBrowscap();

[8] Fix | Delete

$b = $browscap->getBrowser($UA);

[9] Fix | Delete

if (!$b || $b['Parent'] == 'DefaultProperties') {

[10] Fix | Delete

$IP = wfUtils::getIP();

[11] Fix | Delete

return !wfLog::isHumanRequest($IP, $UA);

[12] Fix | Delete

}

[13] Fix | Delete

else if (isset($b['Crawler']) && $b['Crawler']) {

[14] Fix | Delete

return true;

[15] Fix | Delete

}

[16] Fix | Delete

[17] Fix | Delete

return false;

[18] Fix | Delete

}

[19] Fix | Delete

public static function verifyCrawlerPTR($hostPattern, $IP){

[20] Fix | Delete

$table = wfDB::networkTable('wfCrawlers');

[21] Fix | Delete

$db = new wfDB();

[22] Fix | Delete

$IPn = wfUtils::inet_pton($IP);

[23] Fix | Delete

$ipHex = wfDB::binaryValueToSQLHex(wfUtils::inet_pton($IPn));

[24] Fix | Delete

$status = $db->querySingle("select status from $table where IP={$ipHex} and patternSig=UNHEX(MD5('%s')) and lastUpdate > unix_timestamp() - %d", $hostPattern, WORDFENCE_CRAWLER_VERIFY_CACHE_TIME);

[25] Fix | Delete

if($status){

[26] Fix | Delete

if($status == 'verified'){

[27] Fix | Delete

return true;

[28] Fix | Delete

} else {

[29] Fix | Delete

return false;

[30] Fix | Delete

}

[31] Fix | Delete

}

[32] Fix | Delete

$host = wfUtils::reverseLookup($IP);

[33] Fix | Delete

if(! $host){

[34] Fix | Delete

$db->queryWrite("insert into $table (IP, patternSig, status, lastUpdate, PTR) values ({$ipHex}, UNHEX(MD5('%s')), '%s', unix_timestamp(), '%s') ON DUPLICATE KEY UPDATE status='%s', lastUpdate=unix_timestamp(), PTR='%s'", $hostPattern, 'noPTR', '', 'noPTR', '');

[35] Fix | Delete

return false;

[36] Fix | Delete

}

[37] Fix | Delete

if(preg_match($hostPattern, $host)){

[38] Fix | Delete

$resultIPs = wfUtils::resolveDomainName($host);

[39] Fix | Delete

$addrsMatch = false;

[40] Fix | Delete

foreach($resultIPs as $resultIP){

[41] Fix | Delete

if($resultIP == $IP){

[42] Fix | Delete

$addrsMatch = true;

[43] Fix | Delete

break;

[44] Fix | Delete

}

[45] Fix | Delete

}

[46] Fix | Delete

if($addrsMatch){

[47] Fix | Delete

$db->queryWrite("insert into $table (IP, patternSig, status, lastUpdate, PTR) values ({$ipHex}, UNHEX(MD5('%s')), '%s', unix_timestamp(), '%s') ON DUPLICATE KEY UPDATE status='%s', lastUpdate=unix_timestamp(), PTR='%s'", $hostPattern, 'verified', $host, 'verified', $host);

[48] Fix | Delete

return true;

[49] Fix | Delete

} else {

[50] Fix | Delete

$db->queryWrite("insert into $table (IP, patternSig, status, lastUpdate, PTR) values ({$ipHex}, UNHEX(MD5('%s')), '%s', unix_timestamp(), '%s') ON DUPLICATE KEY UPDATE status='%s', lastUpdate=unix_timestamp(), PTR='%s'", $hostPattern, 'fwdFail', $host, 'fwdFail', $host);

[51] Fix | Delete

return false;

[52] Fix | Delete

}

[53] Fix | Delete

} else {

[54] Fix | Delete

$db->queryWrite("insert into $table (IP, patternSig, status, lastUpdate, PTR) values ({$ipHex}, UNHEX(MD5('%s')), '%s', unix_timestamp(), '%s') ON DUPLICATE KEY UPDATE status='%s', lastUpdate=unix_timestamp(), PTR='%s'", $hostPattern, 'badPTR', $host, 'badPTR', $host);

[55] Fix | Delete

return false;

[56] Fix | Delete

}

[57] Fix | Delete

}

[58] Fix | Delete

public static function isGooglebot($userAgent = null){

[59] Fix | Delete

if ($userAgent === null) {

[60] Fix | Delete

$userAgent = isset($_SERVER['HTTP_USER_AGENT']) ? $_SERVER['HTTP_USER_AGENT'] : '';

[61] Fix | Delete

}

[62] Fix | Delete

return (bool) preg_match('/Googlebot\/\d\.\d/', $userAgent);

[63] Fix | Delete

}

[64] Fix | Delete

public static function isGoogleCrawler($userAgent = null){

[65] Fix | Delete

if ($userAgent === null) {

[66] Fix | Delete

$userAgent = isset($_SERVER['HTTP_USER_AGENT']) ? $_SERVER['HTTP_USER_AGENT'] : '';

[67] Fix | Delete

}

[68] Fix | Delete

foreach (self::$googPat as $pat) {

[69] Fix | Delete

if (preg_match($pat . 'i', $userAgent)) {

[70] Fix | Delete

return true;

[71] Fix | Delete

}

[72] Fix | Delete

}

[73] Fix | Delete

return false;

[74] Fix | Delete

}

[75] Fix | Delete

private static $googPat = array(

[76] Fix | Delete

'@^Mozilla/5\\.0 \\(.*Google Keyword Tool.*\\)$@',

[77] Fix | Delete

'@^Mozilla/5\\.0 \\(.*Feedfetcher\\-Google.*\\)$@',

[78] Fix | Delete

'@^Feedfetcher\\-Google\\-iGoogleGadgets.*$@',

[79] Fix | Delete

'@^searchbot admin\\@google\\.com$@',

[80] Fix | Delete

'@^Google\\-Site\\-Verification.*$@',

[81] Fix | Delete

'@^Google OpenSocial agent.*$@',

[82] Fix | Delete

'@^.*Googlebot\\-Mobile/2\\..*$@',

[83] Fix | Delete

'@^AdsBot\\-Google\\-Mobile.*$@',

[84] Fix | Delete

'@^google \\(.*Enterprise.*\\)$@',

[85] Fix | Delete

'@^Mediapartners\\-Google.*$@',

[86] Fix | Delete

'@^GoogleFriendConnect.*$@',

[87] Fix | Delete

'@^googlebot\\-urlconsole$@',

[88] Fix | Delete

'@^.*Google Web Preview.*$@',

[89] Fix | Delete

'@^Feedfetcher\\-Google.*$@',

[90] Fix | Delete

'@^AppEngine\\-Google.*$@',

[91] Fix | Delete

'@^Googlebot\\-Video.*$@',

[92] Fix | Delete

'@^Googlebot\\-Image.*$@',

[93] Fix | Delete

'@^Google\\-Sitemaps.*$@',

[94] Fix | Delete

'@^Googlebot/Test.*$@',

[95] Fix | Delete

'@^Googlebot\\-News.*$@',

[96] Fix | Delete

'@^.*Googlebot/2\\.1.*$@',

[97] Fix | Delete

'@^AdsBot\\-Google.*$@',

[98] Fix | Delete

'@^Google$@'

[99] Fix | Delete

);

[100] Fix | Delete

[101] Fix | Delete

[102] Fix | Delete

/**

[103] Fix | Delete

* Has correct user agent and PTR record points to .googlebot.com domain.

[104] Fix | Delete

*

[105] Fix | Delete

* @param string|null $ip

[106] Fix | Delete

* @param string|null $ua

[107] Fix | Delete

* @return bool

[108] Fix | Delete

*/

[109] Fix | Delete

public static function isVerifiedGoogleCrawler($ip = null, $ua = null) {

[110] Fix | Delete

static $verified;

[111] Fix | Delete

if (!isset($verified)) {

[112] Fix | Delete

$verified = array();

[113] Fix | Delete

}

[114] Fix | Delete

if ($ip === null) {

[115] Fix | Delete

$ip = wfUtils::getIP();

[116] Fix | Delete

}

[117] Fix | Delete

[118] Fix | Delete

if ($ip === null || $ip === false) { //Likely a CLI execution

[119] Fix | Delete

return false;

[120] Fix | Delete

}

[121] Fix | Delete

[122] Fix | Delete

if (array_key_exists($ip, $verified)) {

[123] Fix | Delete

return $verified[$ip];

[124] Fix | Delete

}

[125] Fix | Delete

if (self::isGoogleCrawler($ua)) {

[126] Fix | Delete

if (self::verifyCrawlerPTR(wordfence::getLog()->getGooglePattern(), $ip)) {

[127] Fix | Delete

$verified[$ip] = true;

[128] Fix | Delete

return $verified[$ip];

[129] Fix | Delete

}

[130] Fix | Delete

$noc1Status = self::verifyGooglebotViaNOC1($ip);

[131] Fix | Delete

if ($noc1Status == self::GOOGLE_BOT_VERIFIED) {

[132] Fix | Delete

$verified[$ip] = true;

[133] Fix | Delete

return $verified[$ip];

[134] Fix | Delete

}

[135] Fix | Delete

else if ($noc1Status == self::GOOGLE_BOT_FAKE) {

[136] Fix | Delete

$verified[$ip] = false;

[137] Fix | Delete

return $verified[$ip];

[138] Fix | Delete

}

[139] Fix | Delete

[140] Fix | Delete

return true; //We were unable to successfully validate Googlebot status so default to being permissive

[141] Fix | Delete

}

[142] Fix | Delete

$verified[$ip] = false;

[143] Fix | Delete

return $verified[$ip];

[144] Fix | Delete

}

[145] Fix | Delete

[146] Fix | Delete

/**

[147] Fix | Delete

* Attempts to verify whether an IP claiming to be Googlebot is actually Googlebot.

[148] Fix | Delete

*

[149] Fix | Delete

* @param string|null $ip

[150] Fix | Delete

* @return string

[151] Fix | Delete

*/

[152] Fix | Delete

public static function verifyGooglebotViaNOC1($ip = null) {

[153] Fix | Delete

$table = wfDB::networkTable('wfCrawlers');

[154] Fix | Delete

if ($ip === null) {

[155] Fix | Delete

$ip = wfUtils::getIP();

[156] Fix | Delete

}

[157] Fix | Delete

$db = new wfDB();

[158] Fix | Delete

$IPn = wfUtils::inet_pton($ip);

[159] Fix | Delete

$ipHex = wfDB::binaryValueToSQLHex($IPn);

[160] Fix | Delete

$patternSig = 'googlenoc1';

[161] Fix | Delete

$status = $db->querySingle("select status from $table

[162] Fix | Delete

where IP={$ipHex}

[163] Fix | Delete

and patternSig=UNHEX(MD5('%s'))

[164] Fix | Delete

and lastUpdate > unix_timestamp() - %d",

[165] Fix | Delete

$patternSig,

[166] Fix | Delete

WORDFENCE_CRAWLER_VERIFY_CACHE_TIME);

[167] Fix | Delete

if ($status === 'verified') {

[168] Fix | Delete

return self::GOOGLE_BOT_VERIFIED;

[169] Fix | Delete

} else if ($status === 'fakeBot') {

[170] Fix | Delete

return self::GOOGLE_BOT_FAKE;

[171] Fix | Delete

}

[172] Fix | Delete

[173] Fix | Delete

$api = new wfAPI(wfConfig::get('apiKey'), wfUtils::getWPVersion());

[174] Fix | Delete

try {

[175] Fix | Delete

$data = $api->call('verify_googlebot', array(

[176] Fix | Delete

'ip' => $ip,

[177] Fix | Delete

));

[178] Fix | Delete

if (is_array($data) && !empty($data['verified'])) {

[179] Fix | Delete

// Cache results

[180] Fix | Delete

$db->queryWrite("INSERT INTO {$table} (IP, patternSig, status, lastUpdate) VALUES ({$ipHex}, UNHEX(MD5('%s')), '%s', unix_timestamp()) ON DUPLICATE KEY UPDATE status = VALUES(status), lastUpdate = VALUES(lastUpdate)", $patternSig, 'verified');

[181] Fix | Delete

return self::GOOGLE_BOT_VERIFIED;

[182] Fix | Delete

} else {

[183] Fix | Delete

$db->queryWrite("INSERT INTO {$table} (IP, patternSig, status, lastUpdate) VALUES ({$ipHex}, UNHEX(MD5('%s')), '%s', unix_timestamp()) ON DUPLICATE KEY UPDATE status = VALUES(status), lastUpdate = VALUES(lastUpdate)", $patternSig, 'fakeBot');

[184] Fix | Delete

self::GOOGLE_BOT_FAKE;

[185] Fix | Delete

}

[186] Fix | Delete

} catch (Exception $e) {

[187] Fix | Delete

// Do nothing, bail

[188] Fix | Delete

}

[189] Fix | Delete

return self::GOOGLE_BOT_UNDETERMINED;

[190] Fix | Delete

}

[191] Fix | Delete

}

[192] Fix | Delete

[193] Fix | Delete