<?php

[0] Fix | Delete

require_once(dirname(__FILE__) . '/wfAPI.php');

[1] Fix | Delete

require_once(dirname(__FILE__) . '/wfArray.php');

[2] Fix | Delete

class wordfenceURLHoover {

[3] Fix | Delete

private $debug = false;

[4] Fix | Delete

public $errorMsg = false;

[5] Fix | Delete

private $hostsToAdd = false;

[6] Fix | Delete

private $table = '';

[7] Fix | Delete

private $apiKey = false;

[8] Fix | Delete

private $wordpressVersion = false;

[9] Fix | Delete

private $useDB = true;

[10] Fix | Delete

private $hostKeys = array();

[11] Fix | Delete

private $hostList = array();

[12] Fix | Delete

public $currentHooverID = false;

[13] Fix | Delete

private $_foundSome = false;

[14] Fix | Delete

private $_excludedHosts = array();

[15] Fix | Delete

private $api = false;

[16] Fix | Delete

private $db = false;

[17] Fix | Delete

[18] Fix | Delete

public static function standardExcludedHosts() {

[19] Fix | Delete

static $standardExcludedHosts = null;

[20] Fix | Delete

if ($standardExcludedHosts !== null) {

[21] Fix | Delete

return $standardExcludedHosts;

[22] Fix | Delete

}

[23] Fix | Delete

[24] Fix | Delete

global $wpdb;

[25] Fix | Delete

$excludedHosts = array();

[26] Fix | Delete

if (is_multisite()) {

[27] Fix | Delete

$blogIDs = $wpdb->get_col("SELECT blog_id FROM {$wpdb->blogs}"); //Can't use wp_get_sites or get_sites because they return empty at 10k sites

[28] Fix | Delete

foreach ($blogIDs as $id) {

[29] Fix | Delete

$homeURL = get_home_url($id);

[30] Fix | Delete

$host = parse_url($homeURL, PHP_URL_HOST);

[31] Fix | Delete

if ($host) {

[32] Fix | Delete

$excludedHosts[$host] = 1;

[33] Fix | Delete

}

[34] Fix | Delete

$siteURL = get_site_url($id);

[35] Fix | Delete

$host = parse_url($siteURL, PHP_URL_HOST);

[36] Fix | Delete

if ($host) {

[37] Fix | Delete

$excludedHosts[$host] = 1;

[38] Fix | Delete

}

[39] Fix | Delete

}

[40] Fix | Delete

}

[41] Fix | Delete

else {

[42] Fix | Delete

$homeURL = wfUtils::wpHomeURL();

[43] Fix | Delete

$host = parse_url($homeURL, PHP_URL_HOST);

[44] Fix | Delete

if ($host) {

[45] Fix | Delete

$excludedHosts[$host] = 1;

[46] Fix | Delete

}

[47] Fix | Delete

$siteURL = wfUtils::wpSiteURL();

[48] Fix | Delete

$host = parse_url($siteURL, PHP_URL_HOST);

[49] Fix | Delete

if ($host) {

[50] Fix | Delete

$excludedHosts[$host] = 1;

[51] Fix | Delete

}

[52] Fix | Delete

}

[53] Fix | Delete

[54] Fix | Delete

$standardExcludedHosts = array_keys($excludedHosts);

[55] Fix | Delete

return $standardExcludedHosts;

[56] Fix | Delete

}

[57] Fix | Delete

[58] Fix | Delete

public function __sleep() {

[59] Fix | Delete

$this->writeHosts();

[60] Fix | Delete

return array('debug', 'errorMsg', 'table', 'apiKey', 'wordpressVersion');

[61] Fix | Delete

}

[62] Fix | Delete

[63] Fix | Delete

public function __wakeup() {

[64] Fix | Delete

$this->hostsToAdd = new wfArray(array('owner', 'host', 'path', 'hostKey'));

[65] Fix | Delete

$this->api = new wfAPI($this->apiKey, $this->wordpressVersion);

[66] Fix | Delete

$this->db = new wfDB();

[67] Fix | Delete

}

[68] Fix | Delete

[69] Fix | Delete

public function __construct($apiKey, $wordpressVersion, $db = false, $continuation = false) {

[70] Fix | Delete

$this->hostsToAdd = new wfArray(array('owner', 'host', 'path', 'hostKey'));

[71] Fix | Delete

$this->apiKey = $apiKey;

[72] Fix | Delete

$this->wordpressVersion = $wordpressVersion;

[73] Fix | Delete

$this->api = new wfAPI($apiKey, $wordpressVersion);

[74] Fix | Delete

if($db){

[75] Fix | Delete

$this->db = $db;

[76] Fix | Delete

} else {

[77] Fix | Delete

$this->db = new wfDB();

[78] Fix | Delete

}

[79] Fix | Delete

global $wpdb;

[80] Fix | Delete

if(isset($wpdb)){

[81] Fix | Delete

$this->table = wfDB::networkTable('wfHoover');

[82] Fix | Delete

} else {

[83] Fix | Delete

$this->table = 'wp_wfHoover';

[84] Fix | Delete

}

[85] Fix | Delete

[86] Fix | Delete

if (!$continuation) {

[87] Fix | Delete

$this->cleanup();

[88] Fix | Delete

}

[89] Fix | Delete

}

[90] Fix | Delete

[91] Fix | Delete

public function cleanup() {

[92] Fix | Delete

$this->db->truncate($this->table);

[93] Fix | Delete

}

[94] Fix | Delete

[95] Fix | Delete

public function hoover($id, $data, $excludedHosts = array()) {

[96] Fix | Delete

$this->currentHooverID = $id;

[97] Fix | Delete

$this->_foundSome = 0;

[98] Fix | Delete

$this->_excludedHosts = $excludedHosts;

[99] Fix | Delete

@preg_replace_callback('_((?:(?://)(?:\S+(?::\S*)?@)?(?:(?:(?:[a-z\xa1-\xff0-9.-]+)(?:\.(?:(?:xn--[a-z\xa1-\xff0-9-]+)|[a-z\xa1-\xff]{2,}))))(?::\d{2,5})?)(?:/[a-z0-9\-\_\.~\!\*\(\);\:@&\=\+\$,\?#\[\]%]*)*)_iS', array($this, 'captureURL'), $data);

[100] Fix | Delete

$this->writeHosts();

[101] Fix | Delete

return $this->_foundSome;

[102] Fix | Delete

}

[103] Fix | Delete

[104] Fix | Delete

private function dbg($msg) {

[105] Fix | Delete

if ($this->debug) { wordfence::status(4, 'info', $msg); }

[106] Fix | Delete

}

[107] Fix | Delete

[108] Fix | Delete

public function captureURL($matches) {

[109] Fix | Delete

$id = $this->currentHooverID;

[110] Fix | Delete

$url = 'http:' . $matches[0];

[111] Fix | Delete

if (!filter_var($url, FILTER_VALIDATE_URL)) {

[112] Fix | Delete

return;

[113] Fix | Delete

}

[114] Fix | Delete

[115] Fix | Delete

$components = parse_url($url);

[116] Fix | Delete

if (preg_match('/\.(xn--(?:[a-z0-9-]*)[a-z0-9]+|[a-z\xa1-\xff0-9]{2,})$/i', $components['host'], $tld)) {

[117] Fix | Delete

$tld = strtolower($tld[1]);

[118] Fix | Delete

if (strpos(wfConfig::get('tldlist', ''), '|' . $tld . '|') === false) {

[119] Fix | Delete

return;

[120] Fix | Delete

}

[121] Fix | Delete

}

[122] Fix | Delete

else {

[123] Fix | Delete

return;

[124] Fix | Delete

}

[125] Fix | Delete

[126] Fix | Delete

foreach ($this->_excludedHosts as $h) {

[127] Fix | Delete

if (strcasecmp($h, $components['host']) === 0) {

[128] Fix | Delete

return;

[129] Fix | Delete

}

[130] Fix | Delete

}

[131] Fix | Delete

[132] Fix | Delete

$this->_foundSome++;

[133] Fix | Delete

[134] Fix | Delete

$host = (isset($components['host']) ? $components['host'] : '');

[135] Fix | Delete

$path = (isset($components['path']) && !empty($components['path']) ? $components['path'] : '/');

[136] Fix | Delete

$hashes = $this->_generateHashes($url);

[137] Fix | Delete

foreach ($hashes as $h) {

[138] Fix | Delete

$this->hostsToAdd->push(array('owner' => $id, 'host' => $host, 'path' => $path, 'hostKey' => wfUtils::substr($h, 0, 4)));

[139] Fix | Delete

}

[140] Fix | Delete

[141] Fix | Delete

if($this->hostsToAdd->size() > 1000){ $this->writeHosts(); }

[142] Fix | Delete

}

[143] Fix | Delete

[144] Fix | Delete

private function writeHosts() {

[145] Fix | Delete

if ($this->hostsToAdd->size() < 1) { return; }

[146] Fix | Delete

if ($this->useDB) {

[147] Fix | Delete

$sql = "INSERT INTO " . $this->table . " (owner, host, path, hostKey) VALUES ";

[148] Fix | Delete

while ($elem = $this->hostsToAdd->shift()) {

[149] Fix | Delete

//This may be an issue for hyperDB or other abstraction layers, but leaving it for now.

[150] Fix | Delete

$sql .= sprintf("('%s', '%s', '%s', '%s'),",

[151] Fix | Delete

$this->db->realEscape($elem['owner']),

[152] Fix | Delete

$this->db->realEscape($elem['host']),

[153] Fix | Delete

$this->db->realEscape($elem['path']),

[154] Fix | Delete

$this->db->realEscape($elem['hostKey'])

[155] Fix | Delete

);

[156] Fix | Delete

}

[157] Fix | Delete

$sql = rtrim($sql, ',');

[158] Fix | Delete

$this->db->queryWrite($sql);

[159] Fix | Delete

$this->hostsToAdd->collectGarbage();

[160] Fix | Delete

}

[161] Fix | Delete

else {

[162] Fix | Delete

while ($elem = $this->hostsToAdd->shift()) {

[163] Fix | Delete

$keys = str_split($elem['hostKey'], 4);

[164] Fix | Delete

foreach ($keys as $k) {

[165] Fix | Delete

$this->hostKeys[] = $k;

[166] Fix | Delete

}

[167] Fix | Delete

$this->hostList[] = array(

[168] Fix | Delete

'owner' => $elem['owner'],

[169] Fix | Delete

'host' => $elem['host'],

[170] Fix | Delete

'path' => $elem['path'],

[171] Fix | Delete

'hostKey' => $elem['hostKey']

[172] Fix | Delete

);

[173] Fix | Delete

}

[174] Fix | Delete

$this->hostsToAdd->collectGarbage();

[175] Fix | Delete

}

[176] Fix | Delete

}

[177] Fix | Delete

public function getBaddies() {

[178] Fix | Delete

wordfence::status(4, 'info', __("Gathering host keys.", 'wordfence'));

[179] Fix | Delete

$allHostKeys = '';

[180] Fix | Delete

if ($this->useDB) {

[181] Fix | Delete

global $wpdb;

[182] Fix | Delete

$dbh = $wpdb->dbh;

[183] Fix | Delete

$useMySQLi = wfUtils::useMySQLi();

[184] Fix | Delete

if ($useMySQLi) { //If direct-access MySQLi is available, we use it to minimize the memory footprint instead of letting it fetch everything into an array first

[185] Fix | Delete

wordfence::status(4, 'info', __("Using MySQLi directly.", 'wordfence'));

[186] Fix | Delete

$result = $dbh->query("SELECT DISTINCT hostKey FROM {$this->table} ORDER BY hostKey ASC LIMIT 100000"); /* We limit to 100,000 prefixes since more than that cannot be reliably checked within the default max_execution_time */

[187] Fix | Delete

if (!is_object($result)) {

[188] Fix | Delete

$this->errorMsg = "Unable to query database";

[189] Fix | Delete

$this->dbg($this->errorMsg);

[190] Fix | Delete

return false;

[191] Fix | Delete

}

[192] Fix | Delete

while ($row = $result->fetch_assoc()) {

[193] Fix | Delete

$allHostKeys .= $row['hostKey'];

[194] Fix | Delete

}

[195] Fix | Delete

}

[196] Fix | Delete

else {

[197] Fix | Delete

$q1 = $this->db->querySelect("SELECT DISTINCT hostKey FROM {$this->table} ORDER BY hostKey ASC LIMIT 100000"); /* We limit to 100,000 prefixes since more than that cannot be reliably checked within the default max_execution_time */

[198] Fix | Delete

foreach ($q1 as $hRec) {

[199] Fix | Delete

$allHostKeys .= $hRec['hostKey'];

[200] Fix | Delete

}

[201] Fix | Delete

}

[202] Fix | Delete

}

[203] Fix | Delete

else {

[204] Fix | Delete

$allHostKeys = implode('', array_values(array_unique($this->hostKeys)));

[205] Fix | Delete

}

[206] Fix | Delete

[207] Fix | Delete

/**

[208] Fix | Delete

* Check hash prefixes first. Each one is a 4-byte binary prefix of a SHA-256 hash of the URL. The response will

[209] Fix | Delete

* be a binary list of 4-byte indices; The full URL for each index should be sent in the secondary query to

[210] Fix | Delete

* find the true good/bad status.

[211] Fix | Delete

*/

[212] Fix | Delete

[213] Fix | Delete

$allCount = wfUtils::strlen($allHostKeys) / 4;

[214] Fix | Delete

if ($allCount > 0) {

[215] Fix | Delete

if ($this->debug) {

[216] Fix | Delete

$this->dbg("Checking {$allCount} hostkeys");

[217] Fix | Delete

for ($i = 0; $i < $allCount; $i++) {

[218] Fix | Delete

$key = wfUtils::substr($allHostKeys, $i * 4, 4);

[219] Fix | Delete

$this->dbg("Checking hostkey: " . bin2hex($key));

[220] Fix | Delete

}

[221] Fix | Delete

}

[222] Fix | Delete

[223] Fix | Delete

wordfence::status(2, 'info', sprintf(/* translators: Number of domains. */ __("Checking %d host keys against Wordfence scanning servers.", 'wordfence'), $allCount));

[224] Fix | Delete

$resp = $this->api->binCall('check_host_keys', $allHostKeys);

[225] Fix | Delete

wordfence::status(2, 'info', __("Done host key check.", 'wordfence'));

[226] Fix | Delete

$this->dbg("Done host key check");

[227] Fix | Delete

[228] Fix | Delete

$badHostKeys = '';

[229] Fix | Delete

if ($resp['code'] >= 200 && $resp['code'] <= 299) {

[230] Fix | Delete

$this->dbg("Host key response: " . bin2hex($resp['data']));

[231] Fix | Delete

$dataLen = wfUtils::strlen($resp['data']);

[232] Fix | Delete

if ($dataLen > 0 && $dataLen % 2 == 0) {

[233] Fix | Delete

$this->dbg("Checking response indexes");

[234] Fix | Delete

for ($i = 0; $i < $dataLen; $i += 2) {

[235] Fix | Delete

$idx = wfUtils::array_first(unpack('n', wfUtils::substr($resp['data'], $i, 2)));

[236] Fix | Delete

$this->dbg("Checking index {$idx}");

[237] Fix | Delete

if ($idx < $allCount) {

[238] Fix | Delete

$prefix = wfUtils::substr($allHostKeys, $idx * 4, 4);

[239] Fix | Delete

$badHostKeys .= $prefix;

[240] Fix | Delete

$this->dbg("Got bad hostkey for record: " . bin2hex($prefix));

[241] Fix | Delete

}

[242] Fix | Delete

else {

[243] Fix | Delete

$this->dbg("Bad allHostKeys index: {$idx}");

[244] Fix | Delete

$this->errorMsg = "Bad allHostKeys index: {$idx}";

[245] Fix | Delete

return false;

[246] Fix | Delete

}

[247] Fix | Delete

}

[248] Fix | Delete

}

[249] Fix | Delete

else if ($dataLen > 0) {

[250] Fix | Delete

$this->errorMsg = "Invalid data length received from Wordfence server: " . $dataLen;

[251] Fix | Delete

$this->dbg($this->errorMsg);

[252] Fix | Delete

return false;

[253] Fix | Delete

}

[254] Fix | Delete

}

[255] Fix | Delete

else {

[256] Fix | Delete

$this->errorMsg = "Wordfence server responded with an error. HTTP code " . $resp['code'] . " and data: " . $resp['data'];

[257] Fix | Delete

return false;

[258] Fix | Delete

}

[259] Fix | Delete

[260] Fix | Delete

$badCount = wfUtils::strlen($badHostKeys) / 4;

[261] Fix | Delete

if ($badCount > 0) {

[262] Fix | Delete

$urlsToCheck = array();

[263] Fix | Delete

$totalURLs = 0;

[264] Fix | Delete

[265] Fix | Delete

//Reconcile flagged prefixes with their corresponding URLs

[266] Fix | Delete

for ($i = 0; $i < $badCount; $i++) {

[267] Fix | Delete

$prefix = wfUtils::substr($badHostKeys, $i * 4, 4);

[268] Fix | Delete

[269] Fix | Delete

if ($this->useDB) {

[270] Fix | Delete

/**

[271] Fix | Delete

* Putting a 10000 limit in here for sites that have a huge number of items with the same URL

[272] Fix | Delete

* that repeats. This is an edge case. But if the URLs are malicious then presumably the admin

[273] Fix | Delete

* will fix the malicious URLs and on subsequent scans the items (owners) that are above the

[274] Fix | Delete

* 10000 limit will appear.

[275] Fix | Delete

*/

[276] Fix | Delete

$q1 = $this->db->querySelect("SELECT DISTINCT owner, host, path FROM {$this->table} WHERE hostKey = %s LIMIT 10000", $prefix);

[277] Fix | Delete

foreach ($q1 as $rec) {

[278] Fix | Delete

$url = 'http://' . $rec['host'] . $rec['path'];

[279] Fix | Delete

if (!isset($urlsToCheck[$rec['owner']])) {

[280] Fix | Delete

$urlsToCheck[$rec['owner']] = array();

[281] Fix | Delete

}

[282] Fix | Delete

if (!in_array($url, $urlsToCheck[$rec['owner']])) {

[283] Fix | Delete

$urlsToCheck[$rec['owner']][] = $url;

[284] Fix | Delete

$totalURLs++;

[285] Fix | Delete

}

[286] Fix | Delete

}

[287] Fix | Delete

}

[288] Fix | Delete

else {

[289] Fix | Delete

foreach ($this->hostList as $rec) {

[290] Fix | Delete

$pos = wfUtils::strpos($rec['hostKey'], $prefix);

[291] Fix | Delete

if ($pos !== false && $pos % 4 == 0) {

[292] Fix | Delete

$url = 'http://' . $rec['host'] . $rec['path'];

[293] Fix | Delete

if (!isset($urlsToCheck[$rec['owner']])) {

[294] Fix | Delete

$urlsToCheck[$rec['owner']] = array();

[295] Fix | Delete

}

[296] Fix | Delete

if (!in_array($url, $urlsToCheck[$rec['owner']])) {

[297] Fix | Delete

$urlsToCheck[$rec['owner']][] = $url;

[298] Fix | Delete

$totalURLs++;

[299] Fix | Delete

}

[300] Fix | Delete

}

[301] Fix | Delete

}

[302] Fix | Delete

}

[303] Fix | Delete

if ($totalURLs > 10000) { break; }

[304] Fix | Delete

}

[305] Fix | Delete

[306] Fix | Delete

if (count($urlsToCheck) > 0) {

[307] Fix | Delete

wordfence::status(2, 'info', sprintf(

[308] Fix | Delete

/* translators: 1. Number of URLs. 2. Number of files. */

[309] Fix | Delete

__('Checking %1$d URLs from %2$d sources.', 'wordfence'),

[310] Fix | Delete

$totalURLs,

[311] Fix | Delete

sizeof($urlsToCheck)

[312] Fix | Delete

));

[313] Fix | Delete

$badURLs = $this->api->call('check_bad_urls', array(), array('toCheck' => json_encode($urlsToCheck)));

[314] Fix | Delete

wordfence::status(2, 'info', __("Done URL check.", 'wordfence'));

[315] Fix | Delete

$this->dbg("Done URL check");

[316] Fix | Delete

if (is_array($badURLs) && count($badURLs) > 0) {

[317] Fix | Delete

$finalResults = array();

[318] Fix | Delete

foreach ($badURLs as $file => $badSiteList) {

[319] Fix | Delete

if (!isset($finalResults[$file])) {

[320] Fix | Delete

$finalResults[$file] = array();

[321] Fix | Delete

}

[322] Fix | Delete

foreach ($badSiteList as $badSite) {

[323] Fix | Delete

$finalResults[$file][] = array(

[324] Fix | Delete

'URL' => $badSite[0],

[325] Fix | Delete

'badList' => $badSite[1]

[326] Fix | Delete

);

[327] Fix | Delete

}

[328] Fix | Delete

}

[329] Fix | Delete

$this->dbg("Confirmed " . count($badURLs) . " bad URLs");

[330] Fix | Delete

return $finalResults;

[331] Fix | Delete

}

[332] Fix | Delete

}

[333] Fix | Delete

}

[334] Fix | Delete

}

[335] Fix | Delete

[336] Fix | Delete

return array();

[337] Fix | Delete

}

[338] Fix | Delete

[339] Fix | Delete

protected function _generateHashes($url) {

[340] Fix | Delete

//The GSB specification requires generating and sending hash prefixes for a number of additional similar URLs. See: https://developers.google.com/safe-browsing/v4/urls-hashing#suffixprefix-expressions

[341] Fix | Delete

[342] Fix | Delete

$canonicalURL = $this->_canonicalizeURL($url);

[343] Fix | Delete

[344] Fix | Delete

//Extract the scheme

[345] Fix | Delete

$scheme = 'http';

[346] Fix | Delete

if (preg_match('~^([a-z]+[a-z0-9+\.\-]*)://(.*)$~i', $canonicalURL, $matches)) {

[347] Fix | Delete

$scheme = strtolower($matches[1]);

[348] Fix | Delete

$canonicalURL = $matches[2];

[349] Fix | Delete

}

[350] Fix | Delete

[351] Fix | Delete

//Separate URL and query string

[352] Fix | Delete

$query = '';

[353] Fix | Delete

if (preg_match('/^([^?]+)(\??.*)/', $canonicalURL, $matches)) {

[354] Fix | Delete

$canonicalURL = $matches[1];

[355] Fix | Delete

$query = $matches[2];

[356] Fix | Delete

}

[357] Fix | Delete

[358] Fix | Delete

//Separate host and path

[359] Fix | Delete

$path = '';

[360] Fix | Delete

preg_match('~^(.*?)(?:(/.*)|$)~', $canonicalURL, $matches);

[361] Fix | Delete

$host = $matches[1];

[362] Fix | Delete

if (isset($matches[2])) {

[363] Fix | Delete

$path = $matches[2];

[364] Fix | Delete

}

[365] Fix | Delete

[366] Fix | Delete

//Clean host

[367] Fix | Delete

$host = $this->_normalizeHost($host);

[368] Fix | Delete

[369] Fix | Delete

//Generate hosts list

[370] Fix | Delete

$hosts = array();

[371] Fix | Delete

if (filter_var(trim($host, '[]'), FILTER_VALIDATE_IP)) {

[372] Fix | Delete

$hosts[] = $host;

[373] Fix | Delete

}

[374] Fix | Delete

else {

[375] Fix | Delete

$hostComponents = explode('.', $host);

[376] Fix | Delete

[377] Fix | Delete

$numComponents = count($hostComponents) - 7;

[378] Fix | Delete

if ($numComponents < 1) {

[379] Fix | Delete

$numComponents = 1;

[380] Fix | Delete

}

[381] Fix | Delete

[382] Fix | Delete

$hosts[] = $host;

[383] Fix | Delete

for ($i = $numComponents; $i < count($hostComponents) - 1; $i++) {

[384] Fix | Delete

$hosts[] = implode('.', array_slice($hostComponents, $i));

[385] Fix | Delete

}

[386] Fix | Delete

}

[387] Fix | Delete

[388] Fix | Delete

//Generate paths list

[389] Fix | Delete

$paths = array('/');

[390] Fix | Delete

$pathComponents = array_filter(explode('/', $path));

[391] Fix | Delete

[392] Fix | Delete

$numComponents = min(count($pathComponents), 4);

[393] Fix | Delete

for ($i = 1; $i < $numComponents; $i++) {

[394] Fix | Delete

$paths[] = '/' . implode('/', array_slice($pathComponents, 0, $i)) . '/';

[395] Fix | Delete

}

[396] Fix | Delete

if ($path != '/') {

[397] Fix | Delete

$paths[] = $path;

[398] Fix | Delete

}

[399] Fix | Delete

if (strlen($query) > 0) {

[400] Fix | Delete

$paths[] = $path . '?' . $query;

[401] Fix | Delete

}

[402] Fix | Delete

$paths = array_reverse($paths); //So we start at the most specific and move to most generic

[403] Fix | Delete

[404] Fix | Delete

//Generate hashes

[405] Fix | Delete

$hashes = array();

[406] Fix | Delete

foreach ($hosts as $h) {

[407] Fix | Delete

$hashes[$h] = hash('sha256', $h, true); //WFSB compatibility -- it uses hashes without the path

[408] Fix | Delete

foreach ($paths as $p) {

[409] Fix | Delete

$key = $h . $p;

[410] Fix | Delete

$hashes[$key] = hash('sha256', $key, true);

[411] Fix | Delete

break; //We no longer have any use for the extra path variants, so just include the primary one and exit the loop after

[412] Fix | Delete

}

[413] Fix | Delete

}

[414] Fix | Delete

[415] Fix | Delete

return $hashes;

[416] Fix | Delete

}

[417] Fix | Delete

[418] Fix | Delete

protected function _canonicalizeURL($url) { //Based on https://developers.google.com/safe-browsing/v4/urls-hashing#canonicalization and Google's reference implementation https://github.com/google/safebrowsing/blob/master/urls.go

[419] Fix | Delete

//Strip fragment

[420] Fix | Delete

$url = $this->_array_first(explode('#', $url));

[421] Fix | Delete

[422] Fix | Delete

//Trim space

[423] Fix | Delete

$url = trim($url);

[424] Fix | Delete

[425] Fix | Delete

//Remove tabs, CR, LF

[426] Fix | Delete

$url = preg_replace('/[\t\n\r]/', '', $url);

[427] Fix | Delete

[428] Fix | Delete

//Normalize escapes

[429] Fix | Delete

$url = $this->_normalizeEscape($url);

[430] Fix | Delete

if ($url === false) { return false; }

[431] Fix | Delete

[432] Fix | Delete

//Extract the scheme

[433] Fix | Delete

$scheme = 'http';

[434] Fix | Delete

if (preg_match('~^([a-z]+[a-z0-9+\.\-]*)://(.*)$~i', $url, $matches)) {

[435] Fix | Delete

$scheme = strtolower($matches[1]);

[436] Fix | Delete

$url = $matches[2];

[437] Fix | Delete

}

[438] Fix | Delete

[439] Fix | Delete

//Separate URL and query string

[440] Fix | Delete

$query = '';

[441] Fix | Delete

if (preg_match('/^([^?]+)(\??.*)/', $url, $matches)) {

[442] Fix | Delete

$url = $matches[1];

[443] Fix | Delete

$query = $matches[2];

[444] Fix | Delete

}

[445] Fix | Delete

$endsWithSlash = substr($url, -1) == '/';

[446] Fix | Delete

[447] Fix | Delete

//Separate host and path

[448] Fix | Delete

$path = '';

[449] Fix | Delete

preg_match('~^(.*?)(?:(/.*)|$)~', $url, $matches);

[450] Fix | Delete

$host = $matches[1];

[451] Fix | Delete

if (isset($matches[2])) {

[452] Fix | Delete

$path = $matches[2];

[453] Fix | Delete

}

[454] Fix | Delete

[455] Fix | Delete

//Clean host

[456] Fix | Delete

$host = $this->_normalizeHost($host);

[457] Fix | Delete

if ($host === false) { return false; }

[458] Fix | Delete

[459] Fix | Delete

//Clean path

[460] Fix | Delete

$path = preg_replace('~//+~', '/', $path); //Multiple slashes -> single slash

[461] Fix | Delete

$path = preg_replace('~(?:^|/)\.(?:$|/)~', '/', $path); //. path components removed

[462] Fix | Delete

while (preg_match('~/(?!\.\./)[^/]+/\.\.(?:$|/)~', $path)) { //Resolve ..

[463] Fix | Delete

$path = preg_replace('~/(?!\.\./)[^/]+/\.\.(?:$|/)~', '/', $path, 1);

[464] Fix | Delete

}

[465] Fix | Delete

$path = preg_replace('~(?:^|/)\.\.(?:$|/)~', '/', $path); //Eliminate .. at the beginning

[466] Fix | Delete

$path = trim($path, '.');

[467] Fix | Delete

$path = preg_replace('/\.\.+/', '.', $path);

[468] Fix | Delete

[469] Fix | Delete

if ($path == '.' || $path == '') {

[470] Fix | Delete

$path = '/';

[471] Fix | Delete

}

[472] Fix | Delete

else if ($endsWithSlash && substr($path, -1) != '/') {

[473] Fix | Delete

$path .= '/';

[474] Fix | Delete

}

[475] Fix | Delete

[476] Fix | Delete

return $scheme . '://' . $host . $path . $query;

[477] Fix | Delete

}

[478] Fix | Delete

[479] Fix | Delete

protected function _normalizeEscape($url) {

[480] Fix | Delete

$maxDepth = 1024;

[481] Fix | Delete

$i = 0;

[482] Fix | Delete

while (preg_match('/%([0-9a-f]{2})/i', $url)) {

[483] Fix | Delete

$url = preg_replace_callback('/%([0-9a-f]{2})/i', array($this, '_hex2binCallback'), $url);

[484] Fix | Delete

$i++;

[485] Fix | Delete

[486] Fix | Delete

if ($i > $maxDepth) {

[487] Fix | Delete

return false;

[488] Fix | Delete

}

[489] Fix | Delete

}

[490] Fix | Delete

[491] Fix | Delete

return preg_replace_callback('/[\x00-\x20\x7f-\xff#%]/', array($this, '_bin2hexCallback'), $url);

[492] Fix | Delete

}

[493] Fix | Delete

[494] Fix | Delete

protected function _hex2binCallback($matches) {

[495] Fix | Delete

return wfUtils::hex2bin($matches[1]);

[496] Fix | Delete

}

[497] Fix | Delete

[498] Fix | Delete

protected function _bin2hexCallback($matches) {

[499] Fix | Delete