File: /home/gonexcom/scraper.stabene.net/autoProducts/fetchrss.php
<?php
ini_set('display_errors', 1);
ini_set('display_startup_errors', 1);
error_reporting(E_ALL);
ini_set('log_errors', 1);
ini_set('error_log', __DIR__ . '/error.log');
require_once $mainPath . '/vendor/autoload.php';
use GuzzleHttp\Client;
use GuzzleHttp\Exception\RequestException;
use GuzzleHttp\RequestOptions;
use Symfony\Component\DomCrawler\Crawler;
class FetchList
{
private $client;
private $userAgents;
private $dbHandler;
private $mainPath;
public function __construct($mainPath)
{
require_once $mainPath . '/autoProducts/config.php';
require_once $mainPath . '/autoProducts/createTables.php';
$this->dbHandler = new DBHandler();
date_default_timezone_set('Asia/Karachi');
// Define the proxy URL
$proxyUrl = sprintf("http://%s:@proxy.scrape.do:8080", 'c475b5946a9948e2b4f6622d5f9e1e99caf989a29ee');
// Define the user-agent list
$this->userAgents = USER_AGENTS;
$userAgent = $this->getRandomUserAgent();
// Create a new Guzzle client
$this->client = new Client([
//turn off proxy as scrapedo token invalid
// 'proxy' => [
// 'http' => $proxyUrl, // Proxy for HTTP requests
// 'https' => $proxyUrl, // Proxy for HTTPS requests
// ],
'timeout' => 200, // Optional: Set a timeout for the request
'connect_timeout' => 250,
'verify' => false, // Disable SSL verification
'headers' => [
'Accept' => '*/*', // Set the Accept header
'User-Agent' => $userAgent
],
// 'on_stats' => function (\GuzzleHttp\TransferStats $stats) {
// echo "Request took " . $stats->getTransferTime() . " seconds.\n";
// $logMessage = "[" . date('Y-m-d H:i:s') . "] Request took " . $stats->getTransferTime() . " seconds.\n";
// error_log($logMessage, 3, dirname(__DIR__, 1) . "/error.log");
// },
//'debug' => true, // Enable debug output to STDERR
'ssl' => [
'verify_peer' => false,
'verify_peer_name' => false,
]
]);
}
private function getRandomUserAgent()
{
return $this->userAgents[array_rand($this->userAgents)];
}
public function fetchRssData(array $urls)
{
$data = [];
foreach ($urls as $key => $url) {
try {
// Randomly select a user-agent
$userAgent = $this->getRandomUserAgent();
// Send a GET request with a random user-agent
//$response = $this->client->request('GET', $url);
$response = $this->getResponse($url, true);
// Check if the request was successful
if ($response) {
// Get the body of the response
//$body = $response->getBody()->getContents();
// $contentType = strtolower($response->getHeaderLine('Content-Type'));
$contentType = $response['type'];
$content = $response['content'];
$datePast = (new DateTime())->modify('-2 days');
//$datePast = DATE_PAST;
if( isset($key) && ( $key != null || !empty($key) ) ){
try {
$data += $this->directFromPage($url, $content, $data, $datePast, $key);
} catch (Exception $e) {
$eMsg = date('Y-m-d H:i:s') . " Error Fetching SITEMAP: site: $url " . $e->getMessage() . " in " . $e->getFile() . " on line " . $e->getLine();
error_log($eMsg, 3, dirname(__DIR__, 1) . "/error.log");
}
}
}
} catch (RequestException $e) {
$eMsg = date('Y-m-d H:i:s') . " Exception: site: $url " . $e->getMessage() . " in " . $e->getFile() . " on line " . $e->getLine();
error_log($eMsg, 3, dirname(__DIR__, 1) . "/error.log");
}
}
// echo '<pre>';
// print_r( $data );
// echo '</pre>';
// die();
// exit();
return $data;
}
private function getRequest()
{
}
private function seleniumRequests($url)
{
$api = 'https://crawler4ai.wellows.com/crawl';
$body = [
'url' => $url
];
try {
$response = $this->client->post($api, [
'json' => $body,
'headers' => [
'Content-Type' => 'application/json',
'Cookie' => 'AWSALBTG=VOeBUla3xI9HVazuPuW6H3RKBY4HH0Ghac8AoCc227JaxAsKG/v2MZDDFaXyFkgwFaLB89HaMd5kdLca5q0o+ElzPPEUCJmlF46RWVVSEdWHgwYyoF3QGixFksqFUzOoG5kthvcCCa9yjYxZlgRv7mblGh3OLj4pkuFEcUTXAE+IDK+2/04=; AWSALBTGCORS=VOeBUla3xI9HVazuPuW6H3RKBY4HH0Ghac8AoCc227JaxAsKG/v2MZDDFaXyFkgwFaLB89HaMd5kdLca5q0o+ElzPPEUCJmlF46RWVVSEdWHgwYyoF3QGixFksqFUzOoG5kthvcCCa9yjYxZlgRv7mblGh3OLj4pkuFEcUTXAE+IDK+2/04='
],
'timeout' => 60,
'connect_timeout' => 30,
'http_errors' => true
]);
$responseBody = $response->getBody()->getContents();
$responseBody = json_decode($responseBody, true);
// Log successful response
$logMessage = "[" . date('Y-m-d H:i:s') . "] Crawler4AI API Success: $url" . PHP_EOL;
error_log($logMessage, 3, dirname(__DIR__, 1) . "/error.log");
// Return in the format your code expects
if (isset($responseBody['html']) || isset($responseBody['content'])) {
return [
'content' => $responseBody['html'] ?? $responseBody['content'] ?? $responseBody,
'type' => 'text/html'
];
}
// If response structure is different, return as is
return $responseBody;
} catch (\GuzzleHttp\Exception\ConnectException $e) {
// Connection failed
$eMsg = "Crawler4AI Connection Failed for $url: " . $e->getMessage();
$logMessage = "[" . date('Y-m-d H:i:s') . "] " . $eMsg . PHP_EOL;
error_log($logMessage, 3, dirname(__DIR__, 1) . "/error.log");
return false;
} catch (\GuzzleHttp\Exception\ClientException $e) {
// 4xx errors (bad request, unauthorized, etc.)
$eMsg = "Crawler4AI Client Error for $url: " . $e->getMessage();
if ($e->hasResponse()) {
$errorBody = $e->getResponse()->getBody()->getContents();
$eMsg .= " Response: " . $errorBody;
}
$logMessage = "[" . date('Y-m-d H:i:s') . "] " . $eMsg . PHP_EOL;
error_log($logMessage, 3, dirname(__DIR__, 1) . "/error.log");
return false;
} catch (\GuzzleHttp\Exception\ServerException $e) {
// 5xx errors (server errors)
$eMsg = "Crawler4AI Server Error for $url: " . $e->getMessage();
if ($e->hasResponse()) {
$errorBody = $e->getResponse()->getBody()->getContents();
$eMsg .= " Response: " . $errorBody;
}
$logMessage = "[" . date('Y-m-d H:i:s') . "] " . $eMsg . PHP_EOL;
error_log($logMessage, 3, dirname(__DIR__, 1) . "/error.log");
return false;
} catch (\GuzzleHttp\Exception\RequestException $e) {
// Other HTTP errors
$eMsg = "Crawler4AI Request Failed for $url: " . $e->getMessage();
if ($e->hasResponse()) {
$errorBody = $e->getResponse()->getBody()->getContents();
$eMsg .= " Response: " . $errorBody;
}
$logMessage = "[" . date('Y-m-d H:i:s') . "] " . $eMsg . PHP_EOL;
error_log($logMessage, 3, dirname(__DIR__, 1) . "/error.log");
return false;
} catch (Exception $e) {
// Generic errors
$eMsg = "Crawler4AI Error for $url: " . $e->getMessage();
$logMessage = "[" . date('Y-m-d H:i:s') . "] " . $eMsg . PHP_EOL;
error_log($logMessage, 3, dirname(__DIR__, 1) . "/error.log");
return false;
}
}
private function getResponse($url , $viewSource = false, $selenium = false)
{
try {
if ($selenium === true) {
throw new Exception("Skipping try block go to selenium");
}
$response = $this->client->request('GET', $url);
if ($response) {
$eMsg = "Exception: (Enter In guzzle Request URL FETCHING) $url";
$logMessage = "[" . date('Y-m-d H:i:s') . "] " . $eMsg . PHP_EOL;
error_log($logMessage, 3, dirname(__DIR__, 1) . "/error.log"); // Change the path to your error log file
}
if (isset($response) && $response->getStatusCode() == 200) {
$htmlContent = $response->getBody()->getContents();
$response = [
'content' => $htmlContent,
'type' => $htmlContent
] ;
return $response;
}
} catch (Exception $e) {
$eMsg = "Exception: (Response Failed Using Guzzle (Fetch Urls))" . $e->getMessage() . " in " . $e->getFile() . " on line " . $e->getLine();
$logMessage = "[" . date('Y-m-d H:i:s') . "] " . $eMsg . PHP_EOL;
error_log($logMessage, 3, dirname(__DIR__, 1) . "/error.log"); // Change the path to your error log file
//$request = $this->getResponseUsingselenium();
//$url = ($viewSource) ? "view-source:" . $url : $url;
$response = $this->seleniumRequests($url);
// echo'<pre>';
// print_r($response['data']['rawHtml']);
// echo '</pre>';
// die();
// echo'<pre>';
// print_r($response['results'][0]['content']);
// echo '</pre>';
// die();
if ($response['data']['rawHtml']) {
$eMsg = "Exception: (Success Selenium URL FETCH) $url" . $e->getMessage() . " in " . $e->getFile() . " on line " . $e->getLine();
$logMessage = "[" . date('Y-m-d H:i:s') . "] " . $eMsg . PHP_EOL;
error_log($logMessage, 3, dirname(__DIR__, 1) . "/error.log"); // Change the path to your error log file
}
if ($response['error']) {
$eMsg = "Exception: (ERROR Selenium URL FETCH) $url" . $e->getMessage() . " in " . $e->getFile() . " on line " . $e->getLine();
$logMessage = "[" . date('Y-m-d H:i:s') . "] " . $eMsg . PHP_EOL;
error_log($logMessage, 3, dirname(__DIR__, 1) . "/error.log"); // Change the path to your error log file
}
$response = [
'content' => $response['data']['rawHtml'],
'type' => $response['status']
];
return $response ?? null;
}
}
private function filterDomainRegion($url)
{
$allcountries = COUNTRIES_LISTS;
// Convert the URL to lowercase for case-insensitive matching
$lowercaseUrl = strtolower($url);
// Check if any of the country names or domains are in the URL
foreach ($allcountries as $countryName => $countryCode) {
if (strpos($lowercaseUrl, $countryName) !== false) {
return $countryCode;
}
}
// Return null if no match is found
return 'us';
}
private function fetchDateFromSchema($data) {
if (!$data) {
return false;
}
// Pattern to match the JSON-LD datePublished
$pattern1 = '/"datePublished":"(.*?)"/';
// Second regex pattern (fallback)
$pattern2 = '/"datePublished"\s*:\s*"([^"]+)"/';
// Pattern to match the <meta> tag with itemprop="datePublished"
$pattern3 = '/<meta\s+itemprop="datePublished"\s+content="([^"]+)"/';
// Try the first pattern
if (preg_match($pattern1, $data, $matches)) {
return $matches[1]; // Return the date value from the first pattern
}
// If no match, try the second pattern
if (preg_match($pattern2, $data, $matches)) {
return $matches[1]; // Return the date value from the second pattern
}
// If still no match, try the third pattern for <meta> tag
if (preg_match($pattern3, $data, $matches)) {
return $matches[1]; // Return the date value from the <meta> tag
}
return null; // Return null if no datePublished found
}
private function directFromPage($url, $response, &$data, $datePast, $key)
{
if (!$response) {
return false;
}
// echo $response;
// die();
$keys = explode(',' , $key);
$response = preg_replace('/<span[^>]*>|<\/span>/', '', $response);
$response = str_replace(['<link>', '</link>', '<link>', '</link>', 'application/rss+xml</a>" />', '<', '>'] , ['<newslink>' , '</newslink>', '<newslink>' , '</newslink>', 'application/rss+xml</a>" /></atom:link>','<', '>'] , $response);
$rss = new Crawler($response);
$rss = $rss->filterXPath('//body')->html();
$rss = new Crawler($rss);
//print_r($rss->filterXPath("//*[contains(@class, '{$keys[0]}')]")->html());
$links = [];
$rss->filterXPath("//*[contains(@class, '{$keys[0]}')]//a")->each(function (Crawler $node, $i) use(&$data, $datePast, $url, &$keys, &$currentYear, &$previousYear, &$links){
//print_r($node->attr('href'));
if( strpos($node->attr('href') , 'customerId=25046') !== false){
return;
}
if( strpos($node->attr('href') , $keys[2]) === false){
$link = "https://" . $keys[2] . $node->attr('href');
}else{
$link = $node->attr('href');
}
$cols = [
'column' => 'links',
'values' => $link,
'operator' => '='
];
if( $this->dbHandler->slcData('news_scrape_init' , $cols) || $this->dbHandler->slcData('news_scrape_init' , $cols) ){
echo 'already found in temp table';
echo '<br>';
return $data;
}
$newsPage = $this->getResponse($link);
$newsPage = $newsPage['content'];
$newCrawl = new Crawler($newsPage);
$newCrawl = $newCrawl->filterXPath('//body')->html();
$newCrawl = new Crawler($newCrawl);
//echo $newsPage;
if ($newCrawl->filterXPath("//h1")->count() < 1) {
return;
}
$title = $newCrawl->filterXPath("//h1")->text();
try {
$title = str_replace("'", "’", $title);
$data[$title] = [
'topic' => $title,
'links' => $link
];
}catch (Exception $e) {
throw new Exception("Publish Date Invalid, : $url");
}
$links[] = $link;
});
// $rss->filter(".{$keys[0]}")->filter('a')->each(function (Crawler $node, $i) use (&$data, $datePast, $url, &$keys, &$currentYear, &$previousYear) {
// print_r($node->first()->html());
// if( strpos($node->attr('href') , $keys[2]) === false){
// $link = "https://" . $keys[2] . $node->attr('href');
// }else{
// $link = $node->attr('href');
// }
// $cols = [
// 'column' => 'links',
// 'values' => $link,
// 'operator' => '='
// ];
// if( $this->dbHandler->slcData('news_scrape_init' , $cols) || $this->dbHandler->slcData('news_scrape_init' , $cols) ){
// echo 'already found in temp table';
// echo '<br>';
// return $data;
// }
// $newsPage = $this->getResponse($link);
// $newsPage = $newsPage['content'];
// $newCrawl = new Crawler($newsPage);
// //echo $newsPage;
// if ($newCrawl->filter(".{$keys[1]}")->count() < 1) {
// return;
// }
// $title = $newCrawl->filter(".{$keys[1]}")->text();
// try {
// $title = str_replace("'", "’", $title);
// $data[$title] = [
// 'topic' => $title,
// 'links' => $link
// ];
// }catch (Exception $e) {
// throw new Exception("Publish Date Invalid, : $url");
// }
// });
// die();
return $data;
}
private function fetchTypeRss($url, $response, &$data, $datePast)
{
if (!$response) {
return false;
}
$response = preg_replace('/<span[^>]*>|<\/span>/', '', $response);
//$response = str_replace(['<', '>', ':', 'https//', 'application/rss+xml</a>" />', '<title>'], ['<', '>', '', 'https://', 'application/rss+xml</a>" /></atomlink>', '</link><title>'] , $response);
//$response = str_replace(['<link>', '</link>','<', '>', 'application/rss+xml</a>" />'] , ['<lomri>' , '</lomri>', '<', '>', 'application/rss+xml</a>" /></atom:link>'], $response );
$response = str_replace(['<link>', '</link>', '<link>', '</link>', 'application/rss+xml</a>" />', '<', '>'] , ['<newslink>' , '</newslink>', '<newslink>' , '</newslink>', 'application/rss+xml</a>" /></atom:link>','<', '>'] , $response);
//$response = str_replace('atom:link' , 'atom:atomiclnk' , $response);
//$response = html_entity_decode($response);
//$rss = simplexml_load_string($response);
$rss = new Crawler($response);
//print_r($rss->filter('rss')->filter('channel')->text());
$rss->filter('item')->each(function (Crawler $node, $i) use (&$data, $datePast, $url) {
$link = $node->filter('newslink')->text();
$pubDate = $node->filter('pubDate')->text();
$title = $node->filter('title')->text();
$region = $this->filterDomainRegion($url);
// Check for exisiting values in temp table
$cols = [
'column' => 'links',
'values' => $link,
'operator' => '='
];
if( $this->dbHandler->slcData('news_scrape_temp' , $cols) || $this->dbHandler->slcData('news_scrape_init' , $cols) ){
echo 'already found in temp table';
echo '<br>';
return $data;
}
try {
$resDateErr = DateTime::createFromFormat('j F Y - H:i', $pubDate);
$pubDate = ($resDateErr === false) ? $pubDate : $resDateErr->format('D, d M Y H:i:s O');
$datePub = new DateTime($pubDate);
if ($datePub >= $datePast) {
$title = str_replace("'", "’", $title);
$data[$title] = [
'topic' => $title,
'links' => $link,
'region' => $region
];
}
} catch (Exception $e) {
throw new Exception("Publish Date Invalid, : $url");
}
});
return $data;
}
private function fetchNonRss($url, $response, $data)
{
if (!$response) {
return false;
}
//echo $response;
$body = $response;
$crawler = new Crawler($response);
// Check if the RSS feed was loaded successfully
if ($body === false) {
throw new Exception("Failed to parse RSS feed.");
}
$items = $crawler->filter('item')->each(function (Crawler $node) {
return [
'title' => $node->filter('title')->text(),
'pubDate' => $node->filter('pubDate')->text(),
'link' => $node->filter('link')->text(),
];
});
// Iterate through each item in the RSS feed
foreach ($items as $item) {
$pubDate = $item['pubDate'];
$title = $item['title'];
$link = $item['link'];
// CHECK for existing values
$cols = [
'column' => 'links',
'values' => $link,
'operator' => '='
];
if( $this->dbHandler->slcData('news_scrape_temp' , $cols) || $this->dbHandler->slcData('news_scrape_init' , $cols) ){
echo 'already found in temp table';
echo '<br>';
return $data;
}
// Add the title and link to the data array
$dateNow = new DateTime();
$datePast = (new DateTime())->modify('-2 days');
print_r($datePast);
try {
$resDateErr = DateTime::createFromFormat('j F Y - H:i', $pubDate);
$pubDate = ($resDateErr === false) ? $pubDate : $resDateErr->format('D, d M Y H:i:s O');
$datePub = new DateTime($pubDate);
if ($datePub >= $datePast) {
$title = str_replace("'", "’", $title);
$data[$title] = $link;
}
} catch (Exception $e) {
throw new Exception("Publish Date Invalid, : $url" . $response->getStatusCode());
}
}
return $data;
}
private function fetchFromSitemap($url, $response, &$data)
{
if (!$response) {
return false;
}
//$response = str_replace(['<span class="start-tag">', '</span>', '<span class="end-tag">' , '<span>'] , '' , $response);
$response = preg_replace('/<span[^>]*>|<\/span>/', '', $response);
$response = str_replace(['<', '>', ':', 'https//'], ['<', '>', '', 'https://'] , $response);
//$rss = simplexml_load_string($response);
$rss = new Crawler($response);
print_r($response);
$dateNow = new DateTime();
$datePast = (new DateTime())->modify('-2 days');
$region = $this->filterDomainRegion($url);
$rss->filter('url')->each(function (Crawler $node , $i) use (&$data, $datePast, $region) {
//if($i < 1){
$link = $node->filter('loc')->text();
// CHECK for existing values
$cols = [
'column' => 'links',
'values' => $link,
'operator' => '='
];
if( $this->dbHandler->slcData('news_scrape_temp' , $cols) || $this->dbHandler->slcData('news_scrape_init' , $cols) ){
echo 'already found in temp table';
echo '<br>';
return $data;
}
if ($node->filter('newsnews')->count() > 0) {
$data += $this->siteMapVarmultithreads($node, $link, $data, $datePast, $region);
}elseif ($node->filter('lastmod')->count() > 0) {
$data += $this->siteMapVarsingleTHread($node, $link, $data, $datePast, $region);
}else{
$data += $this->siteMapVarOnlyLoc($node, $link, $data, $datePast, $region);
}
// }
});
return $data;
}
private function siteMapVarmultithreads($node, $link , &$data, $datePast, $region){
$pubDate = $node->filter('newsnews newspublication_date')->text();
$pubDate = explode('T', $pubDate)[0];
$datePub = new DateTime($pubDate);
if ($datePub >= $datePast) {
$body = $this->getResponse($link);
$content = $body['content'];
$crawler = new Crawler($content);
try {
$firstH1Text = $crawler->filter('h1');
$firstH1Text = $firstH1Text->count() > 0 ? $firstH1Text->first()->html() : 'No Heading';
$titles = $crawler->filter('h1')->each(function (Crawler $node, $i) {
// Get the HTML content of the node
$htmlContent = $node->html();
// Step 1: Decode HTML entities
$decodedContent = html_entity_decode($htmlContent, ENT_QUOTES | ENT_HTML5, 'UTF-8');
// Step 2: Convert encoding if necessary
// If the original encoding is unknown, you can try different encodings.
// For simplicity, let's assume the source encoding is ISO-8859-1.
$convertedContent = mb_convert_encoding($decodedContent, 'UTF-8', 'ISO-8859-1');
// Return the processed content
return $convertedContent;
});
echo $titles[0];
$title = str_replace("'", "’", $titles[0]);
// echo $title;
$data[$title] = [
'topic' => $title,
'links' => $link,
'region' => $region
];
} catch (Exception $e) {
$eMsg = "Exception Sort Phase (Content Title Search): " . $e->getMessage() . " in " . $e->getFile() . " on line " . $e->getLine();
error_log($eMsg, 3, dirname(__DIR__, 1) . '/error.log'); // Change the
}
}
return $data;
}
private function siteMapVarsingleTHread($node, $link , &$data, $datePast, $region){
$pubDate = $node->filter('lastmod')->text();
$pubDate = explode('T', $pubDate)[0];
$datePub = new DateTime($pubDate);
if ($datePub >= $datePast) {
$body = $this->getResponse($link);
$content = $body['content'];
$content = str_replace('iso-8859-1', 'utf-8', $content);
$crawler = new Crawler($content);
try {
$firstH1Text = $crawler->filter('h1');
$firstH1Text = $firstH1Text->count() > 0 ? $firstH1Text->first()->html() : 'No Heading';
$titles = $crawler->filter('h1')->each(function (Crawler $node, $i) {
// Get the HTML content of the node
return $node->html();
});
$title = str_replace("'", "’", $titles[0]);
// echo $title;
$data[$title] = [
'topic' => $title,
'links' => $link,
'region' => $region
];
} catch (Exception $e) {
$eMsg = "Exception Sort Phase (Content Title Search): " . $e->getMessage() . " in " . $e->getFile() . " on line " . $e->getLine();
error_log($eMsg, 3, dirname(__DIR__, 1) . '/error.log'); // Change the
}
}
return $data;
}
private function siteMapVarOnlyLoc($node, $link , &$data, $datePast, $region){
$body = $this->getResponse($link);
$content = $body['content'];
$content = str_replace('iso-8859-1', 'utf-8', $content);
$crawler = new Crawler($content);
try {
$firstH1Text = $crawler->filter('h1');
$firstH1Text = $firstH1Text->count() > 0 ? $firstH1Text->first()->html() : 'No Heading';
$titles = $crawler->filter('h1')->each(function (Crawler $node, $i) {
// Get the HTML content of the node
return $node->html();
});
$title = str_replace("'", "’", $titles[0]);
// echo $title;
$data[$title] = [
'topic' => $title,
'links' => $link,
'region' => $region
];
} catch (Exception $e) {
$eMsg = "Exception Sort Phase (Content Title Search): " . $e->getMessage() . " in " . $e->getFile() . " on line " . $e->getLine();
error_log($eMsg, 3, dirname(__DIR__, 1) . '/error.log'); // Change the
}
return $data;
}
}
// Example usage
// $fetchList = new FetchList();
// $urls = ['https://www.cyber.gov.au/rss/news']; // Replace with your URLs
// $data = $fetchList->fetchRssData($urls);
// print_r($data);