oreolek/news-script
Archived
1
0
Fork 0
Code Issues Releases Activity

Обновление для дайджестов

Browse source
This commit is contained in:
Alexander Yakovlev 2018-03-29 22:16:19 +07:00
parent d931fd568b
commit 79f44c2cdc
7 changed files with 79 additions and 44 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
Source.php
View file

@ -15,6 +15,12 @@ abstract class Source {
$this->period = strtotime("1 week ago");
$this->output = '';
}
/**
* Should be load the page before the parsing or during
*
* @var boolean
*/
public $delayedLoad = false;
public function loadStr($html) {
$this->dom = new Crawler($html);

10
Source/Anivisual.php
View file

@ -20,6 +20,9 @@ class Anivisual extends Source {
'Декабря' => 'December',
];
protected function parse() {
$text = $this->get_text('http://anivisual.net/stuff/1');
$this->loadStr($text);
unset($text);
$this->dom->filter('.entryBlock')->each(function($gameBlock) {
$date = trim($gameBlock->filter('.icon-calendar')->text());
foreach ($this->months as $ruM => $enM) {
@ -30,8 +33,8 @@ class Anivisual extends Source {
if ($date < $this->period) return;
$game = new Game;
$link = $gameBlock->filter('.novel-ttl a')->first();
$game->title = htmlspecialchars_decode($link->innerHtml);
$game->url = 'http://anivisual.net'.$link->getAttribute('href');
$game->title = htmlspecialchars_decode($link->html());
$game->url = 'http://anivisual.net'.$link->attr('href');
$game->description = $gameBlock->filter('span')->first()->text();
$games[] = $game;
@ -43,9 +46,6 @@ class Anivisual extends Source {
return (strpos($url,'http://anivisual.net/stuff/') !== FALSE);
}
public function page($url) {
$text = $this->get_text($url);
$this->loadStr($text);
unset($text);
$game = new Game;
$game->url = $url;
$gameBlock = $this->dom->filter('#casing-box');

6
Source/Apero.php
View file

@ -28,9 +28,9 @@ class Apero extends Source {
if ($date < $this->period) return;
$game = new Game;
$game->author = trim($gameBlock->filter('.game-author-block:first-child a')->text());
$game->title = trim($gameBlock->find('h2:first-child a:first-child')->first()->text());
$game->url = trim($gameBlock->find('h2:first-child a:first-child')->first()->getAttribute('href'));
$game->description = trim($gameBlock->find('.game-desc-block')->first()->text());
$game->title = trim($gameBlock->filter('h2 a')->first()->text());
$game->url = trim($gameBlock->filter('h2 a')->first()->attr('href'));
$game->description = trim($gameBlock->filter('.game-desc-block')->first()->text());
$this->output .= $game->print();
});
}

17
Source/Itch.php
View file

@ -28,19 +28,20 @@ class Itch extends Source {
$game->description = trim(strip_tags($keyValue['{}description'], '<p><a><br>'));
}
$game_page = $this->get_text($game->url);
$this->dom->loadStr($game_page, []);
$lines = $this->dom->find('.game_info_panel_widget tr');
foreach ($lines as $line) {
$text = $line->find('td');
if ($text->innerHtml == 'Author') {
$game->author = strip_tags($text->nextSibling()->innerHtml);
$this->loadStr($game_page, []);
$lines = $this->dom->filter('.game_info_panel_widget tr')->each(function($line) use($game){
$text = $line->filter('td');
if (trim($text->text()) == 'Author') {
$game->author = strip_tags($text->nextAll()->first()->html());
}
}
});
$this->output .= $game->print();
return $game;
},
];
$dom = $service->parse($xml);
try {
$dom = $service->parse($xml);
} catch (\Exception $e) {} // ignore malformed XML
}
protected function parse() {
$this->parse_tag("https://itch.io/games/newest/tag-text-based.xml");

69
Source/Steam.php
View file

@ -3,9 +3,11 @@ namespace Source;
use \Game;
use \Symfony\Component\DomCrawler\Crawler;
use \GuzzleHttp\Cookie\CookieJar;
class Steam extends Source {
public $title = "Steam";
public $delayedLoad = true;
protected $months = [
'янв.' => 'January',
'фев.' => 'February',
@ -35,8 +37,10 @@ class Steam extends Source {
$url = $gameLink->attr('href');
$url = substr($url,0,strpos($url, '?')); // remove query string
$game = $this->page($url);
$date = $game->date->format('U');
if ($date < $this->period) return;
if ($game->date) {
$date = $game->date->format('U');
if ($date < $this->period) return;
}
$this->output .= $game->print();
});
}
@ -47,32 +51,55 @@ class Steam extends Source {
return (strpos($url,'http://store.steampowered.com/') !== FALSE);
}
public function page($url) {
$this->cookies = 'mature_content=1; Steam_Language=russian';
$this->cookies = new CookieJar([
'mature_content' => 1,
'Steam_Language' => 'russian',
'timezoneOffset' => '0,0'
]);
$this->loadStr($this->get_text($url));
$game = new Game;
$game->url = $url;
$game->title = trim($this->dom->filter('div.apphub_AppName')->first()->text());
$game->description = trim($this->dom->filter('div.game_description_snippet')->first()->text());
$game->author = trim($this->dom->filter('div#developers_list')->first()->text());
if (strpos($game->author, ',') !== FALSE) {
$game->author = explode(',', $game->author);
$game->author = array_map('trim', $game->author);
$title = $this->dom->filter('div.apphub_AppName')->first();
if ($title->count() > 0) {
$game->title = trim($title->text());
}
$dsc = $this->dom->filter('div.game_description_snippet')->first();
if ($dsc->count() > 0) {
$game->description = trim($dsc->text());
}
$author = $this->dom->filter('div#developers_list')->first();
if ($author->count() > 0) {
$game->author = trim($author->text());
if (strpos($game->author, ',') !== FALSE) {
$game->author = explode(',', $game->author);
$game->author = array_map('trim', $game->author);
}
}
$image = $this->dom->filter('img.game_header_image_full')->first();
if ($image->count() > 0) {
$game->image = $image->attr('src');
}
$game->image = $this->dom->filter('img.game_header_image_full')->first()->attr('src');
$game->categories = 'Коммерческая ИЛ';
$languages = $this->dom->filter('.game_language_options tr td:first-child');
$game->language = [];
foreach ($languages as $language) {
$game->language[] = trim($language->nodeValue);
}
$game->language = implode(', ', $game->language);
$date = $this->dom->filter('div.date')->first()->text();
$game->date = \DateTime::createFromFormat('d M, Y', $date);
if ($game->date === FALSE) { // если Steam отдал страницу на русском
foreach ($this->months as $ruM => $enM) {
$date = str_replace($ruM, $enM, $date);
if ($languages->count() > 0) {
$game->language = [];
foreach ($languages as $language) {
$game->language[] = trim($language->nodeValue);
}
$game->language = implode(', ', $game->language);
}
$date = $this->dom->filter('div.date')->first();
if ($date->count() > 0) {
$date = $date->text();
$game->date = \DateTime::createFromFormat('d M, Y', $date);
if ($game->date === FALSE) { // если Steam отдал страницу на русском
foreach ($this->months as $ruM => $enM) {
$date = str_replace($ruM, $enM, $date);
}
$game->date = \DateTime::createFromFormat('d F Y', $date);
}
$game->date = \DateTime::createFromFormat('d F Y', $date);
}
return $game;
}

11
Source/Urq.php
View file

@ -9,14 +9,13 @@ class Urq extends Source {
$text = $this->get_text('http://urq.plut.info/node/209');
$this->loadStr($text);
unset($text);
$games = $this->dom->find('.view-NewGames tr');
foreach ($games as $gameBlock) {
$games = $this->dom->filter('.view-NewGames tr')->each(function($gameBlock) {
$game = new Game;
$game->author = trim($gameBlock->find('.views-field-taxonomy-vocabulary-2')->first()->text());
$game->title = trim($gameBlock->find('.views-field-title')->text);
$game->url = trim($gameBlock->find('.views-field-title a')->getAttribute('href'));
$game->author = trim($gameBlock->filter('.views-field-taxonomy-vocabulary-2')->first()->text());
$game->title = trim($gameBlock->filter('.views-field-title')->text());
$game->url = trim($gameBlock->filter('.views-field-title a')->attr('href'));
$this->output .= $game->print();
}
});
}
public function checkPage($url) {
return (strpos($url,'http://urq.plut.info/node/') !== FALSE);

4
wiki.php
View file

@ -24,7 +24,9 @@ function check($classname) {
$cl = (new $cname());
if ($cl->checkPage($url)) {
echo 'Используется правило "'.$cl->title.'".'.PHP_EOL;
$cl->loadStr($cl->get_text($url));
if (!$cl->delayedLoad) {
$cl->loadStr($cl->get_text($url));
}
$game = $cl->page($url);
if ($game) {
$page = new Wikipage($game);