Archived
1
0
Fork 0
This commit is contained in:
Alexander Yakovlev 2018-03-25 04:11:27 +07:00
parent 89ae1da15f
commit 8f4f11e692
3 changed files with 56 additions and 542 deletions

View file

@ -9,6 +9,7 @@ abstract class Source {
protected $dom;
protected $period;
protected $output;
protected $cookies = '';
public function __construct() {
$this->period = strtotime("1 week ago");
$this->output = '';
@ -57,6 +58,7 @@ abstract class Source {
CURLOPT_RETURNTRANSFER => 1,
CURLOPT_URL => $url,
CURLOPT_CONNECTTIMEOUT => 30,
CURLOPT_COOKIE => $this->cookies,
));
$resp = curl_exec($curl);
curl_close($curl);

View file

@ -2,10 +2,24 @@
namespace Source;
use \Game;
use \PHPHtmlParser\Dom;
use \Symfony\Component\DomCrawler\Crawler;
class Steam extends Source {
public $title = "Steam";
protected $months = [
'янв.' => 'January',
'фев.' => 'February',
'мар.' => 'March',
'апр.' => 'April',
'мая.' => 'May',
'июн.' => 'June',
'июл.' => 'July',
'авг.' => 'August',
'сен.' => 'September',
'окт.' => 'October',
'ноя.' => 'November',
'дек.' => 'December',
];
protected function parse_tag($tag) {
$url = 'http://store.steampowered.com/search/';
$url .= '?'.http_build_query([
@ -14,40 +28,17 @@ class Steam extends Source {
'displayterm' => $tag,
'category1' => 998, // only games
]);
try {
$text = $this->get_text($url);
$this->dom->loadStr($text, []);
} catch (\Exception $e) {
echo $e->getMessage();
echo $e->getTraceAsString();
return "";
}
$text = $this->get_text($url);
$this->loadStr($text);
unset($text);
$games = $this->dom->find('#search_result_container a.search_result_row');
foreach ($games as $gameLink) {
$url = $gameLink->getAttribute('href');
$this->dom->filter('#search_result_container a.search_result_row')->each(function($gameLink){
$url = $gameLink->attr('href');
$url = substr($url,0,strpos($url, '?')); // remove query string
$game = new Game;
$game->url = $url;
try {
$text = $this->get_text($url);
$this->dom = new Dom;
$this->dom->loadStr($text, []);
unset($text);
$name = $this->dom->find('div.apphub_AppName');
$description = $this->dom->find('meta[property=og:description]');
try {
$game->title = $name->innerHtml;
$game->description = $description->innerHtml;
} catch (\Exception $e) {
echo 'No title or description found for '.$url.PHP_EOL;
}
} catch (\Exception $e) {
echo $e->getMessage();
echo $e->getTraceAsString();
}
$game = $this->page($url);
$date = $game->date->format('U');
if ($date < $this->period) return;
$this->output .= $game->print();
}
});
}
protected function parse() {
$this->parse_tag("text-based");
@ -56,29 +47,25 @@ class Steam extends Source {
return (strpos($url,'http://store.steampowered.com/') !== FALSE);
}
public function page($url) {
$this->cookies = 'mature_content=1; Steam_Language=russian';
$text = $this->get_text($url);
try {
$this->dom->loadStr($text, []);
} catch (\Exception $e) {
echo $e->getMessage();
echo $e->getTraceAsString();
return "";
}
$game_page = new Crawler($text);
unset($text);
$game = new Game;
$game->url = $url;
$name = $this->dom->find('div.apphub_AppName');
$description = $this->dom->find('meta[property=og:description]');
try {
$game->title = $name->innerHtml;
$game->description = $description->innerHtml;
} catch (\Exception $e) {
echo 'No title or description found for '.$url.PHP_EOL;
$game->title = trim($game_page->filter('div.apphub_AppName')->first()->text());
$game->description = trim($game_page->filter('div.game_description_snippet')->first()->text());
$game->author = trim($game_page->filter('div#developers_list')->first()->text());
$game->image = $game_page->filter('img.game_header_image_full')->first()->attr('src');
$date = $game_page->filter('div.date')->first()->text();
$game->date = \DateTime::createFromFormat('d M, Y', $date);
if ($game->date === FALSE) { // если Steam отдал страницу на русском
foreach ($this->months as $ruM => $enM) {
$date = str_replace($ruM, $enM, $date);
}
$game->date = \DateTime::createFromFormat('d F Y', $date);
}
$date = trim($gameBlock->find('.icon-calendar')->first()->text);
$date = \DateTime::createFromFormat('d F Y', $date);
$date = $date->format('U');
$game->date = $date;
return $game;
}
}

509
composer.lock generated
View file

@ -387,16 +387,16 @@
},
{
"name": "league/html-to-markdown",
"version": "4.4.1",
"version": "4.6.2",
"source": {
"type": "git",
"url": "https://github.com/thephpleague/html-to-markdown.git",
"reference": "82ea375b5b2b1da1da222644c0565c695bf88186"
"reference": "3af14d8f44838257a75822819784e83819b34e2e"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/thephpleague/html-to-markdown/zipball/82ea375b5b2b1da1da222644c0565c695bf88186",
"reference": "82ea375b5b2b1da1da222644c0565c695bf88186",
"url": "https://api.github.com/repos/thephpleague/html-to-markdown/zipball/3af14d8f44838257a75822819784e83819b34e2e",
"reference": "3af14d8f44838257a75822819784e83819b34e2e",
"shasum": ""
},
"require": {
@ -415,7 +415,7 @@
"type": "library",
"extra": {
"branch-alias": {
"dev-master": "4.5-dev"
"dev-master": "4.7-dev"
}
},
"autoload": {
@ -447,195 +447,7 @@
"html",
"markdown"
],
"time": "2017-03-16T00:45:59+00:00"
},
{
"name": "neitanod/forceutf8",
"version": "v2.0.1",
"source": {
"type": "git",
"url": "https://github.com/neitanod/forceutf8.git",
"reference": "47c883ab2739e7938a8bb0bfd1c29d48c88858de"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/neitanod/forceutf8/zipball/47c883ab2739e7938a8bb0bfd1c29d48c88858de",
"reference": "47c883ab2739e7938a8bb0bfd1c29d48c88858de",
"shasum": ""
},
"require": {
"php": ">=5.3.0"
},
"type": "library",
"autoload": {
"psr-0": {
"ForceUTF8\\": "src/"
}
},
"notification-url": "https://packagist.org/downloads/",
"authors": [
{
"name": "Sebastián Grignoli",
"email": "grignoli@gmail.com"
}
],
"description": "PHP Class Encoding featuring popular Encoding::toUTF8() function --formerly known as forceUTF8()-- that fixes mixed encoded strings.",
"homepage": "https://github.com/neitanod/forceutf8",
"time": "2017-05-22T18:50:57+00:00"
},
{
"name": "paquettg/php-html-parser",
"version": "1.7.0",
"source": {
"type": "git",
"url": "https://github.com/paquettg/php-html-parser.git",
"reference": "18845e09831dd0772b88b51e788a4f74c701224c"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/paquettg/php-html-parser/zipball/18845e09831dd0772b88b51e788a4f74c701224c",
"reference": "18845e09831dd0772b88b51e788a4f74c701224c",
"shasum": ""
},
"require": {
"paquettg/string-encode": "~0.1.0",
"php": ">=5.4"
},
"require-dev": {
"mockery/mockery": "~0.9.0",
"phpunit/phpunit": "~4.8.0",
"satooshi/php-coveralls": "~0.6.0"
},
"type": "library",
"autoload": {
"psr-0": {
"PHPHtmlParser": "src/"
}
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
"authors": [
{
"name": "Gilles Paquette",
"email": "paquettg@gmail.com",
"homepage": "http://gillespaquette.ca"
}
],
"description": "An HTML DOM parser. It allows you to manipulate HTML. Find tags on an HTML page with selectors just like jQuery.",
"homepage": "https://github.com/paquettg/php-html-parser",
"keywords": [
"dom",
"html",
"parser"
],
"time": "2016-04-06T15:24:40+00:00"
},
{
"name": "paquettg/string-encode",
"version": "0.1.1",
"source": {
"type": "git",
"url": "https://github.com/paquettg/string-encoder.git",
"reference": "cf08343649701979f581c1601d01247fa3782439"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/paquettg/string-encoder/zipball/cf08343649701979f581c1601d01247fa3782439",
"reference": "cf08343649701979f581c1601d01247fa3782439",
"shasum": ""
},
"require": {
"php": ">=5.4"
},
"require-dev": {
"phpunit/phpunit": "3.7.*"
},
"type": "library",
"autoload": {
"psr-0": {
"stringEncode": "src/"
}
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
"authors": [
{
"name": "Gilles Paquette",
"email": "paquettg@gmail.com",
"homepage": "http://gillespaquette.ca"
}
],
"description": "Facilitating the process of altering string encoding in PHP.",
"homepage": "https://github.com/paquettg/string-encoder",
"keywords": [
"charset",
"encoding",
"string"
],
"time": "2014-05-29T18:38:09+00:00"
},
{
"name": "patchwork/utf8",
"version": "v1.3.1",
"source": {
"type": "git",
"url": "https://github.com/tchwork/utf8.git",
"reference": "30ec6451aec7d2536f0af8fe535f70c764f2c47a"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/tchwork/utf8/zipball/30ec6451aec7d2536f0af8fe535f70c764f2c47a",
"reference": "30ec6451aec7d2536f0af8fe535f70c764f2c47a",
"shasum": ""
},
"require": {
"lib-pcre": ">=7.3",
"php": ">=5.3.0"
},
"suggest": {
"ext-iconv": "Use iconv for best performance",
"ext-intl": "Use Intl for best performance",
"ext-mbstring": "Use Mbstring for best performance",
"ext-wfio": "Use WFIO for UTF-8 filesystem access on Windows"
},
"type": "library",
"extra": {
"branch-alias": {
"dev-master": "1.3-dev"
}
},
"autoload": {
"psr-4": {
"Patchwork\\": "src/Patchwork/"
},
"classmap": [
"src/Normalizer.php"
]
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"(Apache-2.0 or GPL-2.0)"
],
"authors": [
{
"name": "Nicolas Grekas",
"email": "p@tchwork.com"
}
],
"description": "Portable and performant UTF-8, Unicode and Grapheme Clusters for PHP",
"homepage": "https://github.com/tchwork/utf8",
"keywords": [
"grapheme",
"i18n",
"unicode",
"utf-8",
"utf8"
],
"time": "2016-05-18T13:57:10+00:00"
"time": "2018-01-07T19:45:06+00:00"
},
{
"name": "psr/http-message",
@ -787,16 +599,16 @@
},
{
"name": "sabre/xml",
"version": "2.0.0",
"version": "2.1.0",
"source": {
"type": "git",
"url": "https://github.com/sabre-io/xml.git",
"reference": "054292959a1f2b64c10c9c7a03a816ba1872b8a3"
"reference": "22e96661458c55a2e532885922bf4a232186f6a1"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/sabre-io/xml/zipball/054292959a1f2b64c10c9c7a03a816ba1872b8a3",
"reference": "054292959a1f2b64c10c9c7a03a816ba1872b8a3",
"url": "https://api.github.com/repos/sabre-io/xml/zipball/22e96661458c55a2e532885922bf4a232186f6a1",
"reference": "22e96661458c55a2e532885922bf4a232186f6a1",
"shasum": ""
},
"require": {
@ -846,76 +658,7 @@
"dom",
"xml"
],
"time": "2016-11-16T00:41:01+00:00"
},
{
"name": "symfony/console",
"version": "v3.3.6",
"source": {
"type": "git",
"url": "https://github.com/symfony/console.git",
"reference": "b0878233cb5c4391347e5495089c7af11b8e6201"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/symfony/console/zipball/b0878233cb5c4391347e5495089c7af11b8e6201",
"reference": "b0878233cb5c4391347e5495089c7af11b8e6201",
"shasum": ""
},
"require": {
"php": ">=5.5.9",
"symfony/debug": "~2.8|~3.0",
"symfony/polyfill-mbstring": "~1.0"
},
"conflict": {
"symfony/dependency-injection": "<3.3"
},
"require-dev": {
"psr/log": "~1.0",
"symfony/config": "~3.3",
"symfony/dependency-injection": "~3.3",
"symfony/event-dispatcher": "~2.8|~3.0",
"symfony/filesystem": "~2.8|~3.0",
"symfony/http-kernel": "~2.8|~3.0",
"symfony/process": "~2.8|~3.0"
},
"suggest": {
"psr/log": "For using the console logger",
"symfony/event-dispatcher": "",
"symfony/filesystem": "",
"symfony/process": ""
},
"type": "library",
"extra": {
"branch-alias": {
"dev-master": "3.3-dev"
}
},
"autoload": {
"psr-4": {
"Symfony\\Component\\Console\\": ""
},
"exclude-from-classmap": [
"/Tests/"
]
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
"authors": [
{
"name": "Fabien Potencier",
"email": "fabien@symfony.com"
},
{
"name": "Symfony Community",
"homepage": "https://symfony.com/contributors"
}
],
"description": "Symfony Console Component",
"homepage": "https://symfony.com",
"time": "2017-07-29T21:27:59+00:00"
"time": "2018-02-08T11:00:49+00:00"
},
{
"name": "symfony/css-selector",
@ -970,62 +713,6 @@
"homepage": "https://symfony.com",
"time": "2018-02-03T14:58:37+00:00"
},
{
"name": "symfony/debug",
"version": "v3.3.6",
"source": {
"type": "git",
"url": "https://github.com/symfony/debug.git",
"reference": "7c13ae8ce1e2adbbd574fc39de7be498e1284e13"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/symfony/debug/zipball/7c13ae8ce1e2adbbd574fc39de7be498e1284e13",
"reference": "7c13ae8ce1e2adbbd574fc39de7be498e1284e13",
"shasum": ""
},
"require": {
"php": ">=5.5.9",
"psr/log": "~1.0"
},
"conflict": {
"symfony/http-kernel": ">=2.3,<2.3.24|~2.4.0|>=2.5,<2.5.9|>=2.6,<2.6.2"
},
"require-dev": {
"symfony/http-kernel": "~2.8|~3.0"
},
"type": "library",
"extra": {
"branch-alias": {
"dev-master": "3.3-dev"
}
},
"autoload": {
"psr-4": {
"Symfony\\Component\\Debug\\": ""
},
"exclude-from-classmap": [
"/Tests/"
]
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
"authors": [
{
"name": "Fabien Potencier",
"email": "fabien@symfony.com"
},
{
"name": "Symfony Community",
"homepage": "https://symfony.com/contributors"
}
],
"description": "Symfony Debug Component",
"homepage": "https://symfony.com",
"time": "2017-07-28T15:27:31+00:00"
},
{
"name": "symfony/dom-crawler",
"version": "v4.0.6",
@ -1082,81 +769,18 @@
"homepage": "https://symfony.com",
"time": "2018-02-22T10:50:29+00:00"
},
{
"name": "symfony/event-dispatcher",
"version": "v3.3.6",
"source": {
"type": "git",
"url": "https://github.com/symfony/event-dispatcher.git",
"reference": "67535f1e3fd662bdc68d7ba317c93eecd973617e"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/symfony/event-dispatcher/zipball/67535f1e3fd662bdc68d7ba317c93eecd973617e",
"reference": "67535f1e3fd662bdc68d7ba317c93eecd973617e",
"shasum": ""
},
"require": {
"php": ">=5.5.9"
},
"conflict": {
"symfony/dependency-injection": "<3.3"
},
"require-dev": {
"psr/log": "~1.0",
"symfony/config": "~2.8|~3.0",
"symfony/dependency-injection": "~3.3",
"symfony/expression-language": "~2.8|~3.0",
"symfony/stopwatch": "~2.8|~3.0"
},
"suggest": {
"symfony/dependency-injection": "",
"symfony/http-kernel": ""
},
"type": "library",
"extra": {
"branch-alias": {
"dev-master": "3.3-dev"
}
},
"autoload": {
"psr-4": {
"Symfony\\Component\\EventDispatcher\\": ""
},
"exclude-from-classmap": [
"/Tests/"
]
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
"authors": [
{
"name": "Fabien Potencier",
"email": "fabien@symfony.com"
},
{
"name": "Symfony Community",
"homepage": "https://symfony.com/contributors"
}
],
"description": "Symfony EventDispatcher Component",
"homepage": "https://symfony.com",
"time": "2017-06-09T14:53:08+00:00"
},
{
"name": "symfony/polyfill-mbstring",
"version": "v1.5.0",
"version": "v1.7.0",
"source": {
"type": "git",
"url": "https://github.com/symfony/polyfill-mbstring.git",
"reference": "7c8fae0ac1d216eb54349e6a8baa57d515fe8803"
"reference": "78be803ce01e55d3491c1397cf1c64beb9c1b63b"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/symfony/polyfill-mbstring/zipball/7c8fae0ac1d216eb54349e6a8baa57d515fe8803",
"reference": "7c8fae0ac1d216eb54349e6a8baa57d515fe8803",
"url": "https://api.github.com/repos/symfony/polyfill-mbstring/zipball/78be803ce01e55d3491c1397cf1c64beb9c1b63b",
"reference": "78be803ce01e55d3491c1397cf1c64beb9c1b63b",
"shasum": ""
},
"require": {
@ -1168,7 +792,7 @@
"type": "library",
"extra": {
"branch-alias": {
"dev-master": "1.5-dev"
"dev-master": "1.7-dev"
}
},
"autoload": {
@ -1202,106 +826,7 @@
"portable",
"shim"
],
"time": "2017-06-14T15:44:48+00:00"
},
{
"name": "symfony/process",
"version": "v3.3.6",
"source": {
"type": "git",
"url": "https://github.com/symfony/process.git",
"reference": "07432804942b9f6dd7b7377faf9920af5f95d70a"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/symfony/process/zipball/07432804942b9f6dd7b7377faf9920af5f95d70a",
"reference": "07432804942b9f6dd7b7377faf9920af5f95d70a",
"shasum": ""
},
"require": {
"php": ">=5.5.9"
},
"type": "library",
"extra": {
"branch-alias": {
"dev-master": "3.3-dev"
}
},
"autoload": {
"psr-4": {
"Symfony\\Component\\Process\\": ""
},
"exclude-from-classmap": [
"/Tests/"
]
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
"authors": [
{
"name": "Fabien Potencier",
"email": "fabien@symfony.com"
},
{
"name": "Symfony Community",
"homepage": "https://symfony.com/contributors"
}
],
"description": "Symfony Process Component",
"homepage": "https://symfony.com",
"time": "2017-07-13T13:05:09+00:00"
},
{
"name": "webmozart/assert",
"version": "1.2.0",
"source": {
"type": "git",
"url": "https://github.com/webmozart/assert.git",
"reference": "2db61e59ff05fe5126d152bd0655c9ea113e550f"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/webmozart/assert/zipball/2db61e59ff05fe5126d152bd0655c9ea113e550f",
"reference": "2db61e59ff05fe5126d152bd0655c9ea113e550f",
"shasum": ""
},
"require": {
"php": "^5.3.3 || ^7.0"
},
"require-dev": {
"phpunit/phpunit": "^4.6",
"sebastian/version": "^1.0.1"
},
"type": "library",
"extra": {
"branch-alias": {
"dev-master": "1.3-dev"
}
},
"autoload": {
"psr-4": {
"Webmozart\\Assert\\": "src/"
}
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
"authors": [
{
"name": "Bernhard Schussek",
"email": "bschussek@gmail.com"
}
],
"description": "Assertions to validate method input/output with nice error messages.",
"keywords": [
"assert",
"check",
"validate"
],
"time": "2016-11-23T20:04:58+00:00"
"time": "2018-01-30T19:27:44+00:00"
}
],
"packages-dev": [],