WIP on collection
This commit is contained in:
parent
d6afa0e6fa
commit
997dd373f5
|
@ -28,8 +28,9 @@ class Collect extends Command
|
|||
* @var array
|
||||
*/
|
||||
protected $parsers = [
|
||||
'Urq',
|
||||
//'Urq',
|
||||
'Anivisual',
|
||||
/*
|
||||
'Kvester',
|
||||
//'vndb',
|
||||
'Apero',
|
||||
|
@ -43,7 +44,8 @@ class Collect extends Command
|
|||
'Dashingdon',
|
||||
'Itch',
|
||||
'Gamejolt',
|
||||
'Steam',
|
||||
'Steam',
|
||||
*/
|
||||
];
|
||||
|
||||
/**
|
||||
|
@ -56,7 +58,11 @@ class Collect extends Command
|
|||
foreach ($this->parsers as $parser) {
|
||||
$cname = 'App\\Sources\\'.$parser;
|
||||
try {
|
||||
(new $cname())->parse();
|
||||
$parser = new $cname();
|
||||
$games = $parser->parse();
|
||||
foreach ($games as $game) {
|
||||
$parser->saveGame($game);
|
||||
}
|
||||
} catch (\Exception $e) {
|
||||
Log::error($e->getMessage());
|
||||
Log::debug($e->getTraceAsString());
|
||||
|
|
|
@ -90,7 +90,29 @@ abstract class Source {
|
|||
/**
|
||||
* Save the game if not a duplicate.
|
||||
*/
|
||||
protected function saveGame(Game $game) {
|
||||
$game->save();
|
||||
public function saveGame(Game $game) {
|
||||
$game->source = self::class;
|
||||
$dbmodel = NULL;
|
||||
if (isset($game->source_id)) {
|
||||
$dbmodel = Game::where('source', $game->source)
|
||||
->where('source_id', $game->source_id)
|
||||
->first();
|
||||
}
|
||||
if ($dbmodel) {
|
||||
$dbmodel->fill($game);
|
||||
$dbmodel->save();
|
||||
} else {
|
||||
$game->save();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the date of the last scraped game for this source.
|
||||
*/
|
||||
protected function getLastDate() {
|
||||
Game::where('source', self::class)
|
||||
->orderBy('created_at', 'desc')
|
||||
->limit(1)
|
||||
->value('created_at');
|
||||
}
|
||||
}
|
||||
|
|
|
@ -41,23 +41,25 @@ class Anivisual extends Source {
|
|||
$text = $this->get_text('http://anivisual.net/stuff/1');
|
||||
$this->loadStr($text);
|
||||
unset($text);
|
||||
$this->dom->filter('.entryBlock')->each(function($gameBlock) {
|
||||
$lastDate = $this->getLastDate();
|
||||
$games = [];
|
||||
$this->dom->filter('.entryBlock')->each(function($gameBlock) use($lastDate, &$games) {
|
||||
$date = trim($gameBlock->filter('.icon-calendar')->text());
|
||||
foreach ($this->months as $ruM => $enM) {
|
||||
$date = str_replace($ruM, $enM, $date);
|
||||
}
|
||||
$date = \DateTime::createFromFormat('d F Y', $date);
|
||||
if ($date >= $lastDate) {
|
||||
return;
|
||||
}
|
||||
$game->date = $date;
|
||||
$link = $gameBlock->filter('.novel-ttl a')->first();
|
||||
$link = 'http://anivisual.net'.$link->attr('href');
|
||||
$game = $this->page($link);
|
||||
//$game = new Game;
|
||||
//$game->title = htmlspecialchars_decode($link->html());
|
||||
//$game->url = 'http://anivisual.net'.$link->attr('href');
|
||||
//$game->description = $gameBlock->filter('span')->first()->text();
|
||||
|
||||
$this->saveGame($game);
|
||||
$games[] = $game;
|
||||
});
|
||||
return $games;
|
||||
}
|
||||
|
||||
public function checkPage($url) {
|
||||
|
|
|
@ -22,7 +22,7 @@ use \App\Models\Game;
|
|||
use \App\Source;
|
||||
|
||||
class Urq extends Source {
|
||||
public $title = "Библиотека URQ";
|
||||
public $title = "Библиотека URQ";
|
||||
public function parse() {
|
||||
$text = $this->get_text('http://urq.plut.info/node/209');
|
||||
$this->loadStr($text);
|
||||
|
@ -32,6 +32,7 @@ class Urq extends Source {
|
|||
$game->author = trim($gameBlock->filter('.views-field-taxonomy-vocabulary-2')->first()->text());
|
||||
$game->title = trim($gameBlock->filter('.views-field-title')->text());
|
||||
$game->url = 'http://urq.plut.info'.trim($gameBlock->filter('.views-field-title a')->attr('href'));
|
||||
$game->source_id = (int) str_replace('http://urq.plut.info/node/', '', $game->url);
|
||||
$this->saveGame($game);
|
||||
});
|
||||
}
|
||||
|
|
|
@ -1,59 +1,57 @@
|
|||
<?php
|
||||
|
||||
return [
|
||||
/*
|
||||
|--------------------------------------------------------------------------
|
||||
| Application Name
|
||||
|--------------------------------------------------------------------------
|
||||
|
|
||||
| This value is the name of your application. This value is used when the
|
||||
| framework needs to place the application's name in a notification or
|
||||
| any other location as required by the application or its packages.
|
||||
|
|
||||
*/
|
||||
|
||||
/*
|
||||
|--------------------------------------------------------------------------
|
||||
| Application Name
|
||||
|--------------------------------------------------------------------------
|
||||
|
|
||||
| This value is the name of your application. This value is used when the
|
||||
| framework needs to place the application's name in a notification or
|
||||
| any other location as required by the application or its packages.
|
||||
|
|
||||
*/
|
||||
'name' => 'Ifnews',
|
||||
|
||||
'name' => 'Ifnews',
|
||||
/*
|
||||
|--------------------------------------------------------------------------
|
||||
| Application Version
|
||||
|--------------------------------------------------------------------------
|
||||
|
|
||||
| This value determines the "version" your application is currently running
|
||||
| in. You may want to follow the "Semantic Versioning" - Given a version
|
||||
| number MAJOR.MINOR.PATCH when an update happens: https://semver.org.
|
||||
|
|
||||
*/
|
||||
|
||||
/*
|
||||
|--------------------------------------------------------------------------
|
||||
| Application Version
|
||||
|--------------------------------------------------------------------------
|
||||
|
|
||||
| This value determines the "version" your application is currently running
|
||||
| in. You may want to follow the "Semantic Versioning" - Given a version
|
||||
| number MAJOR.MINOR.PATCH when an update happens: https://semver.org.
|
||||
|
|
||||
*/
|
||||
'version' => app('git.version'),
|
||||
|
||||
'version' => app('git.version'),
|
||||
/*
|
||||
|--------------------------------------------------------------------------
|
||||
| Application Environment
|
||||
|--------------------------------------------------------------------------
|
||||
|
|
||||
| This value determines the "environment" your application is currently
|
||||
| running in. This may determine how you prefer to configure various
|
||||
| services your application utilizes. Should be true in production.
|
||||
|
|
||||
*/
|
||||
|
||||
/*
|
||||
|--------------------------------------------------------------------------
|
||||
| Application Environment
|
||||
|--------------------------------------------------------------------------
|
||||
|
|
||||
| This value determines the "environment" your application is currently
|
||||
| running in. This may determine how you prefer to configure various
|
||||
| services your application utilizes. Should be true in production.
|
||||
|
|
||||
*/
|
||||
'production' => env('DEBUG'),
|
||||
|
||||
'production' => false,
|
||||
|
||||
/*
|
||||
|--------------------------------------------------------------------------
|
||||
| Autoloaded Service Providers
|
||||
|--------------------------------------------------------------------------
|
||||
|
|
||||
| The service providers listed here will be automatically loaded on the
|
||||
| request to your application. Feel free to add your own services to
|
||||
| this array to grant expanded functionality to your applications.
|
||||
|
|
||||
*/
|
||||
|
||||
'providers' => [
|
||||
App\Providers\AppServiceProvider::class,
|
||||
],
|
||||
/*
|
||||
|--------------------------------------------------------------------------
|
||||
| Autoloaded Service Providers
|
||||
|--------------------------------------------------------------------------
|
||||
|
|
||||
| The service providers listed here will be automatically loaded on the
|
||||
| request to your application. Feel free to add your own services to
|
||||
| this array to grant expanded functionality to your applications.
|
||||
|
|
||||
*/
|
||||
|
||||
'providers' => [
|
||||
App\Providers\AppServiceProvider::class,
|
||||
],
|
||||
];
|
||||
|
|
|
@ -2,79 +2,78 @@
|
|||
|
||||
return [
|
||||
|
||||
/*
|
||||
|--------------------------------------------------------------------------
|
||||
| Default Command
|
||||
|--------------------------------------------------------------------------
|
||||
|
|
||||
| Laravel Zero will always run the command specified below when no command name is
|
||||
| provided. Consider update the default command for single command applications.
|
||||
| You cannot pass arguments to the default command because they are ignored.
|
||||
|
|
||||
*/
|
||||
/*
|
||||
|--------------------------------------------------------------------------
|
||||
| Default Command
|
||||
|--------------------------------------------------------------------------
|
||||
|
|
||||
| Laravel Zero will always run the command specified below when no command name is
|
||||
| provided. Consider update the default command for single command applications.
|
||||
| You cannot pass arguments to the default command because they are ignored.
|
||||
|
|
||||
*/
|
||||
|
||||
'default' => NunoMaduro\LaravelConsoleSummary\SummaryCommand::class,
|
||||
'default' => NunoMaduro\LaravelConsoleSummary\SummaryCommand::class,
|
||||
|
||||
/*
|
||||
|--------------------------------------------------------------------------
|
||||
| Commands Paths
|
||||
|--------------------------------------------------------------------------
|
||||
|
|
||||
| This value determines the "paths" that should be loaded by the console's
|
||||
| kernel. Foreach "path" present on the array provided below the kernel
|
||||
| will extract all "Illuminate\Console\Command" based class commands.
|
||||
|
|
||||
*/
|
||||
/*
|
||||
|--------------------------------------------------------------------------
|
||||
| Commands Paths
|
||||
|--------------------------------------------------------------------------
|
||||
|
|
||||
| This value determines the "paths" that should be loaded by the console's
|
||||
| kernel. Foreach "path" present on the array provided below the kernel
|
||||
| will extract all "Illuminate\Console\Command" based class commands.
|
||||
|
|
||||
*/
|
||||
|
||||
'paths' => [app_path('Commands')],
|
||||
'paths' => [app_path('Commands')],
|
||||
|
||||
/*
|
||||
|--------------------------------------------------------------------------
|
||||
| Added Commands
|
||||
|--------------------------------------------------------------------------
|
||||
|
|
||||
| You may want to include a single command class without having to load an
|
||||
| entire folder. Here you can specify which commands should be added to
|
||||
| your list of commands. The console's kernel will try to load them.
|
||||
|
|
||||
*/
|
||||
/*
|
||||
|--------------------------------------------------------------------------
|
||||
| Added Commands
|
||||
|--------------------------------------------------------------------------
|
||||
|
|
||||
| You may want to include a single command class without having to load an
|
||||
| entire folder. Here you can specify which commands should be added to
|
||||
| your list of commands. The console's kernel will try to load them.
|
||||
|
|
||||
*/
|
||||
|
||||
'add' => [
|
||||
// ..
|
||||
],
|
||||
'add' => [
|
||||
// ..
|
||||
],
|
||||
|
||||
/*
|
||||
|--------------------------------------------------------------------------
|
||||
| Hidden Commands
|
||||
|--------------------------------------------------------------------------
|
||||
|
|
||||
| Your application commands will always be visible on the application list
|
||||
| of commands. But you can still make them "hidden" specifying an array
|
||||
| of commands below. All "hidden" commands can still be run/executed.
|
||||
|
|
||||
*/
|
||||
/*
|
||||
|--------------------------------------------------------------------------
|
||||
| Hidden Commands
|
||||
|--------------------------------------------------------------------------
|
||||
|
|
||||
| Your application commands will always be visible on the application list
|
||||
| of commands. But you can still make them "hidden" specifying an array
|
||||
| of commands below. All "hidden" commands can still be run/executed.
|
||||
|
|
||||
*/
|
||||
|
||||
'hidden' => [
|
||||
NunoMaduro\LaravelConsoleSummary\SummaryCommand::class,
|
||||
Symfony\Component\Console\Command\HelpCommand::class,
|
||||
Illuminate\Console\Scheduling\ScheduleRunCommand::class,
|
||||
Illuminate\Console\Scheduling\ScheduleFinishCommand::class,
|
||||
Illuminate\Foundation\Console\VendorPublishCommand::class,
|
||||
],
|
||||
'hidden' => [
|
||||
NunoMaduro\LaravelConsoleSummary\SummaryCommand::class,
|
||||
Symfony\Component\Console\Command\HelpCommand::class,
|
||||
Illuminate\Console\Scheduling\ScheduleRunCommand::class,
|
||||
Illuminate\Console\Scheduling\ScheduleFinishCommand::class,
|
||||
Illuminate\Foundation\Console\VendorPublishCommand::class,
|
||||
],
|
||||
|
||||
/*
|
||||
|--------------------------------------------------------------------------
|
||||
| Removed Commands
|
||||
|--------------------------------------------------------------------------
|
||||
|
|
||||
| Do you have a service provider that loads a list of commands that
|
||||
| you don't need? No problem. Laravel Zero allows you to specify
|
||||
| below a list of commands that you don't to see in your app.
|
||||
|
|
||||
*/
|
||||
|
||||
'remove' => [
|
||||
// ..
|
||||
],
|
||||
/*
|
||||
|--------------------------------------------------------------------------
|
||||
| Removed Commands
|
||||
|--------------------------------------------------------------------------
|
||||
|
|
||||
| Do you have a service provider that loads a list of commands that
|
||||
| you don't need? No problem. Laravel Zero allows you to specify
|
||||
| below a list of commands that you don't to see in your app.
|
||||
|
|
||||
*/
|
||||
|
||||
'remove' => [
|
||||
// ..
|
||||
],
|
||||
];
|
||||
|
|
|
@ -1,35 +1,34 @@
|
|||
<?php
|
||||
|
||||
return [
|
||||
|
||||
/*
|
||||
|--------------------------------------------------------------------------
|
||||
| Default Database Connection Name
|
||||
|--------------------------------------------------------------------------
|
||||
|
|
||||
| Here you may specify which of the database connections below you wish
|
||||
| to use as your default connection for all database work. Of course
|
||||
| you may use many connections at once using the Database library.
|
||||
|
|
||||
*/
|
||||
/*
|
||||
|--------------------------------------------------------------------------
|
||||
| Default Database Connection Name
|
||||
|--------------------------------------------------------------------------
|
||||
|
|
||||
| Here you may specify which of the database connections below you wish
|
||||
| to use as your default connection for all database work. Of course
|
||||
| you may use many connections at once using the Database library.
|
||||
|
|
||||
*/
|
||||
|
||||
'default' => env('DB_CONNECTION', 'sqlite'),
|
||||
|
||||
/*
|
||||
|--------------------------------------------------------------------------
|
||||
| Database Connections
|
||||
|--------------------------------------------------------------------------
|
||||
|
|
||||
| Here are each of the database connections setup for your application.
|
||||
| Of course, examples of configuring each database platform that is
|
||||
| supported by Laravel is shown below to make development simple.
|
||||
|
|
||||
|
|
||||
| All database work in Laravel is done through the PHP PDO facilities
|
||||
| so make sure you have the driver for your particular database of
|
||||
| choice installed on your machine before you begin development.
|
||||
|
|
||||
*/
|
||||
/*
|
||||
|--------------------------------------------------------------------------
|
||||
| Database Connections
|
||||
|--------------------------------------------------------------------------
|
||||
|
|
||||
| Here are each of the database connections setup for your application.
|
||||
| Of course, examples of configuring each database platform that is
|
||||
| supported by Laravel is shown below to make development simple.
|
||||
|
|
||||
|
|
||||
| All database work in Laravel is done through the PHP PDO facilities
|
||||
| so make sure you have the driver for your particular database of
|
||||
| choice installed on your machine before you begin development.
|
||||
|
|
||||
*/
|
||||
|
||||
'connections' => [
|
||||
|
||||
|
@ -80,29 +79,29 @@ return [
|
|||
|
||||
],
|
||||
|
||||
/*
|
||||
|--------------------------------------------------------------------------
|
||||
| Migration Repository Table
|
||||
|--------------------------------------------------------------------------
|
||||
|
|
||||
| This table keeps track of all the migrations that have already run for
|
||||
| your application. Using this information, we can determine which of
|
||||
| the migrations on disk haven't actually been run in the database.
|
||||
|
|
||||
*/
|
||||
/*
|
||||
|--------------------------------------------------------------------------
|
||||
| Migration Repository Table
|
||||
|--------------------------------------------------------------------------
|
||||
|
|
||||
| This table keeps track of all the migrations that have already run for
|
||||
| your application. Using this information, we can determine which of
|
||||
| the migrations on disk haven't actually been run in the database.
|
||||
|
|
||||
*/
|
||||
|
||||
'migrations' => 'migrations',
|
||||
|
||||
/*
|
||||
|--------------------------------------------------------------------------
|
||||
| Redis Databases
|
||||
|--------------------------------------------------------------------------
|
||||
|
|
||||
| Redis is an open source, fast, and advanced key-value store that also
|
||||
| provides a richer set of commands than a typical key-value systems
|
||||
| such as APC or Memcached. Laravel makes it easy to dig right in.
|
||||
|
|
||||
*/
|
||||
/*
|
||||
|--------------------------------------------------------------------------
|
||||
| Redis Databases
|
||||
|--------------------------------------------------------------------------
|
||||
|
|
||||
| Redis is an open source, fast, and advanced key-value store that also
|
||||
| provides a richer set of commands than a typical key-value systems
|
||||
| such as APC or Memcached. Laravel makes it easy to dig right in.
|
||||
|
|
||||
*/
|
||||
|
||||
'redis' => [
|
||||
|
||||
|
|
Reference in a new issue