diff --git a/app/Commands/Collect.php b/app/Commands/Collect.php index 66c7d57..4b24bfd 100644 --- a/app/Commands/Collect.php +++ b/app/Commands/Collect.php @@ -28,8 +28,9 @@ class Collect extends Command * @var array */ protected $parsers = [ - 'Urq', + //'Urq', 'Anivisual', + /* 'Kvester', //'vndb', 'Apero', @@ -43,7 +44,8 @@ class Collect extends Command 'Dashingdon', 'Itch', 'Gamejolt', - 'Steam', + 'Steam', + */ ]; /** @@ -56,7 +58,11 @@ class Collect extends Command foreach ($this->parsers as $parser) { $cname = 'App\\Sources\\'.$parser; try { - (new $cname())->parse(); + $parser = new $cname(); + $games = $parser->parse(); + foreach ($games as $game) { + $parser->saveGame($game); + } } catch (\Exception $e) { Log::error($e->getMessage()); Log::debug($e->getTraceAsString()); diff --git a/app/Source.php b/app/Source.php index bb509e0..32af3f4 100644 --- a/app/Source.php +++ b/app/Source.php @@ -90,7 +90,29 @@ abstract class Source { /** * Save the game if not a duplicate. */ - protected function saveGame(Game $game) { - $game->save(); + public function saveGame(Game $game) { + $game->source = self::class; + $dbmodel = NULL; + if (isset($game->source_id)) { + $dbmodel = Game::where('source', $game->source) + ->where('source_id', $game->source_id) + ->first(); + } + if ($dbmodel) { + $dbmodel->fill($game); + $dbmodel->save(); + } else { + $game->save(); + } + } + + /** + * Get the date of the last scraped game for this source. + */ + protected function getLastDate() { + Game::where('source', self::class) + ->orderBy('created_at', 'desc') + ->limit(1) + ->value('created_at'); } } diff --git a/app/Sources/Anivisual.php b/app/Sources/Anivisual.php index 061de40..1344ca5 100644 --- a/app/Sources/Anivisual.php +++ b/app/Sources/Anivisual.php @@ -41,23 +41,25 @@ class Anivisual extends Source { $text = $this->get_text('http://anivisual.net/stuff/1'); $this->loadStr($text); unset($text); - $this->dom->filter('.entryBlock')->each(function($gameBlock) { + $lastDate = $this->getLastDate(); + $games = []; + $this->dom->filter('.entryBlock')->each(function($gameBlock) use($lastDate, &$games) { $date = trim($gameBlock->filter('.icon-calendar')->text()); foreach ($this->months as $ruM => $enM) { $date = str_replace($ruM, $enM, $date); } $date = \DateTime::createFromFormat('d F Y', $date); + if ($date >= $lastDate) { + return; + } $game->date = $date; $link = $gameBlock->filter('.novel-ttl a')->first(); $link = 'http://anivisual.net'.$link->attr('href'); $game = $this->page($link); - //$game = new Game; - //$game->title = htmlspecialchars_decode($link->html()); - //$game->url = 'http://anivisual.net'.$link->attr('href'); - //$game->description = $gameBlock->filter('span')->first()->text(); - $this->saveGame($game); + $games[] = $game; }); + return $games; } public function checkPage($url) { diff --git a/app/Sources/Urq.php b/app/Sources/Urq.php index ac70c94..57a5721 100644 --- a/app/Sources/Urq.php +++ b/app/Sources/Urq.php @@ -22,7 +22,7 @@ use \App\Models\Game; use \App\Source; class Urq extends Source { - public $title = "Библиотека URQ"; + public $title = "Библиотека URQ"; public function parse() { $text = $this->get_text('http://urq.plut.info/node/209'); $this->loadStr($text); @@ -32,6 +32,7 @@ class Urq extends Source { $game->author = trim($gameBlock->filter('.views-field-taxonomy-vocabulary-2')->first()->text()); $game->title = trim($gameBlock->filter('.views-field-title')->text()); $game->url = 'http://urq.plut.info'.trim($gameBlock->filter('.views-field-title a')->attr('href')); + $game->source_id = (int) str_replace('http://urq.plut.info/node/', '', $game->url); $this->saveGame($game); }); } diff --git a/config/app.php b/config/app.php index b5350d6..123b4e0 100644 --- a/config/app.php +++ b/config/app.php @@ -1,59 +1,57 @@ 'Ifnews', - 'name' => 'Ifnews', + /* + |-------------------------------------------------------------------------- + | Application Version + |-------------------------------------------------------------------------- + | + | This value determines the "version" your application is currently running + | in. You may want to follow the "Semantic Versioning" - Given a version + | number MAJOR.MINOR.PATCH when an update happens: https://semver.org. + | + */ - /* - |-------------------------------------------------------------------------- - | Application Version - |-------------------------------------------------------------------------- - | - | This value determines the "version" your application is currently running - | in. You may want to follow the "Semantic Versioning" - Given a version - | number MAJOR.MINOR.PATCH when an update happens: https://semver.org. - | - */ + 'version' => app('git.version'), - 'version' => app('git.version'), + /* + |-------------------------------------------------------------------------- + | Application Environment + |-------------------------------------------------------------------------- + | + | This value determines the "environment" your application is currently + | running in. This may determine how you prefer to configure various + | services your application utilizes. Should be true in production. + | + */ - /* - |-------------------------------------------------------------------------- - | Application Environment - |-------------------------------------------------------------------------- - | - | This value determines the "environment" your application is currently - | running in. This may determine how you prefer to configure various - | services your application utilizes. Should be true in production. - | - */ + 'production' => env('DEBUG'), - 'production' => false, - - /* - |-------------------------------------------------------------------------- - | Autoloaded Service Providers - |-------------------------------------------------------------------------- - | - | The service providers listed here will be automatically loaded on the - | request to your application. Feel free to add your own services to - | this array to grant expanded functionality to your applications. - | - */ - - 'providers' => [ - App\Providers\AppServiceProvider::class, - ], + /* + |-------------------------------------------------------------------------- + | Autoloaded Service Providers + |-------------------------------------------------------------------------- + | + | The service providers listed here will be automatically loaded on the + | request to your application. Feel free to add your own services to + | this array to grant expanded functionality to your applications. + | + */ + 'providers' => [ + App\Providers\AppServiceProvider::class, + ], ]; diff --git a/config/commands.php b/config/commands.php index 838b65f..907bc06 100644 --- a/config/commands.php +++ b/config/commands.php @@ -2,79 +2,78 @@ return [ - /* - |-------------------------------------------------------------------------- - | Default Command - |-------------------------------------------------------------------------- - | - | Laravel Zero will always run the command specified below when no command name is - | provided. Consider update the default command for single command applications. - | You cannot pass arguments to the default command because they are ignored. - | - */ + /* + |-------------------------------------------------------------------------- + | Default Command + |-------------------------------------------------------------------------- + | + | Laravel Zero will always run the command specified below when no command name is + | provided. Consider update the default command for single command applications. + | You cannot pass arguments to the default command because they are ignored. + | + */ - 'default' => NunoMaduro\LaravelConsoleSummary\SummaryCommand::class, + 'default' => NunoMaduro\LaravelConsoleSummary\SummaryCommand::class, - /* - |-------------------------------------------------------------------------- - | Commands Paths - |-------------------------------------------------------------------------- - | - | This value determines the "paths" that should be loaded by the console's - | kernel. Foreach "path" present on the array provided below the kernel - | will extract all "Illuminate\Console\Command" based class commands. - | - */ + /* + |-------------------------------------------------------------------------- + | Commands Paths + |-------------------------------------------------------------------------- + | + | This value determines the "paths" that should be loaded by the console's + | kernel. Foreach "path" present on the array provided below the kernel + | will extract all "Illuminate\Console\Command" based class commands. + | + */ - 'paths' => [app_path('Commands')], + 'paths' => [app_path('Commands')], - /* - |-------------------------------------------------------------------------- - | Added Commands - |-------------------------------------------------------------------------- - | - | You may want to include a single command class without having to load an - | entire folder. Here you can specify which commands should be added to - | your list of commands. The console's kernel will try to load them. - | - */ + /* + |-------------------------------------------------------------------------- + | Added Commands + |-------------------------------------------------------------------------- + | + | You may want to include a single command class without having to load an + | entire folder. Here you can specify which commands should be added to + | your list of commands. The console's kernel will try to load them. + | + */ - 'add' => [ - // .. - ], + 'add' => [ + // .. + ], - /* - |-------------------------------------------------------------------------- - | Hidden Commands - |-------------------------------------------------------------------------- - | - | Your application commands will always be visible on the application list - | of commands. But you can still make them "hidden" specifying an array - | of commands below. All "hidden" commands can still be run/executed. - | - */ + /* + |-------------------------------------------------------------------------- + | Hidden Commands + |-------------------------------------------------------------------------- + | + | Your application commands will always be visible on the application list + | of commands. But you can still make them "hidden" specifying an array + | of commands below. All "hidden" commands can still be run/executed. + | + */ - 'hidden' => [ - NunoMaduro\LaravelConsoleSummary\SummaryCommand::class, - Symfony\Component\Console\Command\HelpCommand::class, - Illuminate\Console\Scheduling\ScheduleRunCommand::class, - Illuminate\Console\Scheduling\ScheduleFinishCommand::class, - Illuminate\Foundation\Console\VendorPublishCommand::class, - ], + 'hidden' => [ + NunoMaduro\LaravelConsoleSummary\SummaryCommand::class, + Symfony\Component\Console\Command\HelpCommand::class, + Illuminate\Console\Scheduling\ScheduleRunCommand::class, + Illuminate\Console\Scheduling\ScheduleFinishCommand::class, + Illuminate\Foundation\Console\VendorPublishCommand::class, + ], - /* - |-------------------------------------------------------------------------- - | Removed Commands - |-------------------------------------------------------------------------- - | - | Do you have a service provider that loads a list of commands that - | you don't need? No problem. Laravel Zero allows you to specify - | below a list of commands that you don't to see in your app. - | - */ - - 'remove' => [ - // .. - ], + /* + |-------------------------------------------------------------------------- + | Removed Commands + |-------------------------------------------------------------------------- + | + | Do you have a service provider that loads a list of commands that + | you don't need? No problem. Laravel Zero allows you to specify + | below a list of commands that you don't to see in your app. + | + */ + 'remove' => [ + // .. + ], ]; diff --git a/config/database.php b/config/database.php index 6f7a4ad..eb0de31 100644 --- a/config/database.php +++ b/config/database.php @@ -1,35 +1,34 @@ env('DB_CONNECTION', 'sqlite'), - /* - |-------------------------------------------------------------------------- - | Database Connections - |-------------------------------------------------------------------------- - | - | Here are each of the database connections setup for your application. - | Of course, examples of configuring each database platform that is - | supported by Laravel is shown below to make development simple. - | - | - | All database work in Laravel is done through the PHP PDO facilities - | so make sure you have the driver for your particular database of - | choice installed on your machine before you begin development. - | - */ + /* + |-------------------------------------------------------------------------- + | Database Connections + |-------------------------------------------------------------------------- + | + | Here are each of the database connections setup for your application. + | Of course, examples of configuring each database platform that is + | supported by Laravel is shown below to make development simple. + | + | + | All database work in Laravel is done through the PHP PDO facilities + | so make sure you have the driver for your particular database of + | choice installed on your machine before you begin development. + | + */ 'connections' => [ @@ -80,29 +79,29 @@ return [ ], - /* - |-------------------------------------------------------------------------- - | Migration Repository Table - |-------------------------------------------------------------------------- - | - | This table keeps track of all the migrations that have already run for - | your application. Using this information, we can determine which of - | the migrations on disk haven't actually been run in the database. - | - */ + /* + |-------------------------------------------------------------------------- + | Migration Repository Table + |-------------------------------------------------------------------------- + | + | This table keeps track of all the migrations that have already run for + | your application. Using this information, we can determine which of + | the migrations on disk haven't actually been run in the database. + | + */ 'migrations' => 'migrations', - /* - |-------------------------------------------------------------------------- - | Redis Databases - |-------------------------------------------------------------------------- - | - | Redis is an open source, fast, and advanced key-value store that also - | provides a richer set of commands than a typical key-value systems - | such as APC or Memcached. Laravel makes it easy to dig right in. - | - */ + /* + |-------------------------------------------------------------------------- + | Redis Databases + |-------------------------------------------------------------------------- + | + | Redis is an open source, fast, and advanced key-value store that also + | provides a richer set of commands than a typical key-value systems + | such as APC or Memcached. Laravel makes it easy to dig right in. + | + */ 'redis' => [