WIP on collection
This commit is contained in:
parent
d6afa0e6fa
commit
997dd373f5
GPG key ID:
1CDC4B7820C93BD3
|
|
@ -28,8 +28,9 @@ class Collect extends Command
|
||||||
* @var array
|
* @var array
|
||||||
*/
|
*/
|
||||||
protected $parsers = [
|
protected $parsers = [
|
||||||
'Urq',
|
//'Urq',
|
||||||
'Anivisual',
|
'Anivisual',
|
||||||
|
/*
|
||||||
'Kvester',
|
'Kvester',
|
||||||
//'vndb',
|
//'vndb',
|
||||||
'Apero',
|
'Apero',
|
||||||
|
|
@ -43,7 +44,8 @@ class Collect extends Command
|
||||||
'Dashingdon',
|
'Dashingdon',
|
||||||
'Itch',
|
'Itch',
|
||||||
'Gamejolt',
|
'Gamejolt',
|
||||||
'Steam',
|
'Steam',
|
||||||
|
*/
|
||||||
];
|
];
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
@ -56,7 +58,11 @@ class Collect extends Command
|
||||||
foreach ($this->parsers as $parser) {
|
foreach ($this->parsers as $parser) {
|
||||||
$cname = 'App\\Sources\\'.$parser;
|
$cname = 'App\\Sources\\'.$parser;
|
||||||
try {
|
try {
|
||||||
(new $cname())->parse();
|
$parser = new $cname();
|
||||||
|
$games = $parser->parse();
|
||||||
|
foreach ($games as $game) {
|
||||||
|
$parser->saveGame($game);
|
||||||
|
}
|
||||||
} catch (\Exception $e) {
|
} catch (\Exception $e) {
|
||||||
Log::error($e->getMessage());
|
Log::error($e->getMessage());
|
||||||
Log::debug($e->getTraceAsString());
|
Log::debug($e->getTraceAsString());
|
||||||
|
|
|
||||||
|
|
@ -90,7 +90,29 @@ abstract class Source {
|
||||||
/**
|
/**
|
||||||
* Save the game if not a duplicate.
|
* Save the game if not a duplicate.
|
||||||
*/
|
*/
|
||||||
protected function saveGame(Game $game) {
|
public function saveGame(Game $game) {
|
||||||
$game->save();
|
$game->source = self::class;
|
||||||
|
$dbmodel = NULL;
|
||||||
|
if (isset($game->source_id)) {
|
||||||
|
$dbmodel = Game::where('source', $game->source)
|
||||||
|
->where('source_id', $game->source_id)
|
||||||
|
->first();
|
||||||
|
}
|
||||||
|
if ($dbmodel) {
|
||||||
|
$dbmodel->fill($game);
|
||||||
|
$dbmodel->save();
|
||||||
|
} else {
|
||||||
|
$game->save();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get the date of the last scraped game for this source.
|
||||||
|
*/
|
||||||
|
protected function getLastDate() {
|
||||||
|
Game::where('source', self::class)
|
||||||
|
->orderBy('created_at', 'desc')
|
||||||
|
->limit(1)
|
||||||
|
->value('created_at');
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -41,23 +41,25 @@ class Anivisual extends Source {
|
||||||
$text = $this->get_text('http://anivisual.net/stuff/1');
|
$text = $this->get_text('http://anivisual.net/stuff/1');
|
||||||
$this->loadStr($text);
|
$this->loadStr($text);
|
||||||
unset($text);
|
unset($text);
|
||||||
$this->dom->filter('.entryBlock')->each(function($gameBlock) {
|
$lastDate = $this->getLastDate();
|
||||||
|
$games = [];
|
||||||
|
$this->dom->filter('.entryBlock')->each(function($gameBlock) use($lastDate, &$games) {
|
||||||
$date = trim($gameBlock->filter('.icon-calendar')->text());
|
$date = trim($gameBlock->filter('.icon-calendar')->text());
|
||||||
foreach ($this->months as $ruM => $enM) {
|
foreach ($this->months as $ruM => $enM) {
|
||||||
$date = str_replace($ruM, $enM, $date);
|
$date = str_replace($ruM, $enM, $date);
|
||||||
}
|
}
|
||||||
$date = \DateTime::createFromFormat('d F Y', $date);
|
$date = \DateTime::createFromFormat('d F Y', $date);
|
||||||
|
if ($date >= $lastDate) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
$game->date = $date;
|
$game->date = $date;
|
||||||
$link = $gameBlock->filter('.novel-ttl a')->first();
|
$link = $gameBlock->filter('.novel-ttl a')->first();
|
||||||
$link = 'http://anivisual.net'.$link->attr('href');
|
$link = 'http://anivisual.net'.$link->attr('href');
|
||||||
$game = $this->page($link);
|
$game = $this->page($link);
|
||||||
//$game = new Game;
|
|
||||||
//$game->title = htmlspecialchars_decode($link->html());
|
|
||||||
//$game->url = 'http://anivisual.net'.$link->attr('href');
|
|
||||||
//$game->description = $gameBlock->filter('span')->first()->text();
|
|
||||||
|
|
||||||
$this->saveGame($game);
|
$games[] = $game;
|
||||||
});
|
});
|
||||||
|
return $games;
|
||||||
}
|
}
|
||||||
|
|
||||||
public function checkPage($url) {
|
public function checkPage($url) {
|
||||||
|
|
|
||||||
|
|
@ -22,7 +22,7 @@ use \App\Models\Game;
|
||||||
use \App\Source;
|
use \App\Source;
|
||||||
|
|
||||||
class Urq extends Source {
|
class Urq extends Source {
|
||||||
public $title = "Библиотека URQ";
|
public $title = "Библиотека URQ";
|
||||||
public function parse() {
|
public function parse() {
|
||||||
$text = $this->get_text('http://urq.plut.info/node/209');
|
$text = $this->get_text('http://urq.plut.info/node/209');
|
||||||
$this->loadStr($text);
|
$this->loadStr($text);
|
||||||
|
|
@ -32,6 +32,7 @@ class Urq extends Source {
|
||||||
$game->author = trim($gameBlock->filter('.views-field-taxonomy-vocabulary-2')->first()->text());
|
$game->author = trim($gameBlock->filter('.views-field-taxonomy-vocabulary-2')->first()->text());
|
||||||
$game->title = trim($gameBlock->filter('.views-field-title')->text());
|
$game->title = trim($gameBlock->filter('.views-field-title')->text());
|
||||||
$game->url = 'http://urq.plut.info'.trim($gameBlock->filter('.views-field-title a')->attr('href'));
|
$game->url = 'http://urq.plut.info'.trim($gameBlock->filter('.views-field-title a')->attr('href'));
|
||||||
|
$game->source_id = (int) str_replace('http://urq.plut.info/node/', '', $game->url);
|
||||||
$this->saveGame($game);
|
$this->saveGame($game);
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -1,59 +1,57 @@
|
||||||
<?php
|
<?php
|
||||||
|
|
||||||
return [
|
return [
|
||||||
|
/*
|
||||||
|
|--------------------------------------------------------------------------
|
||||||
|
| Application Name
|
||||||
|
|--------------------------------------------------------------------------
|
||||||
|
|
|
||||||
|
| This value is the name of your application. This value is used when the
|
||||||
|
| framework needs to place the application's name in a notification or
|
||||||
|
| any other location as required by the application or its packages.
|
||||||
|
|
|
||||||
|
*/
|
||||||
|
|
||||||
/*
|
'name' => 'Ifnews',
|
||||||
|--------------------------------------------------------------------------
|
|
||||||
| Application Name
|
|
||||||
|--------------------------------------------------------------------------
|
|
||||||
|
|
|
||||||
| This value is the name of your application. This value is used when the
|
|
||||||
| framework needs to place the application's name in a notification or
|
|
||||||
| any other location as required by the application or its packages.
|
|
||||||
|
|
|
||||||
*/
|
|
||||||
|
|
||||||
'name' => 'Ifnews',
|
/*
|
||||||
|
|--------------------------------------------------------------------------
|
||||||
|
| Application Version
|
||||||
|
|--------------------------------------------------------------------------
|
||||||
|
|
|
||||||
|
| This value determines the "version" your application is currently running
|
||||||
|
| in. You may want to follow the "Semantic Versioning" - Given a version
|
||||||
|
| number MAJOR.MINOR.PATCH when an update happens: https://semver.org.
|
||||||
|
|
|
||||||
|
*/
|
||||||
|
|
||||||
/*
|
'version' => app('git.version'),
|
||||||
|--------------------------------------------------------------------------
|
|
||||||
| Application Version
|
|
||||||
|--------------------------------------------------------------------------
|
|
||||||
|
|
|
||||||
| This value determines the "version" your application is currently running
|
|
||||||
| in. You may want to follow the "Semantic Versioning" - Given a version
|
|
||||||
| number MAJOR.MINOR.PATCH when an update happens: https://semver.org.
|
|
||||||
|
|
|
||||||
*/
|
|
||||||
|
|
||||||
'version' => app('git.version'),
|
/*
|
||||||
|
|--------------------------------------------------------------------------
|
||||||
|
| Application Environment
|
||||||
|
|--------------------------------------------------------------------------
|
||||||
|
|
|
||||||
|
| This value determines the "environment" your application is currently
|
||||||
|
| running in. This may determine how you prefer to configure various
|
||||||
|
| services your application utilizes. Should be true in production.
|
||||||
|
|
|
||||||
|
*/
|
||||||
|
|
||||||
/*
|
'production' => env('DEBUG'),
|
||||||
|--------------------------------------------------------------------------
|
|
||||||
| Application Environment
|
|
||||||
|--------------------------------------------------------------------------
|
|
||||||
|
|
|
||||||
| This value determines the "environment" your application is currently
|
|
||||||
| running in. This may determine how you prefer to configure various
|
|
||||||
| services your application utilizes. Should be true in production.
|
|
||||||
|
|
|
||||||
*/
|
|
||||||
|
|
||||||
'production' => false,
|
/*
|
||||||
|
|--------------------------------------------------------------------------
|
||||||
/*
|
| Autoloaded Service Providers
|
||||||
|--------------------------------------------------------------------------
|
|--------------------------------------------------------------------------
|
||||||
| Autoloaded Service Providers
|
|
|
||||||
|--------------------------------------------------------------------------
|
| The service providers listed here will be automatically loaded on the
|
||||||
|
|
| request to your application. Feel free to add your own services to
|
||||||
| The service providers listed here will be automatically loaded on the
|
| this array to grant expanded functionality to your applications.
|
||||||
| request to your application. Feel free to add your own services to
|
|
|
||||||
| this array to grant expanded functionality to your applications.
|
*/
|
||||||
|
|
|
||||||
*/
|
|
||||||
|
|
||||||
'providers' => [
|
|
||||||
App\Providers\AppServiceProvider::class,
|
|
||||||
],
|
|
||||||
|
|
||||||
|
'providers' => [
|
||||||
|
App\Providers\AppServiceProvider::class,
|
||||||
|
],
|
||||||
];
|
];
|
||||||
|
|
|
||||||
|
|
@ -2,79 +2,78 @@
|
||||||
|
|
||||||
return [
|
return [
|
||||||
|
|
||||||
/*
|
/*
|
||||||
|--------------------------------------------------------------------------
|
|--------------------------------------------------------------------------
|
||||||
| Default Command
|
| Default Command
|
||||||
|--------------------------------------------------------------------------
|
|--------------------------------------------------------------------------
|
||||||
|
|
|
|
||||||
| Laravel Zero will always run the command specified below when no command name is
|
| Laravel Zero will always run the command specified below when no command name is
|
||||||
| provided. Consider update the default command for single command applications.
|
| provided. Consider update the default command for single command applications.
|
||||||
| You cannot pass arguments to the default command because they are ignored.
|
| You cannot pass arguments to the default command because they are ignored.
|
||||||
|
|
|
|
||||||
*/
|
*/
|
||||||
|
|
||||||
'default' => NunoMaduro\LaravelConsoleSummary\SummaryCommand::class,
|
'default' => NunoMaduro\LaravelConsoleSummary\SummaryCommand::class,
|
||||||
|
|
||||||
/*
|
/*
|
||||||
|--------------------------------------------------------------------------
|
|--------------------------------------------------------------------------
|
||||||
| Commands Paths
|
| Commands Paths
|
||||||
|--------------------------------------------------------------------------
|
|--------------------------------------------------------------------------
|
||||||
|
|
|
|
||||||
| This value determines the "paths" that should be loaded by the console's
|
| This value determines the "paths" that should be loaded by the console's
|
||||||
| kernel. Foreach "path" present on the array provided below the kernel
|
| kernel. Foreach "path" present on the array provided below the kernel
|
||||||
| will extract all "Illuminate\Console\Command" based class commands.
|
| will extract all "Illuminate\Console\Command" based class commands.
|
||||||
|
|
|
|
||||||
*/
|
*/
|
||||||
|
|
||||||
'paths' => [app_path('Commands')],
|
'paths' => [app_path('Commands')],
|
||||||
|
|
||||||
/*
|
/*
|
||||||
|--------------------------------------------------------------------------
|
|--------------------------------------------------------------------------
|
||||||
| Added Commands
|
| Added Commands
|
||||||
|--------------------------------------------------------------------------
|
|--------------------------------------------------------------------------
|
||||||
|
|
|
|
||||||
| You may want to include a single command class without having to load an
|
| You may want to include a single command class without having to load an
|
||||||
| entire folder. Here you can specify which commands should be added to
|
| entire folder. Here you can specify which commands should be added to
|
||||||
| your list of commands. The console's kernel will try to load them.
|
| your list of commands. The console's kernel will try to load them.
|
||||||
|
|
|
|
||||||
*/
|
*/
|
||||||
|
|
||||||
'add' => [
|
'add' => [
|
||||||
// ..
|
// ..
|
||||||
],
|
],
|
||||||
|
|
||||||
/*
|
/*
|
||||||
|--------------------------------------------------------------------------
|
|--------------------------------------------------------------------------
|
||||||
| Hidden Commands
|
| Hidden Commands
|
||||||
|--------------------------------------------------------------------------
|
|--------------------------------------------------------------------------
|
||||||
|
|
|
|
||||||
| Your application commands will always be visible on the application list
|
| Your application commands will always be visible on the application list
|
||||||
| of commands. But you can still make them "hidden" specifying an array
|
| of commands. But you can still make them "hidden" specifying an array
|
||||||
| of commands below. All "hidden" commands can still be run/executed.
|
| of commands below. All "hidden" commands can still be run/executed.
|
||||||
|
|
|
|
||||||
*/
|
*/
|
||||||
|
|
||||||
'hidden' => [
|
'hidden' => [
|
||||||
NunoMaduro\LaravelConsoleSummary\SummaryCommand::class,
|
NunoMaduro\LaravelConsoleSummary\SummaryCommand::class,
|
||||||
Symfony\Component\Console\Command\HelpCommand::class,
|
Symfony\Component\Console\Command\HelpCommand::class,
|
||||||
Illuminate\Console\Scheduling\ScheduleRunCommand::class,
|
Illuminate\Console\Scheduling\ScheduleRunCommand::class,
|
||||||
Illuminate\Console\Scheduling\ScheduleFinishCommand::class,
|
Illuminate\Console\Scheduling\ScheduleFinishCommand::class,
|
||||||
Illuminate\Foundation\Console\VendorPublishCommand::class,
|
Illuminate\Foundation\Console\VendorPublishCommand::class,
|
||||||
],
|
],
|
||||||
|
|
||||||
/*
|
/*
|
||||||
|--------------------------------------------------------------------------
|
|--------------------------------------------------------------------------
|
||||||
| Removed Commands
|
| Removed Commands
|
||||||
|--------------------------------------------------------------------------
|
|--------------------------------------------------------------------------
|
||||||
|
|
|
|
||||||
| Do you have a service provider that loads a list of commands that
|
| Do you have a service provider that loads a list of commands that
|
||||||
| you don't need? No problem. Laravel Zero allows you to specify
|
| you don't need? No problem. Laravel Zero allows you to specify
|
||||||
| below a list of commands that you don't to see in your app.
|
| below a list of commands that you don't to see in your app.
|
||||||
|
|
|
|
||||||
*/
|
*/
|
||||||
|
|
||||||
'remove' => [
|
|
||||||
// ..
|
|
||||||
],
|
|
||||||
|
|
||||||
|
'remove' => [
|
||||||
|
// ..
|
||||||
|
],
|
||||||
];
|
];
|
||||||
|
|
|
||||||
|
|
@ -1,35 +1,34 @@
|
||||||
<?php
|
<?php
|
||||||
|
|
||||||
return [
|
return [
|
||||||
|
/*
|
||||||
/*
|
|--------------------------------------------------------------------------
|
||||||
|--------------------------------------------------------------------------
|
| Default Database Connection Name
|
||||||
| Default Database Connection Name
|
|--------------------------------------------------------------------------
|
||||||
|--------------------------------------------------------------------------
|
|
|
||||||
|
|
| Here you may specify which of the database connections below you wish
|
||||||
| Here you may specify which of the database connections below you wish
|
| to use as your default connection for all database work. Of course
|
||||||
| to use as your default connection for all database work. Of course
|
| you may use many connections at once using the Database library.
|
||||||
| you may use many connections at once using the Database library.
|
|
|
||||||
|
|
*/
|
||||||
*/
|
|
||||||
|
|
||||||
'default' => env('DB_CONNECTION', 'sqlite'),
|
'default' => env('DB_CONNECTION', 'sqlite'),
|
||||||
|
|
||||||
/*
|
/*
|
||||||
|--------------------------------------------------------------------------
|
|--------------------------------------------------------------------------
|
||||||
| Database Connections
|
| Database Connections
|
||||||
|--------------------------------------------------------------------------
|
|--------------------------------------------------------------------------
|
||||||
|
|
|
|
||||||
| Here are each of the database connections setup for your application.
|
| Here are each of the database connections setup for your application.
|
||||||
| Of course, examples of configuring each database platform that is
|
| Of course, examples of configuring each database platform that is
|
||||||
| supported by Laravel is shown below to make development simple.
|
| supported by Laravel is shown below to make development simple.
|
||||||
|
|
|
|
||||||
|
|
|
|
||||||
| All database work in Laravel is done through the PHP PDO facilities
|
| All database work in Laravel is done through the PHP PDO facilities
|
||||||
| so make sure you have the driver for your particular database of
|
| so make sure you have the driver for your particular database of
|
||||||
| choice installed on your machine before you begin development.
|
| choice installed on your machine before you begin development.
|
||||||
|
|
|
|
||||||
*/
|
*/
|
||||||
|
|
||||||
'connections' => [
|
'connections' => [
|
||||||
|
|
||||||
|
|
@ -80,29 +79,29 @@ return [
|
||||||
|
|
||||||
],
|
],
|
||||||
|
|
||||||
/*
|
/*
|
||||||
|--------------------------------------------------------------------------
|
|--------------------------------------------------------------------------
|
||||||
| Migration Repository Table
|
| Migration Repository Table
|
||||||
|--------------------------------------------------------------------------
|
|--------------------------------------------------------------------------
|
||||||
|
|
|
|
||||||
| This table keeps track of all the migrations that have already run for
|
| This table keeps track of all the migrations that have already run for
|
||||||
| your application. Using this information, we can determine which of
|
| your application. Using this information, we can determine which of
|
||||||
| the migrations on disk haven't actually been run in the database.
|
| the migrations on disk haven't actually been run in the database.
|
||||||
|
|
|
|
||||||
*/
|
*/
|
||||||
|
|
||||||
'migrations' => 'migrations',
|
'migrations' => 'migrations',
|
||||||
|
|
||||||
/*
|
/*
|
||||||
|--------------------------------------------------------------------------
|
|--------------------------------------------------------------------------
|
||||||
| Redis Databases
|
| Redis Databases
|
||||||
|--------------------------------------------------------------------------
|
|--------------------------------------------------------------------------
|
||||||
|
|
|
|
||||||
| Redis is an open source, fast, and advanced key-value store that also
|
| Redis is an open source, fast, and advanced key-value store that also
|
||||||
| provides a richer set of commands than a typical key-value systems
|
| provides a richer set of commands than a typical key-value systems
|
||||||
| such as APC or Memcached. Laravel makes it easy to dig right in.
|
| such as APC or Memcached. Laravel makes it easy to dig right in.
|
||||||
|
|
|
|
||||||
*/
|
*/
|
||||||
|
|
||||||
'redis' => [
|
'redis' => [
|
||||||
|
|
||||||
|
|
|
||||||
Reference in a new issue