ifhub
/
ifnews
Archived
1
0
Fork 0

WIP on collection

This commit is contained in:
Alexander Yakovlev 2019-09-13 12:56:52 +07:00
parent d6afa0e6fa
commit 997dd373f5
Signed by: oreolek
GPG Key ID: 1CDC4B7820C93BD3
7 changed files with 199 additions and 172 deletions

View File

@ -28,8 +28,9 @@ class Collect extends Command
* @var array
*/
protected $parsers = [
'Urq',
//'Urq',
'Anivisual',
/*
'Kvester',
//'vndb',
'Apero',
@ -43,7 +44,8 @@ class Collect extends Command
'Dashingdon',
'Itch',
'Gamejolt',
'Steam',
'Steam',
*/
];
/**
@ -56,7 +58,11 @@ class Collect extends Command
foreach ($this->parsers as $parser) {
$cname = 'App\\Sources\\'.$parser;
try {
(new $cname())->parse();
$parser = new $cname();
$games = $parser->parse();
foreach ($games as $game) {
$parser->saveGame($game);
}
} catch (\Exception $e) {
Log::error($e->getMessage());
Log::debug($e->getTraceAsString());

View File

@ -90,7 +90,29 @@ abstract class Source {
/**
* Save the game if not a duplicate.
*/
protected function saveGame(Game $game) {
$game->save();
public function saveGame(Game $game) {
$game->source = self::class;
$dbmodel = NULL;
if (isset($game->source_id)) {
$dbmodel = Game::where('source', $game->source)
->where('source_id', $game->source_id)
->first();
}
if ($dbmodel) {
$dbmodel->fill($game);
$dbmodel->save();
} else {
$game->save();
}
}
/**
* Get the date of the last scraped game for this source.
*/
protected function getLastDate() {
Game::where('source', self::class)
->orderBy('created_at', 'desc')
->limit(1)
->value('created_at');
}
}

View File

@ -41,23 +41,25 @@ class Anivisual extends Source {
$text = $this->get_text('http://anivisual.net/stuff/1');
$this->loadStr($text);
unset($text);
$this->dom->filter('.entryBlock')->each(function($gameBlock) {
$lastDate = $this->getLastDate();
$games = [];
$this->dom->filter('.entryBlock')->each(function($gameBlock) use($lastDate, &$games) {
$date = trim($gameBlock->filter('.icon-calendar')->text());
foreach ($this->months as $ruM => $enM) {
$date = str_replace($ruM, $enM, $date);
}
$date = \DateTime::createFromFormat('d F Y', $date);
if ($date >= $lastDate) {
return;
}
$game->date = $date;
$link = $gameBlock->filter('.novel-ttl a')->first();
$link = 'http://anivisual.net'.$link->attr('href');
$game = $this->page($link);
//$game = new Game;
//$game->title = htmlspecialchars_decode($link->html());
//$game->url = 'http://anivisual.net'.$link->attr('href');
//$game->description = $gameBlock->filter('span')->first()->text();
$this->saveGame($game);
$games[] = $game;
});
return $games;
}
public function checkPage($url) {

View File

@ -22,7 +22,7 @@ use \App\Models\Game;
use \App\Source;
class Urq extends Source {
public $title = "Библиотека URQ";
public $title = "Библиотека URQ";
public function parse() {
$text = $this->get_text('http://urq.plut.info/node/209');
$this->loadStr($text);
@ -32,6 +32,7 @@ class Urq extends Source {
$game->author = trim($gameBlock->filter('.views-field-taxonomy-vocabulary-2')->first()->text());
$game->title = trim($gameBlock->filter('.views-field-title')->text());
$game->url = 'http://urq.plut.info'.trim($gameBlock->filter('.views-field-title a')->attr('href'));
$game->source_id = (int) str_replace('http://urq.plut.info/node/', '', $game->url);
$this->saveGame($game);
});
}

View File

@ -1,59 +1,57 @@
<?php
return [
/*
|--------------------------------------------------------------------------
| Application Name
|--------------------------------------------------------------------------
|
| This value is the name of your application. This value is used when the
| framework needs to place the application's name in a notification or
| any other location as required by the application or its packages.
|
*/
/*
|--------------------------------------------------------------------------
| Application Name
|--------------------------------------------------------------------------
|
| This value is the name of your application. This value is used when the
| framework needs to place the application's name in a notification or
| any other location as required by the application or its packages.
|
*/
'name' => 'Ifnews',
'name' => 'Ifnews',
/*
|--------------------------------------------------------------------------
| Application Version
|--------------------------------------------------------------------------
|
| This value determines the "version" your application is currently running
| in. You may want to follow the "Semantic Versioning" - Given a version
| number MAJOR.MINOR.PATCH when an update happens: https://semver.org.
|
*/
/*
|--------------------------------------------------------------------------
| Application Version
|--------------------------------------------------------------------------
|
| This value determines the "version" your application is currently running
| in. You may want to follow the "Semantic Versioning" - Given a version
| number MAJOR.MINOR.PATCH when an update happens: https://semver.org.
|
*/
'version' => app('git.version'),
'version' => app('git.version'),
/*
|--------------------------------------------------------------------------
| Application Environment
|--------------------------------------------------------------------------
|
| This value determines the "environment" your application is currently
| running in. This may determine how you prefer to configure various
| services your application utilizes. Should be true in production.
|
*/
/*
|--------------------------------------------------------------------------
| Application Environment
|--------------------------------------------------------------------------
|
| This value determines the "environment" your application is currently
| running in. This may determine how you prefer to configure various
| services your application utilizes. Should be true in production.
|
*/
'production' => env('DEBUG'),
'production' => false,
/*
|--------------------------------------------------------------------------
| Autoloaded Service Providers
|--------------------------------------------------------------------------
|
| The service providers listed here will be automatically loaded on the
| request to your application. Feel free to add your own services to
| this array to grant expanded functionality to your applications.
|
*/
'providers' => [
App\Providers\AppServiceProvider::class,
],
/*
|--------------------------------------------------------------------------
| Autoloaded Service Providers
|--------------------------------------------------------------------------
|
| The service providers listed here will be automatically loaded on the
| request to your application. Feel free to add your own services to
| this array to grant expanded functionality to your applications.
|
*/
'providers' => [
App\Providers\AppServiceProvider::class,
],
];

View File

@ -2,79 +2,78 @@
return [
/*
|--------------------------------------------------------------------------
| Default Command
|--------------------------------------------------------------------------
|
| Laravel Zero will always run the command specified below when no command name is
| provided. Consider update the default command for single command applications.
| You cannot pass arguments to the default command because they are ignored.
|
*/
/*
|--------------------------------------------------------------------------
| Default Command
|--------------------------------------------------------------------------
|
| Laravel Zero will always run the command specified below when no command name is
| provided. Consider update the default command for single command applications.
| You cannot pass arguments to the default command because they are ignored.
|
*/
'default' => NunoMaduro\LaravelConsoleSummary\SummaryCommand::class,
'default' => NunoMaduro\LaravelConsoleSummary\SummaryCommand::class,
/*
|--------------------------------------------------------------------------
| Commands Paths
|--------------------------------------------------------------------------
|
| This value determines the "paths" that should be loaded by the console's
| kernel. Foreach "path" present on the array provided below the kernel
| will extract all "Illuminate\Console\Command" based class commands.
|
*/
/*
|--------------------------------------------------------------------------
| Commands Paths
|--------------------------------------------------------------------------
|
| This value determines the "paths" that should be loaded by the console's
| kernel. Foreach "path" present on the array provided below the kernel
| will extract all "Illuminate\Console\Command" based class commands.
|
*/
'paths' => [app_path('Commands')],
'paths' => [app_path('Commands')],
/*
|--------------------------------------------------------------------------
| Added Commands
|--------------------------------------------------------------------------
|
| You may want to include a single command class without having to load an
| entire folder. Here you can specify which commands should be added to
| your list of commands. The console's kernel will try to load them.
|
*/
/*
|--------------------------------------------------------------------------
| Added Commands
|--------------------------------------------------------------------------
|
| You may want to include a single command class without having to load an
| entire folder. Here you can specify which commands should be added to
| your list of commands. The console's kernel will try to load them.
|
*/
'add' => [
// ..
],
'add' => [
// ..
],
/*
|--------------------------------------------------------------------------
| Hidden Commands
|--------------------------------------------------------------------------
|
| Your application commands will always be visible on the application list
| of commands. But you can still make them "hidden" specifying an array
| of commands below. All "hidden" commands can still be run/executed.
|
*/
/*
|--------------------------------------------------------------------------
| Hidden Commands
|--------------------------------------------------------------------------
|
| Your application commands will always be visible on the application list
| of commands. But you can still make them "hidden" specifying an array
| of commands below. All "hidden" commands can still be run/executed.
|
*/
'hidden' => [
NunoMaduro\LaravelConsoleSummary\SummaryCommand::class,
Symfony\Component\Console\Command\HelpCommand::class,
Illuminate\Console\Scheduling\ScheduleRunCommand::class,
Illuminate\Console\Scheduling\ScheduleFinishCommand::class,
Illuminate\Foundation\Console\VendorPublishCommand::class,
],
'hidden' => [
NunoMaduro\LaravelConsoleSummary\SummaryCommand::class,
Symfony\Component\Console\Command\HelpCommand::class,
Illuminate\Console\Scheduling\ScheduleRunCommand::class,
Illuminate\Console\Scheduling\ScheduleFinishCommand::class,
Illuminate\Foundation\Console\VendorPublishCommand::class,
],
/*
|--------------------------------------------------------------------------
| Removed Commands
|--------------------------------------------------------------------------
|
| Do you have a service provider that loads a list of commands that
| you don't need? No problem. Laravel Zero allows you to specify
| below a list of commands that you don't to see in your app.
|
*/
'remove' => [
// ..
],
/*
|--------------------------------------------------------------------------
| Removed Commands
|--------------------------------------------------------------------------
|
| Do you have a service provider that loads a list of commands that
| you don't need? No problem. Laravel Zero allows you to specify
| below a list of commands that you don't to see in your app.
|
*/
'remove' => [
// ..
],
];

View File

@ -1,35 +1,34 @@
<?php
return [
/*
|--------------------------------------------------------------------------
| Default Database Connection Name
|--------------------------------------------------------------------------
|
| Here you may specify which of the database connections below you wish
| to use as your default connection for all database work. Of course
| you may use many connections at once using the Database library.
|
*/
/*
|--------------------------------------------------------------------------
| Default Database Connection Name
|--------------------------------------------------------------------------
|
| Here you may specify which of the database connections below you wish
| to use as your default connection for all database work. Of course
| you may use many connections at once using the Database library.
|
*/
'default' => env('DB_CONNECTION', 'sqlite'),
/*
|--------------------------------------------------------------------------
| Database Connections
|--------------------------------------------------------------------------
|
| Here are each of the database connections setup for your application.
| Of course, examples of configuring each database platform that is
| supported by Laravel is shown below to make development simple.
|
|
| All database work in Laravel is done through the PHP PDO facilities
| so make sure you have the driver for your particular database of
| choice installed on your machine before you begin development.
|
*/
/*
|--------------------------------------------------------------------------
| Database Connections
|--------------------------------------------------------------------------
|
| Here are each of the database connections setup for your application.
| Of course, examples of configuring each database platform that is
| supported by Laravel is shown below to make development simple.
|
|
| All database work in Laravel is done through the PHP PDO facilities
| so make sure you have the driver for your particular database of
| choice installed on your machine before you begin development.
|
*/
'connections' => [
@ -80,29 +79,29 @@ return [
],
/*
|--------------------------------------------------------------------------
| Migration Repository Table
|--------------------------------------------------------------------------
|
| This table keeps track of all the migrations that have already run for
| your application. Using this information, we can determine which of
| the migrations on disk haven't actually been run in the database.
|
*/
/*
|--------------------------------------------------------------------------
| Migration Repository Table
|--------------------------------------------------------------------------
|
| This table keeps track of all the migrations that have already run for
| your application. Using this information, we can determine which of
| the migrations on disk haven't actually been run in the database.
|
*/
'migrations' => 'migrations',
/*
|--------------------------------------------------------------------------
| Redis Databases
|--------------------------------------------------------------------------
|
| Redis is an open source, fast, and advanced key-value store that also
| provides a richer set of commands than a typical key-value systems
| such as APC or Memcached. Laravel makes it easy to dig right in.
|
*/
/*
|--------------------------------------------------------------------------
| Redis Databases
|--------------------------------------------------------------------------
|
| Redis is an open source, fast, and advanced key-value store that also
| provides a richer set of commands than a typical key-value systems
| such as APC or Memcached. Laravel makes it easy to dig right in.
|
*/
'redis' => [