WIP on collection

app/Commands/Collect.php

@@ -28,8 +28,9 @@ class Collect extends Command
 	 * @var array
 	 */
 	protected $parsers = [
-    'Urq',
+		//'Urq',
     'Anivisual',
+		/*
     'Kvester',
     //'vndb',
     'Apero',
@@ -44,6 +45,7 @@ class Collect extends Command
     'Itch',
     'Gamejolt',
 		'Steam',
+		 */
 	];
 
 	/**
@@ -56,7 +58,11 @@ class Collect extends Command
 		foreach ($this->parsers as $parser) {
 			$cname = 'App\\Sources\\'.$parser;
 			try {
-				(new $cname())->parse();
+				$parser = new $cname();
+				$games = $parser->parse();
+				foreach ($games as $game) {
+					$parser->saveGame($game);
+				}
 			} catch (\Exception $e) {
 				Log::error($e->getMessage());
 				Log::debug($e->getTraceAsString());

app/Source.php

@@ -90,7 +90,29 @@ abstract class Source {
 	/**
 	 * Save the game if not a duplicate.
 	 */
-	protected function saveGame(Game $game) {
+	public function saveGame(Game $game) {
+		$game->source = self::class;
+		$dbmodel = NULL;
+		if (isset($game->source_id)) {
+			$dbmodel = Game::where('source', $game->source)
+				->where('source_id', $game->source_id)
+				->first();
+		}
+		if ($dbmodel) {
+			$dbmodel->fill($game);
+			$dbmodel->save();
+		} else {
 			$game->save();
 		}
+	}
+
+	/**
+	 * Get the date of the last scraped game for this source.
+	 */
+	protected function getLastDate() {
+		Game::where('source', self::class)
+			->orderBy('created_at', 'desc')
+			->limit(1)
+			->value('created_at');
+	}
 }

app/Sources/Anivisual.php

@@ -41,23 +41,25 @@ class Anivisual extends Source {
 		$text = $this->get_text('http://anivisual.net/stuff/1');
 		$this->loadStr($text);
 		unset($text);
-		$this->dom->filter('.entryBlock')->each(function($gameBlock) {
+		$lastDate = $this->getLastDate();
+		$games = [];
+		$this->dom->filter('.entryBlock')->each(function($gameBlock) use($lastDate, &$games) {
 			$date = trim($gameBlock->filter('.icon-calendar')->text());
 			foreach ($this->months as $ruM => $enM) {
 				$date = str_replace($ruM, $enM, $date);
 			}
 			$date = \DateTime::createFromFormat('d F Y', $date);
+			if ($date >= $lastDate) {
+				return;
+			}
 			$game->date = $date;
 			$link = $gameBlock->filter('.novel-ttl a')->first();
 			$link = 'http://anivisual.net'.$link->attr('href');
 			$game = $this->page($link);
-			//$game = new Game;
-			//$game->title = htmlspecialchars_decode($link->html());
-			//$game->url = 'http://anivisual.net'.$link->attr('href');
-			//$game->description = $gameBlock->filter('span')->first()->text();
 
-			$this->saveGame($game);
+			$games[] = $game;
 		});
+		return $games;
 	}
 
 	public function checkPage($url) {

app/Sources/Urq.php

@@ -32,6 +32,7 @@ class Urq extends Source {
       $game->author = trim($gameBlock->filter('.views-field-taxonomy-vocabulary-2')->first()->text());
       $game->title = trim($gameBlock->filter('.views-field-title')->text());
 			$game->url = 'http://urq.plut.info'.trim($gameBlock->filter('.views-field-title a')->attr('href'));
+			$game->source_id = (int) str_replace('http://urq.plut.info/node/', '', $game->url);
 			$this->saveGame($game);
     });
   }

config/app.php

@@ -1,7 +1,6 @@
 <?php
 
 return [
-
 	/*
 	|--------------------------------------------------------------------------
 	| Application Name
@@ -39,7 +38,7 @@ return [
 	|
 	*/
 
-    'production' => false,
+	'production' => env('DEBUG'),
 
 	/*
 	|--------------------------------------------------------------------------
@@ -55,5 +54,4 @@ return [
 	'providers' => [
 			App\Providers\AppServiceProvider::class,
 	],
-
 ];

config/commands.php

@@ -76,5 +76,4 @@ return [
 	'remove' => [
 			// ..
 	],
-
 ];

config/database.php

@@ -1,7 +1,6 @@
 <?php
 
 return [
-
 	/*
 	|--------------------------------------------------------------------------
 	| Default Database Connection Name