1
0
Fork 0

Vporn WIP

This commit is contained in:
Alexander Yakovlev 2016-04-22 16:10:24 +07:00
parent 9c61c66cb7
commit 2864c7f9a9
12 changed files with 492 additions and 51 deletions

View File

@ -5,33 +5,17 @@
"homepage": "http://www.yiiframework.com/",
"type": "project",
"license": "BSD-3-Clause",
"support": {
"issues": "https://github.com/yiisoft/yii2/issues?state=open",
"forum": "http://www.yiiframework.com/forum/",
"wiki": "http://www.yiiframework.com/wiki/",
"irc": "irc://irc.freenode.net/yii",
"source": "https://github.com/yiisoft/yii2"
},
"minimum-stability": "stable",
"require": {
"php": ">=5.4.0",
"yiisoft/yii2": ">=2.0.6",
"yiisoft/yii2-bootstrap": "*",
"yiisoft/yii2-swiftmailer": "*"
"yiisoft/yii2": ">=2.0.6"
},
"require-dev": {
"yiisoft/yii2-codeception": "*",
"yiisoft/yii2-debug": "*",
"yiisoft/yii2-gii": "*",
"yiisoft/yii2-faker": "*"
},
"config": {
"process-timeout": 1800
},
"extra": {
"asset-installer-paths": {
"npm-asset-library": "vendor/npm",
"bower-asset-library": "vendor/bower"
}
}
}

50
composer.lock generated
View File

@ -4,8 +4,56 @@
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#composer-lock-the-lock-file",
"This file is @generated automatically"
],
"hash": "8580bd82955b1fbb80d47024e184056e",
"hash": "123907092d7a4d2ed90b2ab034148610",
"packages": [
{
"name": "arogachev/yii2-many-to-many",
"version": "0.2.1",
"source": {
"type": "git",
"url": "https://github.com/arogachev/yii2-many-to-many.git",
"reference": "0c277acbfabf7675aeec213dd56e8232b5bc73cf"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/arogachev/yii2-many-to-many/zipball/0c277acbfabf7675aeec213dd56e8232b5bc73cf",
"reference": "0c277acbfabf7675aeec213dd56e8232b5bc73cf",
"shasum": ""
},
"require": {
"yiisoft/yii2": "*"
},
"require-dev": {
"phpunit/dbunit": "~1.4",
"phpunit/phpunit": "~5.2"
},
"type": "yii2-extension",
"autoload": {
"psr-4": {
"arogachev\\ManyToMany\\": ""
}
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"BSD-3-Clause"
],
"authors": [
{
"name": "Alexey Rogachev",
"email": "arogachev90@gmail.com"
}
],
"description": "Many-to-many ActiveRecord relation for Yii 2 framework",
"homepage": "https://github.com/arogachev/yii2-many-to-many",
"keywords": [
"Behavior",
"many-to-many",
"relation",
"validator",
"yii2"
],
"time": "2016-03-04 10:40:29"
},
{
"name": "bower-asset/bootstrap",
"version": "v3.3.5",

View File

@ -0,0 +1,15 @@
<?php
namespace console\controllers;
use Yii;
use yii\console\Controller;
use yii\helpers\Console;
use common\models\Video;
use app\helpers\parsers\Vporn;
class VideoController extends Controller {
public function actionParse() {
$vporn = new Vporn([]);
$vporn->run();
}
}

43
console/helpers/Curl.php Normal file
View File

@ -0,0 +1,43 @@
<?php
namespace app\helpers;
class Curl {
/**
* cURL download helper.
**/
public static function download_file($url)
{
$ch = curl_init();
$timeout = 5;
$filename = basename($url);
mkdir(\Yii::$app->basePath . '/uploads/'.date('Y').'/'.date('m'), 770, true);
$filename = \Yii::$app->basePath . '/uploads/'.date('Y').'/'.date('m').'/'.$filename;
echo $filename;
$fp = fopen ($filename, 'w+');
curl_setopt($ch, CURLOPT_URL, $url);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 15);
curl_setopt($ch, CURLOPT_FILE, $fp);
curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true);
curl_exec($ch);
curl_close($ch);
fclose($fp);
return $filename;
}
/**
* Downloads $size bytes of a large $url file
**/
public static function download_file_header($url, $size = 1024*15)
{
$rh = fopen($file_source, 'rb');
if (!$rh)
return false;
$data = fread($rh, $size);
if ($data === FALSE)
return false;
fclose($rh);
return $data;
}
}

View File

@ -0,0 +1,23 @@
<?php
namespace app\helpers;
/**
* Abstract Parser class.
* A parser connects to a video hosting site, gets the latest videos and their
* metadata and dumps it to database.
* For simplicity's sake, it also downloads them.
**/
abstract class Parser {
public $options;
public function __construct($options = [])
{
$this->options = $options;
}
abstract public function run();
protected function get_class($domDocument, $classname)
{
$finder = new DomXPath($domDocument);
return $finder->query("//*[contains(@class, '$classname')]");
}
}

View File

@ -0,0 +1,57 @@
<?php
namespace app\helpers\parsers;
use app\helpers\Parser;
use app\models\Video;
use app\models\Picture;
use app\models\Tag;
use app\models\Category;
use DomDocument;
class Vporn extends Parser {
public function run() {
$url = 'http://www.vporn.com/1080p/faye-runaway-is-a-flexible-girl-with-a-wet-cunt/1213542/';
libxml_use_internal_errors(true);
$html = DOMDocument::loadHTML(file_get_contents($url));
$video = new Video;
$picture = new Picture();
$picture->download($this->get_picture_src($html));
$picture->create();
$video->picture_id = $picture->id;
$video->original_url = $this->get_video_src($html);
$video->original_id = $this->get_id($url, $html);
$video->create();
}
protected function get_id($url, $html)
{
preg_match('/\/(\d+)\//', $url, $matches);
return $matches[1];
}
protected function get_picture_src($html)
{
$video_player = $html->getElementById('video_player');
$children = $video_player->childNodes;
$html = '';
foreach ($children as $child) {
/*
if ($child instanceof DOMElement && $child->hasAttribute('src'))
{
return $child->getAttribute('src');
}
*/ // <-- for some reason doesn't work
$html .= $child->ownerDocument->saveXML( $child );
}
preg_match('/img\s*src\s*=\s*[\'"](.*)[\'"]\s*width/', $html, $matches);
return $matches[1];
}
protected function get_video_src($html)
{
preg_match('/bestsaveurl\s*=\s*[\'"](.+)[\'"];/', $html->saveHTML(), $matches);
if (empty($matches))
return FALSE;
return $matches[1];
}
}

View File

@ -1,33 +0,0 @@
<?php
use yii\db\Migration;
class m130524_201442_init extends Migration
{
public function up()
{
$tableOptions = null;
if ($this->db->driverName === 'mysql') {
// http://stackoverflow.com/questions/766809/whats-the-difference-between-utf8-general-ci-and-utf8-unicode-ci
$tableOptions = 'CHARACTER SET utf8 COLLATE utf8_unicode_ci ENGINE=InnoDB';
}
$this->createTable('{{%user}}', [
'id' => $this->primaryKey(),
'username' => $this->string()->notNull()->unique(),
'auth_key' => $this->string(32)->notNull(),
'password_hash' => $this->string()->notNull(),
'password_reset_token' => $this->string()->unique(),
'email' => $this->string()->notNull()->unique(),
'status' => $this->smallInteger()->notNull()->defaultValue(10),
'created_at' => $this->integer()->notNull(),
'updated_at' => $this->integer()->notNull(),
], $tableOptions);
}
public function down()
{
$this->dropTable('{{%user}}');
}
}

View File

@ -0,0 +1,55 @@
<?php
use yii\db\Migration;
class m160422_040239_init extends Migration
{
public function safeUp()
{
$this->createTable('{{%categories}}', [
'id' => $this->primaryKey(),
'name' => $this->string(),
]);
$this->createTable('{{%tags}}', [
'id' => $this->primaryKey(),
'name' => $this->string(),
]);
$this->createTable('{{%pictures}}', [
'id' => $this->primaryKey(),
'filename' => $this->string(),
]);
$this->createTable('{{%videos}}', [
'id' => $this->primaryKey(),
'name' => $this->string(),
'picture_id' => $this->integer(),
'filename' => $this->string(),
'original_url' => $this->string(),
'original_id' => $this->integer(),
]);
$this->createTable('{{%videos_categories}}', [
'id' => $this->primaryKey(),
'video_id' => $this->integer(),
'category_id' => $this->integer(),
]);
$this->createTable('{{%videos_tags}}', [
'id' => $this->primaryKey(),
'video_id' => $this->integer(),
'tag_id' => $this->integer(),
]);
$this->addForeignKey('fk_videos_pictures', '{{%videos}}', 'picture_id', '{{%pictures}}', 'id');
$this->addForeignKey('fk_videos_categories_to_videos', '{{%videos_categories}}', 'video_id', '{{%videos}}', 'id');
$this->addForeignKey('fk_videos_categories_to_categories', '{{%videos_categories}}', 'category_id', '{{%categories}}', 'id');
$this->addForeignKey('fk_videos_tags_to_videos', '{{%videos_tags}}', 'video_id', '{{%videos}}', 'id');
$this->addForeignKey('fk_videos_categories_to_tags', '{{%videos_tags}}', 'tag_id', '{{%tags}}', 'id');
}
public function safeDown()
{
$this->dropTable('{{%videos_tags}}');
$this->dropTable('{{%videos_categories}}');
$this->dropTable('{{%videos}}');
$this->dropTable('{{%categories}}');
$this->dropTable('{{%pictures}}');
$this->dropTable('{{%tags}}');
}
}

View File

@ -0,0 +1,54 @@
<?php
namespace app\models;
use Yii;
/**
* This is the model class for table "categories".
*
* @property integer $id
* @property string $name
*
* @property VideosCategories[] $videosCategories
*/
class Category extends \yii\db\ActiveRecord
{
/**
* @inheritdoc
*/
public static function tableName()
{
return 'categories';
}
/**
* @inheritdoc
*/
public function rules()
{
return [
[['name'], 'string', 'max' => 255],
];
}
/**
* @inheritdoc
*/
public function attributeLabels()
{
return [
'id' => 'ID',
'name' => 'Name',
];
}
/**
* @return \yii\db\ActiveQuery
*/
public function getVideos()
{
return $this->hasMany(Video::className(), ['id' => 'video_id'])
->viaTable('videos_categories', ['video_id' => 'id']);
}
}

View File

@ -0,0 +1,58 @@
<?php
namespace app\models;
use Yii;
use app\helpers\Curl;
/**
* This is the model class for table "pictures".
*
* @property integer $id
* @property string $filename
*
* @property Videos[] $videos
*/
class Picture extends \yii\db\ActiveRecord
{
/**
* @inheritdoc
*/
public static function tableName()
{
return 'pictures';
}
/**
* @inheritdoc
*/
public function rules()
{
return [
[['filename'], 'string', 'max' => 255],
];
}
/**
* @inheritdoc
*/
public function attributeLabels()
{
return [
'id' => 'ID',
'filename' => 'Filename',
];
}
/**
* @return \yii\db\ActiveQuery
*/
public function getVideos()
{
return $this->hasMany(Video::className(), ['picture_id' => 'id']);
}
public function download ($url) {
$this->filename = Curl::download_file($url);
}
}

54
console/models/Tag.php Normal file
View File

@ -0,0 +1,54 @@
<?php
namespace app\models;
use Yii;
/**
* This is the model class for table "tags".
*
* @property integer $id
* @property string $name
*
* @property VideosTags[] $videosTags
*/
class Tag extends \yii\db\ActiveRecord
{
/**
* @inheritdoc
*/
public static function tableName()
{
return 'tags';
}
/**
* @inheritdoc
*/
public function rules()
{
return [
[['name'], 'string', 'max' => 255],
];
}
/**
* @inheritdoc
*/
public function attributeLabels()
{
return [
'id' => 'ID',
'name' => 'Name',
];
}
/**
* @return \yii\db\ActiveQuery
*/
public function getVideos()
{
return $this->hasMany(Video::className(), ['id' => 'video_id'])
->viaTable('videos_tags', ['video_id' => 'id']);
}
}

83
console/models/Video.php Normal file
View File

@ -0,0 +1,83 @@
<?php
namespace app\models;
use Yii;
/**
* This is the model class for table "videos".
*
* @property integer $id
* @property string $name
* @property integer $picture_id
* @property string $filename
* @property string $original_url
* @property integer $original_id
*
* @property Pictures $picture
* @property VideosCategories[] $videosCategories
* @property VideosTags[] $videosTags
*/
class Video extends \yii\db\ActiveRecord
{
/**
* @inheritdoc
*/
public static function tableName()
{
return 'videos';
}
/**
* @inheritdoc
*/
public function rules()
{
return [
[['picture_id', 'original_id'], 'integer'],
[['name', 'filename', 'original_url'], 'string', 'max' => 255],
[['picture_id'], 'exist', 'skipOnError' => true, 'targetClass' => Pictures::className(), 'targetAttribute' => ['picture_id' => 'id']],
];
}
/**
* @inheritdoc
*/
public function attributeLabels()
{
return [
'id' => 'ID',
'name' => 'Name',
'picture_id' => 'Picture ID',
'filename' => 'Filename',
'original_url' => 'Original Url',
'original_id' => 'Original ID',
];
}
/**
* @return \yii\db\ActiveQuery
*/
public function getPicture()
{
return $this->hasOne(Pictures::className(), ['id' => 'picture_id']);
}
/**
* @return \yii\db\ActiveQuery
*/
public function getCategories()
{
return $this->hasMany(Category::className(), ['id' => 'category_id'])
->viaTable('videos_categories', ['category_id' => 'id']);
}
/**
* @return \yii\db\ActiveQuery
*/
public function getTags()
{
return $this->hasMany(Tag::className(), ['id' => 'tag_id'])
->viaTable('videos_tags', ['tag_id' => 'id']);
}
}