Skip to content

Alto / rkn

Go to a project
Toggle navigation
Toggle navigation pinning
Commit 36888283 by Скуратович Александр
Options

очепреди, парсер

1 parent e40fb619
Expand all
Inline Side-by-side
Showing 10 changed files with 197 additions and 41 deletions
commands/HelloController.php deleted 100644 → 0
<?php
/**
* @link http://www.yiiframework.com/
* @copyright Copyright (c) 2008 Yii Software LLC
* @license http://www.yiiframework.com/license/
*/
namespace app\commands;
use yii\console\Controller;
use yii\console\ExitCode;
/**
* This command echoes the first argument that you have entered.
*
* This command is provided as an example for you to learn how to create console commands.
*
* @author Qiang Xue <[email protected]>
* @since 2.0
*/
class HelloController extends Controller
{
/**
* This command echoes what you have entered as the message.
* @param string $message the message to be echoed.
* @return int Exit code
*/
public function actionIndex($message = 'hello world')
{
echo $message . "\n";
return ExitCode::OK;
}
}
commands/ScanController.php 0 → 100644
<?php
namespace app\commands;
use yii;
use yii\console\Controller;
use yii\helpers\Console;
use app\components\Collection;
use app\jobs\HostJob;
/**
* Парсинг запрещенных ресурсов в РФ.
*/
class ScanController extends Controller
{
public $file = '';
public function beforeAction($action)
{
if (!parent::beforeAction($action)) {
return false;
}
return true;
}
public function options($actionID)
{
return ['file'];
}
public function optionAliases()
{
return ['f' => 'file'];
}
/**
* Парсинг запрещенных ресурсов в РФ.
*/
public function actionIndex()
{
$ids = [];
$row = 1;
$hostsArr = Collection::getHostsByCsv(Yii::$app->params['csv_url']);
$grabUrls = Yii::$app->params['registrators_urls'];
foreach ($hostsArr as $host) {
/* if ((int)!($row % 10000)) {
echo $this->ansiFormat('+', Console::FG_BLUE);
flush();
}*/
$ids[] = (int)Yii::$app->queue->push(new HostJob([
'host' => $host,
'grabSource' => $grabUrls[array_rand($grabUrls)]
]));
break;
$row++;
}
echo "\n".$this->ansiFormat('Количество полученных хостов', Console::FG_CYAN) . ' => ' . $this->ansiFormat(count($ids), Console::BOLD, Console::FG_GREEN) . "\n";
}
}
\ No newline at end of file \ No newline at end of file
components/Collection.php 0 → 100644
<?php
namespace app\components;
use Yii;
use yii\base\Component;
class Collection extends Component
{
public static function getHostsByCsv($csvFile, $separator = ";", $seporatorHosts = '|', $pos = 1, $unique = true)
{
$hosts = [];
$row = 1;
if (($handle = fopen($csvFile, "r")) !== FALSE) {
while (($data = fgetcsv($handle, 1000, $separator)) !== FALSE) {
if ($row > 1) {
$hostsString = $data[$pos] ?? '';
if ($hostsString) {
//$arr = array_map('trim', explode($seporatorHosts, $hostsString));
//$hosts = array_merge($hosts, $arr);
$hosts[] = str_replace('*.', '', $hostsString);
}
}
$row++;
}
fclose($handle);
}
if ($unique && $hosts) {
$hosts = array_unique($hosts);
}
return $hosts;
}
public static function getRawData($url, $domain)
{
$post = [
'a' => 'act',
'ip' => $domain,
];
$ch = curl_init('https://2ip.ua/ru/services/information-service/domain-information');
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
curl_setopt($ch, CURLOPT_POSTFIELDS, $post);
$response = curl_exec($ch);
$errorNum = curl_errno($ch);
$errorMsg = curl_error($ch);
curl_close($ch);
return ['data' => $response, 'errorno' => $errorNum, 'errormessage' => $errorMsg];
}
}
composer.json
...@@ -17,7 +17,9 @@ ...@@ -17,7 +17,9 @@
"php": ">=5.4.0", "php": ">=5.4.0",
"yiisoft/yii2": "~2.0.14", "yiisoft/yii2": "~2.0.14",
"yiisoft/yii2-bootstrap": "~2.0.0", "yiisoft/yii2-bootstrap": "~2.0.0",
"yiisoft/yii2-swiftmailer": "~2.0.0 || ~2.1.0" "yiisoft/yii2-swiftmailer": "~2.0.0 || ~2.1.0",
"yiisoft/yii2-queue": "^2.3",
"paquettg/php-html-parser": "^3.1"
}, },
"require-dev": { "require-dev": {
"yiisoft/yii2-debug": "~2.1.0", "yiisoft/yii2-debug": "~2.1.0",
......
config/console.php
...@@ -6,7 +6,7 @@ $db = require __DIR__ . '/db.php'; ...@@ -6,7 +6,7 @@ $db = require __DIR__ . '/db.php';
$config = [ $config = [
'id' => 'basic-console', 'id' => 'basic-console',
'basePath' => dirname(__DIR__), 'basePath' => dirname(__DIR__),
'bootstrap' => ['log'], 'bootstrap' => ['log','queue'],
'controllerNamespace' => 'app\commands', 'controllerNamespace' => 'app\commands',
'aliases' => [ 'aliases' => [
'@bower' => '@vendor/bower-asset', '@bower' => '@vendor/bower-asset',
...@@ -17,6 +17,16 @@ $config = [ ...@@ -17,6 +17,16 @@ $config = [
'cache' => [ 'cache' => [
'class' => 'yii\caching\FileCache', 'class' => 'yii\caching\FileCache',
], ],
'queue' => [
'class' => \yii\queue\db\Queue::class,
'db' => 'db', // DB connection component or its config
'tableName' => '{{%queue}}', // Table name
'channel' => 'default', // Queue channel key
'mutex' => \yii\mutex\MysqlMutex::class, // Mutex used to sync queries
'as log' => \yii\queue\LogBehavior::class,
'ttr' => 3 * 60, // Максимальное время выполнения задания
'attempts' => 3,
],
'authManager' => [ 'authManager' => [
'class' => 'yii\rbac\DbManager', 'class' => 'yii\rbac\DbManager',
'defaultRoles' => ['guest'], 'defaultRoles' => ['guest'],
...@@ -32,13 +42,19 @@ $config = [ ...@@ -32,13 +42,19 @@ $config = [
'db' => $db, 'db' => $db,
], ],
'params' => $params, 'params' => $params,
/*
'controllerMap' => [ 'controllerMap' => [
'fixture' => [ // Fixture generation command line. 'migrate' => [
'class' => 'yii\faker\FixtureController', 'class' => 'yii\console\controllers\MigrateController',
], 'migrationPath' => '@app/migrations',
'migrationNamespaces' => [
'yii\db\Migration',
'yii\queue\db\migrations',
],
]
/* 'fixture' => [ // Fixture generation command line.
'class' => 'yii\faker\FixtureController',
]*/
], ],
*/
]; ];
if (YII_ENV_DEV) { if (YII_ENV_DEV) {
......
config/params.php
...@@ -4,4 +4,8 @@ return [ ...@@ -4,4 +4,8 @@ return [
'adminEmail' => '[email protected]', 'adminEmail' => '[email protected]',
'senderEmail' => '[email protected]', 'senderEmail' => '[email protected]',
'senderName' => 'Example.com mailer', 'senderName' => 'Example.com mailer',
'csv_url' => 'https://raw.githubusercontent.com/zapret-info/z-i/master/dump-00.csv',
'registrators_urls' => [
'https://2ip.ua/ru/services/information-service/domain-information'
]
]; ];
config/web.php
...@@ -35,6 +35,14 @@ $config = [ ...@@ -35,6 +35,14 @@ $config = [
], ],
], ],
], ],
'queue' => [
'class' => \yii\queue\db\Queue::class,
'db' => 'db', // DB connection component or its config
'tableName' => '{{%queue}}', // Table name
'channel' => 'default', // Queue channel key
'mutex' => \yii\mutex\MysqlMutex::class, // Mutex used to sync queries
'as log' => \yii\queue\LogBehavior::class,
],
'authManager' => [ 'authManager' => [
'class' => 'yii\rbac\DbManager', 'class' => 'yii\rbac\DbManager',
......
controllers/SiteController.php
...@@ -2,6 +2,7 @@ ...@@ -2,6 +2,7 @@
namespace app\controllers; namespace app\controllers;
use app\components\Collection;
use Yii; use Yii;
use yii\filters\AccessControl; use yii\filters\AccessControl;
use yii\web\Controller; use yii\web\Controller;
......
jobs/HostJob.php 0 → 100644
<?php
namespace app\jobs;
use Yii;
use PHPHtmlParser\Dom;
use app\components\Collection;
use yii\base\BaseObject;
use \yii\queue\JobInterface;
class HostJob extends BaseObject implements JobInterface
{
//yii queue/listen -v --color
// https://github.com/paquettg/php-html-parser
public $host;
public $grabSource;
public function execute($queue)
{
echo $this->host;
$result = Collection::getRawData($this->grabSource, $this->host);
if (!$result['errorno']) {//без ошибок
$dom = new Dom;
$dom->loadStr($result['data']);
$table = $dom->getElementById('#resultcontainer');
if ($table) {
$td = $table->find('td');
foreach ($td as $t) {
echo $t->text."\n";
}
}
// echo $a->text;
}
return null;
}
}
\ No newline at end of file \ No newline at end of file
Styling with Markdown is supported
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!