очепреди, парсер

commands/HelloController.php

-<?php
-/**
- * @link http://www.yiiframework.com/
- * @copyright Copyright (c) 2008 Yii Software LLC
- * @license http://www.yiiframework.com/license/
- */
-
-namespace app\commands;
-
-use yii\console\Controller;
-use yii\console\ExitCode;
-
-/**
- * This command echoes the first argument that you have entered.
- *
- * This command is provided as an example for you to learn how to create console commands.
- *
- * @author Qiang Xue <[email protected]>
- * @since 2.0
- */
-class HelloController extends Controller
-{
-    /**
-     * This command echoes what you have entered as the message.
-     * @param string $message the message to be echoed.
-     * @return int Exit code
-     */
-    public function actionIndex($message = 'hello world')
-    {
-        echo $message . "\n";
-
-        return ExitCode::OK;
-    }
-}

commands/ScanController.php

+<?php
+
+namespace app\commands;
+
+use yii;
+use yii\console\Controller;
+use yii\helpers\Console;
+use app\components\Collection;
+use app\jobs\HostJob;
+
+/**
+ * Парсинг запрещенных ресурсов в РФ.
+ */
+class ScanController extends Controller
+{
+    public $file = '';
+
+    public function beforeAction($action)
+    {
+        if (!parent::beforeAction($action)) {
+            return false;
+        }
+        return true;
+    }
+
+    public function options($actionID)
+    {
+        return ['file'];
+    }
+
+    public function optionAliases()
+    {
+        return ['f' => 'file'];
+    }
+
+    /**
+     * Парсинг запрещенных ресурсов в РФ.
+     */
+    public function actionIndex()
+    {
+        $ids = [];
+        $row = 1;
+        $hostsArr = Collection::getHostsByCsv(Yii::$app->params['csv_url']);
+        $grabUrls = Yii::$app->params['registrators_urls'];
+        foreach ($hostsArr as $host) {
+          /*  if ((int)!($row % 10000)) {
+                echo $this->ansiFormat('+', Console::FG_BLUE);
+                flush();
+            }*/
+            $ids[] = (int)Yii::$app->queue->push(new HostJob([
+                'host' => $host,
+                'grabSource' => $grabUrls[array_rand($grabUrls)]
+            ]));
+            break;
+            $row++;
+        }
+
+        echo "\n".$this->ansiFormat('Количество полученных хостов', Console::FG_CYAN) . ' => ' . $this->ansiFormat(count($ids), Console::BOLD, Console::FG_GREEN) . "\n";
+    }
+
+}
\ No newline at end of file

components/Collection.php

+<?php
+
+namespace app\components;
+
+use Yii;
+use yii\base\Component;
+
+class Collection extends Component
+{
+    public static function getHostsByCsv($csvFile, $separator = ";", $seporatorHosts = '|', $pos = 1, $unique = true)
+    {
+        $hosts = [];
+        $row = 1;
+        if (($handle = fopen($csvFile, "r")) !== FALSE) {
+            while (($data = fgetcsv($handle, 1000, $separator)) !== FALSE) {
+                if ($row > 1) {
+                    $hostsString = $data[$pos] ?? '';
+                    if ($hostsString) {
+                        //$arr = array_map('trim', explode($seporatorHosts, $hostsString));
+                        //$hosts = array_merge($hosts, $arr);
+                        $hosts[] = str_replace('*.', '', $hostsString);
+                    }
+                }
+                $row++;
+            }
+            fclose($handle);
+        }
+        if ($unique && $hosts) {
+            $hosts = array_unique($hosts);
+        }
+
+        return $hosts;
+    }
+
+
+    public static function getRawData($url, $domain)
+    {
+        $post = [
+            'a' => 'act',
+            'ip' => $domain,
+        ];
+        $ch = curl_init('https://2ip.ua/ru/services/information-service/domain-information');
+        curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
+        curl_setopt($ch, CURLOPT_POSTFIELDS, $post);
+
+        $response = curl_exec($ch);
+        $errorNum = curl_errno($ch);
+        $errorMsg = curl_error($ch);
+
+        curl_close($ch);
+
+        return ['data' => $response, 'errorno' => $errorNum, 'errormessage' => $errorMsg];
+    }
+
+}
+

composer.json

@@ -17,7 +17,9 @@
         "php": ">=5.4.0",
         "yiisoft/yii2": "~2.0.14",
         "yiisoft/yii2-bootstrap": "~2.0.0",
-        "yiisoft/yii2-swiftmailer": "~2.0.0 || ~2.1.0"
+        "yiisoft/yii2-swiftmailer": "~2.0.0 || ~2.1.0",
+        "yiisoft/yii2-queue": "^2.3",
+        "paquettg/php-html-parser": "^3.1"
     },
     "require-dev": {
         "yiisoft/yii2-debug": "~2.1.0",

config/console.php

@@ -6,7 +6,7 @@ $db = require __DIR__ . '/db.php';
 $config = [
     'id' => 'basic-console',
     'basePath' => dirname(__DIR__),
-    'bootstrap' => ['log'],
+    'bootstrap' => ['log','queue'],
     'controllerNamespace' => 'app\commands',
     'aliases' => [
         '@bower' => '@vendor/bower-asset',
@@ -17,6 +17,16 @@ $config = [
         'cache' => [
             'class' => 'yii\caching\FileCache',
         ],
+        'queue' => [
+            'class' => \yii\queue\db\Queue::class,
+            'db' => 'db', // DB connection component or its config
+            'tableName' => '{{%queue}}', // Table name
+            'channel' => 'default', // Queue channel key
+            'mutex' => \yii\mutex\MysqlMutex::class, // Mutex used to sync queries
+            'as log' => \yii\queue\LogBehavior::class,
+            'ttr' => 3 * 60, // Максимальное время выполнения задания
+            'attempts' => 3,
+        ],
         'authManager'  => [
             'class' => 'yii\rbac\DbManager',
             'defaultRoles' => ['guest'],
@@ -32,13 +42,19 @@ $config = [
         'db' => $db,
     ],
     'params' => $params,
-    /*
     'controllerMap' => [
-        'fixture' => [ // Fixture generation command line.
-            'class' => 'yii\faker\FixtureController',
+        'migrate' => [
+            'class' => 'yii\console\controllers\MigrateController',
+            'migrationPath' => '@app/migrations',
+            'migrationNamespaces' => [
+                'yii\db\Migration',
+                'yii\queue\db\migrations',
             ],
+        ]
+        /* 'fixture' => [ // Fixture generation command line.
+             'class' => 'yii\faker\FixtureController',
+         ]*/
     ],
-    */
 ];
 
 if (YII_ENV_DEV) {

config/params.php

@@ -4,4 +4,8 @@ return [
     'adminEmail' => '[email protected]',
     'senderEmail' => '[email protected]',
     'senderName' => 'Example.com mailer',
+    'csv_url' => 'https://raw.githubusercontent.com/zapret-info/z-i/master/dump-00.csv',
+    'registrators_urls' => [
+        'https://2ip.ua/ru/services/information-service/domain-information'
+    ]
 ];

config/web.php

@@ -35,6 +35,14 @@ $config = [
                 ],
             ],
         ],
+        'queue' => [
+            'class' => \yii\queue\db\Queue::class,
+            'db' => 'db', // DB connection component or its config
+            'tableName' => '{{%queue}}', // Table name
+            'channel' => 'default', // Queue channel key
+            'mutex' => \yii\mutex\MysqlMutex::class, // Mutex used to sync queries
+            'as log' => \yii\queue\LogBehavior::class,
+        ],
 
         'authManager' => [
             'class' => 'yii\rbac\DbManager',

controllers/SiteController.php

@@ -2,6 +2,7 @@
 
 namespace app\controllers;
 
+use app\components\Collection;
 use Yii;
 use yii\filters\AccessControl;
 use yii\web\Controller;

jobs/HostJob.php

+<?php
+
+namespace app\jobs;
+
+use Yii;
+use PHPHtmlParser\Dom;
+use app\components\Collection;
+use yii\base\BaseObject;
+use \yii\queue\JobInterface;
+
+class HostJob extends BaseObject implements JobInterface
+{
+    //yii queue/listen -v --color
+    // https://github.com/paquettg/php-html-parser
+
+    public $host;
+    public $grabSource;
+
+    public function execute($queue)
+    {
+        echo $this->host;
+
+        $result = Collection::getRawData($this->grabSource, $this->host);
+        if (!$result['errorno']) {//без ошибок
+            $dom = new Dom;
+            $dom->loadStr($result['data']);
+            $table = $dom->getElementById('#resultcontainer');
+            if ($table) {
+              $td = $table->find('td');
+              foreach ($td as $t) {
+                 echo $t->text."\n";
+              }
+            }
+
+          //  echo $a->text;
+        }
+
+
+        return null;
+    }
+
+}
\ No newline at end of file