<?php namespace app\commands; use yii; use yii\console\Controller; use yii\db\Expression; use yii\helpers\Console; use app\components\Collection; use app\models\Host; use app\models\Filter; use yii\db\Query; /** * Парсинг запрещенных ресурсов в РФ. */ class ScanController extends Controller { public $file = ''; public function beforeAction($action) { if (!parent::beforeAction($action)) { return false; } return true; } public function options($actionID) { return ['file']; } public function optionAliases() { return ['f' => 'file']; } /** * Парсинг запрещенных ресурсов в РФ. */ public function actionIndex() { $rec = ['insert'=>0, 'update'=>0]; $csv_row = 0; $csv_skip_row = 0; echo PHP_EOL.'Start datatime '.$this->ansiFormat(date('Y.m.d H:i:s'), Console::FG_CYAN); echo PHP_EOL.$this->ansiFormat('Download file: '.Yii::$app->params['csv_url'], Console::FG_CYAN); $csv_file = Collection::download_file(Yii::$app->params['csv_url']); if ($csv_file!=='') { $file_size = filesize($csv_file); if ($file_size === false) $file_size = 0; echo PHP_EOL.$this->ansiFormat('Start parse file: '.$csv_file.' size: '.Collection::human_filesize($file_size), Console::FG_CYAN); $csv_handle = fopen($csv_file ,'r'); if ($csv_handle !== false) { $filters = ['before'=>Collection::get_filter(Filter::TYPE_BEFORE), 'after'=>Collection::get_filter(Filter::TYPE_AFTER)]; $csv_datatime = ''; $progress = 0; while (($csv_data = fgetcsv($csv_handle, 1024, ';')) !== false) { $file_pos = ftell($csv_handle); if ($file_pos !== false && $file_size>0) { $progress_current = (int)round(10*$file_pos/$file_size); if ($progress !== $progress_current) { $progress = $progress_current; echo PHP_EOL.'Time: '.date('Y.m.d H:i:s').' Progress: '.(10*$progress).'% domains: '.$csv_row; } unset($progress_current); } $csv_row++; if ($csv_row===1) $csv_datatime = date( "Y-m-d H:i:s", strtotime( str_replace('Updated: ', '', $csv_data[0]) ) ); elseif (isset($csv_data[1])===true) { $host = Collection::domain_filter($csv_data[1], $filters['before']); if ($host === false) $csv_skip_row++; else { $status = (Collection::apply_filter($host, $filters['after'])===false?Filter::STATUS_OFF:Filter::STATUS_ON); $db_request = Yii::$app->db->createCommand(); if ($db_request) { $host_bd = (new Query()) ->select('`csv_date`') ->from('{{%host}}') ->where('domain=:host', array(':host'=>$host)) ->limit(1) ->one(); if ( $host_bd === false ) { $db_request->insert( '{{%host}}', [ 'domain' =>$host, 'created_at' => new Expression('NOW()'), 'csv_date' => $csv_datatime, 'status' => $status ] )->execute(); $rec['insert']++; } elseif (isset($host_bd['csv_date']) && $host_bd['csv_date'] !== $csv_datatime) { $db_request->update( '{{%host}}', [ 'csv_date' => $csv_datatime, 'status' => $status ], [ 'domain' => $host ] )->execute(); $rec['update']++; } unset($host_bd); } else echo PHP_EOL.$this->ansiFormat('! Error create DB request'); unset($db_request); unset($status); } unset($host); } else $csv_skip_row++; unset($file_pos); } unset($progress); unset($csv_data); unset($csv_datatime); unset($filters); fclose($csv_handle); unlink($csv_file); } else echo PHP_EOL.$this->ansiFormat('! Not file open for read'); unset($csv_handle); unset($file_size); } else echo PHP_EOL.$this->ansiFormat('! Not file download'); unset($csv_file); echo PHP_EOL.$this->ansiFormat('Count of rows processed: ', Console::FG_CYAN). $this->ansiFormat($csv_row, Console::BOLD, Console::FG_GREEN). ' Skiped rows: '.$this->ansiFormat($csv_skip_row, Console::BOLD, Console::FG_RED).' (Insert: '.$this->ansiFormat($rec['insert'], Console::BOLD, Console::FG_BLUE).' Update:'.$this->ansiFormat($rec['update'], Console::BOLD, Console::FG_BLUE).')'; echo PHP_EOL; unset($rec); unset($csv_skip_row); unset($csv_row); } }