<?php
namespace app\commands;
use yii;
use yii\console\Controller;
use yii\db\Expression;
use yii\helpers\Console;
use app\components\Collection;
use app\models\Host;
use app\models\Filter;
use yii\db\Query;
/**
* Парсинг запрещенных ресурсов в РФ.
*/
class ScanController extends Controller
{
public $file = '';
public function beforeAction($action)
{
if (!parent::beforeAction($action)) {
return false;
}
return true;
}
public function options($actionID)
{
return ['file'];
}
public function optionAliases()
{
return ['f' => 'file'];
}
/**
* Парсинг запрещенных ресурсов в РФ.
*/
public function actionIndex()
{
$rec = ['insert'=>0, 'update'=>0];
$csv_row = 0;
$csv_skip_row = 0;
echo PHP_EOL.'Start datatime '.$this->ansiFormat(date('Y.m.d H:i:s'), Console::FG_CYAN);
echo PHP_EOL.$this->ansiFormat('Download file: '.Yii::$app->params['csv_url'], Console::FG_CYAN);
$csv_file = Collection::download_file(Yii::$app->params['csv_url']);
if ($csv_file!=='')
{
$file_size = filesize($csv_file);
if ($file_size === false)
$file_size = 0;
echo PHP_EOL.$this->ansiFormat('Start parse file: '.$csv_file.' size: '.Collection::human_filesize($file_size), Console::FG_CYAN);
$csv_handle = fopen($csv_file ,'r');
if ($csv_handle !== false)
{
$filters = ['1'=>Collection::get_filter(Filter::TYPE_1), '2'=>Collection::get_filter(Filter::TYPE_2), '3'=>Collection::get_filter(Filter::TYPE_3)];
$csv_datatime = '';
$progress = 0;
//$time_start = 0;
while (($csv_data = fgetcsv($csv_handle, 1024, ';')) !== false /*&& $csv_row<2920*/)
{
//if ($csv_row === 2900){ $time_cuttent = microtime(true); echo "\n".'t1 '.($time_cuttent - $time_start)."\n";$time_start = $time_cuttent;}
$file_pos = ftell($csv_handle);
//if ($csv_row === 2900){ $time_cuttent = microtime(true); echo 't2 '.($time_cuttent - $time_start)."\n";$time_start = $time_cuttent;}
if ($file_pos !== false && $file_size>0)
{
$progress_current = (int)round(10*$file_pos/$file_size);
if ($progress !== $progress_current)
{
$progress = $progress_current;
echo PHP_EOL.'Time: '.date('Y.m.d H:i:s').' Progress: '.(10*$progress).'% line: '.$csv_row.' (Insert: '.$this->ansiFormat($rec['insert'], Console::BOLD, Console::FG_BLUE).' Update:'.$this->ansiFormat($rec['update'], Console::BOLD, Console::FG_BLUE).')';
}
unset($progress_current);
}
//if ($csv_row === 2900){ $time_cuttent = microtime(true); echo 't3 '.($time_cuttent - $time_start)."\n";$time_start = $time_cuttent;}
$csv_row++;
if ($csv_row===1)
$csv_datatime = date(
"Y-m-d H:i:s",
strtotime(
str_replace('Updated: ', '', $csv_data[0])
)
);
elseif (isset($csv_data[1])===true)
{
//if ($csv_row === 2901){ $time_cuttent = microtime(true); echo 't4 '.($time_cuttent - $time_start)."\n";$time_start = $time_cuttent;}
$host = Collection::domain_filter($csv_data[1]);
//if ($csv_row === 2901){ $time_cuttent = microtime(true); echo 't5 '.($time_cuttent - $time_start)."\n";$time_start = $time_cuttent;}
if ($host === false)
{
$csv_skip_row++;
}
else
{
//if ($csv_row === 2901){ $time_cuttent = microtime(true); echo 't6 '.($time_cuttent - $time_start)."\n";$time_start = $time_cuttent;}
$status = [
'1'=>(Collection::apply_filter($host, $filters['1'])===false?Host::STATUS_OFF:Host::STATUS_ON),
'2'=>(Collection::apply_filter($host, $filters['2'])===false?Host::STATUS_OFF:Host::STATUS_ON),
'3'=>(Collection::apply_filter($host, $filters['3'])===false?Host::STATUS_OFF:Host::STATUS_ON)
];
//if ($csv_row === 2901){ $time_cuttent = microtime(true); echo 't7 '.($time_cuttent - $time_start)."\n";$time_start = $time_cuttent;}
$db_request = Yii::$app->db->createCommand();
//if ($csv_row === 2901){ $time_cuttent = microtime(true); echo 't8 '.($time_cuttent - $time_start)."\n";$time_start = $time_cuttent;}
if ($db_request)
{
$host_bd = (new Query())
->select('`csv_date`')
->from('{{%host}}')
->where('domain=:host', array(':host'=>$host))
->limit(1)
->one();
//if ($csv_row === 2901){ $time_cuttent = microtime(true); echo 't9 '.($time_cuttent - $time_start)."\n";$time_start = $time_cuttent;}
if ( $host_bd === false )
{
$db_request->insert(
'{{%host}}',
[
'domain' =>$host,
'created_at' => new Expression('NOW()'),
'csv_date' => $csv_datatime,
'status' => Host::STATUS_ON,
'f1_status' => $status['1'],
'f2_status' => $status['2'],
'f3_status' => $status['3']
]
)->execute();
$rec['insert']++;
//if ($csv_row === 2901){ $time_cuttent = microtime(true); echo 't10 '.($time_cuttent - $time_start)."\n";$time_start = $time_cuttent;}
}
elseif (isset($host_bd['csv_date']) && $host_bd['csv_date'] !== $csv_datatime)
{
//if ($csv_row === 2901){ $time_cuttent = microtime(true); echo 't11 '.($time_cuttent - $time_start)."\n";$time_start = $time_cuttent;}
$db_request->update(
'{{%host}}',
[
'csv_date' => $csv_datatime,
'f1_status' => $status['1'],
'f2_status' => $status['2'],
'f3_status' => $status['3']
],
[
'domain' => $host
]
)->execute();
$rec['update']++;
//if ($csv_row === 2901){ $time_cuttent = microtime(true); echo 't12 '.($time_cuttent - $time_start)."\n";$time_start = $time_cuttent;}
}
unset($host_bd);
//if ($csv_row === 2901){ $time_cuttent = microtime(true); echo 't13 '.($time_cuttent - $time_start)."\n";$time_start = $time_cuttent;}
}
else
echo PHP_EOL.$this->ansiFormat('! Error create DB request');
unset($db_request);
unset($status);
//if ($csv_row === 2901){ $time_cuttent = microtime(true); echo 't14 '.($time_cuttent - $time_start)."\n";$time_start = $time_cuttent;}
}
unset($host);
//if ($csv_row === 2901){ $time_cuttent = microtime(true); echo 't15 '.($time_cuttent - $time_start)."\n";$time_start = $time_cuttent;}
}
else
$csv_skip_row++;
unset($file_pos);
//if ($csv_row === 2901){ $time_cuttent = microtime(true); echo 't16 '.($time_cuttent - $time_start)."\n";$time_start = $time_cuttent;}
}
unset($progress);
unset($csv_data);
unset($csv_datatime);
unset($filters);
fclose($csv_handle);
unlink($csv_file);
}
else
echo PHP_EOL.$this->ansiFormat('! Not file open for read');
unset($csv_handle);
unset($file_size);
}
else
echo PHP_EOL.$this->ansiFormat('! Not file download');
unset($csv_file);
echo PHP_EOL.$this->ansiFormat('Count of rows processed: ', Console::FG_CYAN). $this->ansiFormat($csv_row, Console::BOLD, Console::FG_GREEN). ' Skiped rows: '.$this->ansiFormat($csv_skip_row, Console::BOLD, Console::FG_RED).' (Insert: '.$this->ansiFormat($rec['insert'], Console::BOLD, Console::FG_BLUE).' Update:'.$this->ansiFormat($rec['update'], Console::BOLD, Console::FG_BLUE).')';
echo PHP_EOL;
unset($rec);
unset($csv_skip_row);
unset($csv_row);
}
}