Skip to content

Alto / rkn

Go to a project
Toggle navigation
Toggle navigation pinning
Commit e0b1fd55 by san58
Options

правильный парс + счетчики

1 parent 9f0a280e
Inline Side-by-side
Showing 2 changed files with 42 additions and 19 deletions
commands/ScanController.php
...@@ -8,6 +8,7 @@ use yii\db\Expression; ...@@ -8,6 +8,7 @@ use yii\db\Expression;
use yii\helpers\Console; use yii\helpers\Console;
use app\components\Collection; use app\components\Collection;
use app\models\Host; use app\models\Host;
use yii\db\Query;
/** /**
* Парсинг запрещенных ресурсов в РФ. * Парсинг запрещенных ресурсов в РФ.
...@@ -39,14 +40,17 @@ class ScanController extends Controller ...@@ -39,14 +40,17 @@ class ScanController extends Controller
*/ */
public function actionIndex() public function actionIndex()
{ {
$rows = 0; $rec = ['insert'=>0, 'update'=>0];
$csv_row = 0;
$csv_skip_row = 0;
echo PHP_EOL.$this->ansiFormat('Download file: '.Yii::$app->params['csv_url'], Console::FG_CYAN);
$csv_file = Collection::download_file(Yii::$app->params['csv_url']); $csv_file = Collection::download_file(Yii::$app->params['csv_url']);
if ($csv_file!=='') if ($csv_file!=='')
{ {
echo PHP_EOL.$this->ansiFormat('Start parse file: '.$csv_file, Console::FG_CYAN);
$csv_handle = fopen($csv_file ,'r'); $csv_handle = fopen($csv_file ,'r');
if ($csv_handle !== false) if ($csv_handle !== false)
{ {
$csv_row = 0;
$csv_datatime = ''; $csv_datatime = '';
while (($csv_data = fgetcsv($csv_handle, 1024, ';')) !== false) while (($csv_data = fgetcsv($csv_handle, 1024, ';')) !== false)
{ {
...@@ -71,7 +75,26 @@ class ScanController extends Controller ...@@ -71,7 +75,26 @@ class ScanController extends Controller
$db_request = Yii::$app->db->createCommand(); $db_request = Yii::$app->db->createCommand();
if ($db_request) if ($db_request)
{ {
if ( Host::find()->where(['domain' =>$host])->count()>0 ) $host_bd = (new Query())
->select('`csv_date`')
->from('{{%host}}')
->where('domain=:host', array(':host'=>$host))
->limit(1)
->one();
if ( $host_bd === false )
{
$db_request->insert(
'{{%host}}',
[
'domain' =>$host,
'created_at' => new Expression('NOW()'),
'csv_date' => $csv_datatime,
]
)->execute();
$rec['insert']++;
}
elseif (isset($host_bd['csv_date']) && $host_bd['csv_date'] !== $csv_datatime)
{
$db_request->update( $db_request->update(
'{{%host}}', '{{%host}}',
[ [
...@@ -80,37 +103,36 @@ class ScanController extends Controller ...@@ -80,37 +103,36 @@ class ScanController extends Controller
[ [
'domain' => $host 'domain' => $host
] ]
); )->execute();
else $rec['update']++;
$db_request->insert( }
'{{%host}}', unset($host_bd);
[
'domain' =>$host,
'created_at' => new Expression('NOW()'),
'csv_date' => $csv_datatime,
]
);
$db_request->execute();
$rows++;
} }
unset($db_request); unset($db_request);
} }
unset($host_level); unset($host_level);
unset($host); unset($host);
} }
else
$csv_skip_row++;
} }
unset($csv_data); unset($csv_data);
unset($csv_datatime); unset($csv_datatime);
unset($csv_row);
fclose($csv_handle); fclose($csv_handle);
unlink($csv_file); unlink($csv_file);
} }
else
echo PHP_EOL.$this->ansiFormat('! Not file open for read');
unset($csv_handle); unset($csv_handle);
} }
else
echo PHP_EOL.$this->ansiFormat('! Not file download');
unset($csv_file); unset($csv_file);
echo "\n".$this->ansiFormat('Count of hosts processed: ', Console::FG_CYAN) . ' => ' . $this->ansiFormat($rows, Console::BOLD, Console::FG_GREEN) . "\n"; echo PHP_EOL.$this->ansiFormat('Count of rows processed: ', Console::FG_CYAN). $this->ansiFormat($csv_row, Console::BOLD, Console::FG_GREEN). ' Skiped rows: '.$this->ansiFormat($csv_skip_row, Console::BOLD, Console::FG_RED).' (Insert: '.$this->ansiFormat($rec['insert'], Console::BOLD, Console::FG_BLUE).' Update:'.$this->ansiFormat($rec['update'], Console::BOLD, Console::FG_BLUE).')';
unset($rows); echo PHP_EOL;
unset($rec);
unset($csv_skip_row);
unset($csv_row);
} }
} }
\ No newline at end of file \ No newline at end of file
components/Collection.php
...@@ -21,6 +21,7 @@ class Collection extends Component ...@@ -21,6 +21,7 @@ class Collection extends Component
if ($file_write!==false) if ($file_write!==false)
$out = $temp_csv_name; $out = $temp_csv_name;
unset($file_write); unset($file_write);
fclose($file_handle);
} }
unset($file_handle); unset($file_handle);
} }
......
Styling with Markdown is supported
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!