Skip to content

Alto / rkn

Go to a project
Toggle navigation
Toggle navigation pinning
Commit e0b1fd55 by san58
Options

правильный парс + счетчики

1 parent 9f0a280e
Inline Side-by-side
Showing 2 changed files with 42 additions and 19 deletions
commands/ScanController.php
......@@ -8,6 +8,7 @@ use yii\db\Expression;
use yii\helpers\Console;
use app\components\Collection;
use app\models\Host;
use yii\db\Query;
/**
* Парсинг запрещенных ресурсов в РФ.
......@@ -39,14 +40,17 @@ class ScanController extends Controller
*/
public function actionIndex()
{
$rows = 0;
$rec = ['insert'=>0, 'update'=>0];
$csv_row = 0;
$csv_skip_row = 0;
echo PHP_EOL.$this->ansiFormat('Download file: '.Yii::$app->params['csv_url'], Console::FG_CYAN);
$csv_file = Collection::download_file(Yii::$app->params['csv_url']);
if ($csv_file!=='')
{
echo PHP_EOL.$this->ansiFormat('Start parse file: '.$csv_file, Console::FG_CYAN);
$csv_handle = fopen($csv_file ,'r');
if ($csv_handle !== false)
{
$csv_row = 0;
$csv_datatime = '';
while (($csv_data = fgetcsv($csv_handle, 1024, ';')) !== false)
{
......@@ -71,7 +75,26 @@ class ScanController extends Controller
$db_request = Yii::$app->db->createCommand();
if ($db_request)
{
if ( Host::find()->where(['domain' =>$host])->count()>0 )
$host_bd = (new Query())
->select('`csv_date`')
->from('{{%host}}')
->where('domain=:host', array(':host'=>$host))
->limit(1)
->one();
if ( $host_bd === false )
{
$db_request->insert(
'{{%host}}',
[
'domain' =>$host,
'created_at' => new Expression('NOW()'),
'csv_date' => $csv_datatime,
]
)->execute();
$rec['insert']++;
}
elseif (isset($host_bd['csv_date']) && $host_bd['csv_date'] !== $csv_datatime)
{
$db_request->update(
'{{%host}}',
[
......@@ -80,37 +103,36 @@ class ScanController extends Controller
[
'domain' => $host
]
);
else
$db_request->insert(
'{{%host}}',
[
'domain' =>$host,
'created_at' => new Expression('NOW()'),
'csv_date' => $csv_datatime,
]
);
$db_request->execute();
$rows++;
)->execute();
$rec['update']++;
}
unset($host_bd);
}
unset($db_request);
}
unset($host_level);
unset($host);
}
else
$csv_skip_row++;
}
unset($csv_data);
unset($csv_datatime);
unset($csv_row);
fclose($csv_handle);
unlink($csv_file);
}
else
echo PHP_EOL.$this->ansiFormat('! Not file open for read');
unset($csv_handle);
}
else
echo PHP_EOL.$this->ansiFormat('! Not file download');
unset($csv_file);
echo "\n".$this->ansiFormat('Count of hosts processed: ', Console::FG_CYAN) . ' => ' . $this->ansiFormat($rows, Console::BOLD, Console::FG_GREEN) . "\n";
unset($rows);
echo PHP_EOL.$this->ansiFormat('Count of rows processed: ', Console::FG_CYAN). $this->ansiFormat($csv_row, Console::BOLD, Console::FG_GREEN). ' Skiped rows: '.$this->ansiFormat($csv_skip_row, Console::BOLD, Console::FG_RED).' (Insert: '.$this->ansiFormat($rec['insert'], Console::BOLD, Console::FG_BLUE).' Update:'.$this->ansiFormat($rec['update'], Console::BOLD, Console::FG_BLUE).')';
echo PHP_EOL;
unset($rec);
unset($csv_skip_row);
unset($csv_row);
}
}
\ No newline at end of file
components/Collection.php
......@@ -21,6 +21,7 @@ class Collection extends Component
if ($file_write!==false)
$out = $temp_csv_name;
unset($file_write);
fclose($file_handle);
}
unset($file_handle);
}
......
Styling with Markdown is supported
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!