|
|
|
<?php
|
|
|
|
|
|
|
|
namespace App\Console\Commands\Update;
|
|
|
|
|
|
|
|
use App\Models\Com\UpdateVisit;
|
|
|
|
use App\Models\Visit\Visit;
|
|
|
|
use App\Models\Visit\VisitItem;
|
|
|
|
use App\Services\ProjectServer;
|
|
|
|
use Illuminate\Console\Command;
|
|
|
|
use Illuminate\Support\Facades\DB;
|
|
|
|
use Illuminate\Support\Facades\Redis;
|
|
|
|
|
|
|
|
/**
|
|
|
|
* 4.0,5.0升级到6.0，访问同步
|
|
|
|
* Class ProjectImport
|
|
|
|
* @package App\Console\Commands
|
|
|
|
* @author Akun
|
|
|
|
* @date 2023/12/18 15:52
|
|
|
|
*/
|
|
|
|
class ProjectVisit extends Command
|
|
|
|
{
|
|
|
|
/**
|
|
|
|
* The name and signature of the console command.
|
|
|
|
*
|
|
|
|
* @var string
|
|
|
|
*/
|
|
|
|
protected $signature = 'project_visit';
|
|
|
|
|
|
|
|
/**
|
|
|
|
* The console command description.
|
|
|
|
*
|
|
|
|
* @var string
|
|
|
|
*/
|
|
|
|
protected $description = '执行项目升级访问任务';
|
|
|
|
|
|
|
|
|
|
|
|
public function handle()
|
|
|
|
{
|
|
|
|
ini_set('memory_limit', '512M');
|
|
|
|
while (true) {
|
|
|
|
$this->start_visit();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
protected function start_visit()
|
|
|
|
{
|
|
|
|
$task_id = $this->get_task();
|
|
|
|
if (!$task_id) {
|
|
|
|
sleep(60);
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
$task = UpdateVisit::where('id', $task_id)->where('status', UpdateVisit::STATUS_UN)->first();
|
|
|
|
if (!$task) {
|
|
|
|
sleep(2);
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
$project_id = $task->project_id;
|
|
|
|
$api_type = $task->api_type;
|
|
|
|
$api_url = $task->api_url;
|
|
|
|
|
|
|
|
$page_size = 1000;
|
|
|
|
|
|
|
|
echo 'date:' . date('Y-m-d H:i:s') . ', task_id: ' . $task->id . ', task_type: ' . $api_type . ', update start' . PHP_EOL;
|
|
|
|
|
|
|
|
$task->status = UpdateVisit::STATUS_ING;//同步中
|
|
|
|
$task->save();
|
|
|
|
|
|
|
|
//设置数据库
|
|
|
|
$project = ProjectServer::useProject($project_id);
|
|
|
|
if ($project) {
|
|
|
|
if ($api_type == 'visit_list') {
|
|
|
|
//访问列表
|
|
|
|
$url = $api_url . '?' . http_build_query(['w' => 'visit_list', 'page' => 1, 'pagesize' => 1]);
|
|
|
|
$data = curl_c($url);
|
|
|
|
if (isset($data['count']) && $data['count'] > 0) {
|
|
|
|
$count = $data['count'];
|
|
|
|
$max_id = $data['data'][0]['id'] ?? 0;
|
|
|
|
$is_stop = 0;
|
|
|
|
|
|
|
|
$total_page = ceil($count / $page_size);
|
|
|
|
for ($page = 1; $page <= $total_page; $page++) {
|
|
|
|
if ($is_stop) {
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
$url_page = $api_url . '?' . http_build_query(['w' => 'visit_list', 'page' => $page, 'pagesize' => $page_size]);
|
|
|
|
$data_page = curl_c($url_page);
|
|
|
|
if (isset($data_page['data']) && $data_page['data']) {
|
|
|
|
$items = $data_page['data'];
|
|
|
|
$model = new Visit();
|
|
|
|
$insert = [];
|
|
|
|
foreach ($items as $item) {
|
|
|
|
if (isset($item['id']) && $item['id']) {
|
|
|
|
if ($item['id'] > $task->max_id) {
|
|
|
|
$url_arr = parse_url($item['request'] ?? '');
|
|
|
|
$insert[] = [
|
|
|
|
'url' => $item['request'] ?? '',
|
|
|
|
'referrer_url' => $item['referrer'] ?? '',
|
|
|
|
'device_port' => isset($item['is_moblie']) && $item['is_moblie'] == 1 ? 2 : 1,
|
|
|
|
'country' => $item['ip_area'] ?? '',
|
|
|
|
'ip' => $item['ip'] ?? '',
|
|
|
|
'depth' => $item['pv'],
|
|
|
|
'domain' => $url_arr['host'] ?? '',
|
|
|
|
'is_inquiry' => $item['is_cf'] ?? 0,
|
|
|
|
'created_at' => date('Y-m-d H:i:s', isset($item['update']) && $item['update'] ? $item['update'] : time()),
|
|
|
|
'updated_at' => date('Y-m-d H:i:s', isset($item['update']) && $item['update'] ? $item['update'] : time()),
|
|
|
|
'updated_date' => date('Y-m-d', isset($item['c_time']) && $item['c_time'] ? strtotime($item['c_time']) : time()),
|
|
|
|
'original_id' => $item['id'],
|
|
|
|
];
|
|
|
|
} else {
|
|
|
|
$is_stop = 1;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if ($insert) {
|
|
|
|
try {
|
|
|
|
$model->insert($insert);
|
|
|
|
} catch (\Exception $e) {
|
|
|
|
echo 'date:' . date('Y-m-d H:i:s') . ', task_id: ' . $task->id . ',page: ' . $page . ' error: ' . $e->getMessage() . PHP_EOL;
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if ($max_id) {
|
|
|
|
$task->max_id = $max_id;
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
//访问明细
|
|
|
|
$url = $api_url . '?' . http_build_query(['w' => 'visit_detail_list', 'page' => 1, 'pagesize' => 1]);
|
|
|
|
$data = curl_c($url);
|
|
|
|
if (isset($data['count']) && $data['count'] > 0) {
|
|
|
|
$count = $data['count'];
|
|
|
|
$max_id = $data['data'][0]['id'] ?? 0;
|
|
|
|
$is_stop = 0;
|
|
|
|
|
|
|
|
$total_page = ceil($count / $page_size);
|
|
|
|
for ($page = 1; $page <= $total_page; $page++) {
|
|
|
|
if ($is_stop) {
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
$url_page = $api_url . '?' . http_build_query(['w' => 'visit_detail_list', 'page' => $page, 'pagesize' => $page_size]);
|
|
|
|
$data_page = curl_c($url_page);
|
|
|
|
if (isset($data_page['data']) && $data_page['data']) {
|
|
|
|
$items = $data_page['data'];
|
|
|
|
$model = new VisitItem();
|
|
|
|
$p_model = new Visit();
|
|
|
|
$insert = [];
|
|
|
|
foreach ($items as $item) {
|
|
|
|
if (isset($item['id']) && $item['id']) {
|
|
|
|
if ($item['id'] > $task->max_id) {
|
|
|
|
$p_info = $p_model->read(['ip' => $item['ip'] ?? '', 'updated_date' => $item['day_at'] ?? '']);
|
|
|
|
if ($p_info) {
|
|
|
|
$insert[] = [
|
|
|
|
'customer_visit_id' => $p_info['id'],
|
|
|
|
'url' => $p_info['url'],
|
|
|
|
'referrer_url' => $p_info['referrer_url'],
|
|
|
|
'device_port' => $p_info['device_port'],
|
|
|
|
'country' => $p_info['country'],
|
|
|
|
'ip' => $p_info['ip'],
|
|
|
|
'domain' => $p_info['domain'],
|
|
|
|
'created_at' => $item['time_str'] ?? $p_info['created_at'],
|
|
|
|
'updated_at' => $item['time_str'] ?? $p_info['updated_at'],
|
|
|
|
'updated_date' => $item['day_at'] ?? $p_info['updated_date'],
|
|
|
|
'original_id' => $item['id'],
|
|
|
|
];
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
$is_stop = 1;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if ($insert) {
|
|
|
|
try {
|
|
|
|
$model->insert($insert);
|
|
|
|
} catch (\Exception $e) {
|
|
|
|
echo 'date:' . date('Y-m-d H:i:s') . ', task_id: ' . $task->id . ',page: ' . $page . ' error: ' . $e->getMessage() . PHP_EOL;
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if ($max_id) {
|
|
|
|
$task->max_id = $max_id;
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
//关闭数据库
|
|
|
|
DB::disconnect('custom_mysql');
|
|
|
|
|
|
|
|
$task->status = UpdateVisit::STATUS_COM;//同步完成
|
|
|
|
$task->save();
|
|
|
|
|
|
|
|
echo 'date:' . date('Y-m-d H:i:s') . ', task_id: ' . $task->id . ', task_type: ' . $api_type . ', update end ' . PHP_EOL;
|
|
|
|
|
|
|
|
sleep(2);
|
|
|
|
}
|
|
|
|
|
|
|
|
//获取任务
|
|
|
|
protected function get_task()
|
|
|
|
{
|
|
|
|
$key = 'console_visit_task';
|
|
|
|
$task_id = Redis::rpop($key);
|
|
|
|
if ($task_id) {
|
|
|
|
return $task_id;
|
|
|
|
}
|
|
|
|
|
|
|
|
$task_list = UpdateVisit::where('status', UpdateVisit::STATUS_UN)->orderBy('sort', 'asc')->orderBy('project_id', 'asc')->limit(20)->get();
|
|
|
|
if ($task_list->count() == 0) {
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
|
|
|
foreach ($task_list as $value) {
|
|
|
|
Redis::lpush($key, $value->id);
|
|
|
|
}
|
|
|
|
|
|
|
|
$task_id = Redis::rpop($key);
|
|
|
|
return $task_id;
|
|
|
|
}
|
|
|
|
} |
...
|
...
|
|
