周海龙 / globalso · 提交

转到一个项目

作者 周海龙
提交 541b36f09ad8ee9207610a081480a1ae3e320e92 2 个父辈 2b34f538 8f9ba861

合并分支 'zhl' 到 'master' 

Zhl



查看合并请求 !1172
内嵌 并排对比
正在显示 4 个修改的文件 包含 230 行增加 和 84 行删除
... ... @@ -19,6 +19,7 @@ use App\Services\ProjectServer;
use Illuminate\Console\Command;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Log;
use Illuminate\Support\Facades\Redis;
/**
* Class SyncInquiryProject
... ... @@ -38,7 +39,7 @@ class SyncInquiryProject extends Command
*
* @var string
*/
protected $description = '同步询盘信息：项目以及对应路由，';
protected $description = '同步询盘信息：项目';
/**
* Create a new command instance.
... ... @@ -61,8 +62,6 @@ class SyncInquiryProject extends Command
$this->syncGloV5();
echo '开始同步v6' . PHP_EOL;
$this->syncGloV6();
echo '删除过期数据' . PHP_EOL;
$this->deleteExpire();
return true;
}
... ... @@ -72,73 +71,50 @@ class SyncInquiryProject extends Command
*/
public function syncGloV5()
{
$date = intval(date('Ymd'));
$result = http_get('https://www.quanqiusou.cn/extend_api/webs/globalso_v5_tg.php');
$channel = Channel::pluck('contact_mobile', 'id')->toArray();
foreach ($result as $key=>$val)
{
echo '同步项目：' . $val['postid'] . ' - ' . $val['company'] . PHP_EOL;
// 记录渠道商, 如果渠道有问题， 记录日志， 跳过当前项目， 处理数据后， 第二天再重新同步
$channel_flag = false;
$channel_tmp = [];
foreach ($val['agent'] as $v) {
$channel_id = array_search($v, $channel);
// 代理信息 只要找到一个， 就需要继续记录
if (empty($channel_id)) {
$this->log('ERROR 代理商找不到不记录当前项目, 代理商手机号码：' . $v);
$channel_flag = true;
break;
continue;
}
array_push($channel_tmp, $channel_id);
}
if ($channel_flag || empty($channel_tmp)) {
if (empty($channel_tmp)) {
$this->log('ERROR 代理商找不到不记录当前项目, 项目ID：' . $val['postid']);
continue;
}
// 保存项目 以及路由
$project = InquiryProject::saveProject($val['v'], $val['postid'], $val['company'], implode(',', $channel_tmp), $val['main_url'], $val['is_split'], $val['test_url']);
// 保存项目 以及 获取路由信息
$project = InquiryProject::saveProject($date, $val['v'], $val['postid'], $val['company'], implode(',', $channel_tmp), $val['main_url'], $val['is_split'], $val['test_url']);
$route_domain = $val['is_split'] && $val['test_url'] ? $val['test_url'] : $val['main_url'];
$this->syncGloV5Route($project->id, $route_domain);
}
return true;
}
/**
* 同步v4 v5项目路由
* @param $project_id
* @param $route_domain
* @return bool
*/
public function syncGloV5Route($project_id, $route_domain)
{
try{
$result = http_get($route_domain . 'k_u_api.php');
if (empty($result)) {
$this->log('syncGloV5Route 未获取到路由信息：' . $project_id . ', 路由获取地址：' . $route_domain . 'k_u_api.php');
return false;
}
} catch (\Exception $e) {
$this->log('syncGloV5Route 未获取到路由信息：' . $project_id . ', 路由获取地址：' . $route_domain . 'k_u_api.php' . ', 错误信息：' . $e->getMessage());
echo 'syncGloV5Route 未获取到路由信息：' . $project_id . ', 路由获取地址：' . $route_domain . 'k_u_api.php' . ', 错误信息：' . $e->getMessage() . PHP_EOL;
return false;
}
foreach ($result as $key=>$val) {
try {
$tmp = explode('|', $val);
$url_tmp = parse_url($tmp[0]);
$route = trim($url_tmp['path'], '/');
$title = str_replace('+', ' ', $tmp[1]);
if (strlen($title) > 200 || strlen($route) > 200) {
$this->log('syncGloV5Route 路由或标题过长，无效记录');
continue;
try{
$result = http_get($route_domain . 'k_u_api.php');
if (empty($result)) {
$this->log('syncGloV5Route 未获取到路由信息：' . $project->id . ', 路由获取地址：' . $route_domain . 'k_u_api.php');
return false;
}
InquiryProjectRoute::saveProjectRoute($project_id, $title, $route);
file_put_contents(storage_path('logs/sync_inquiry_project_route/' . $project->id . '.json'), json_encode($result));
$this->pushTask($project->id);
} catch (\Exception $e) {
$this->log('syncGloV5Route 解析路径：' . $val . ', 错误信息：' . $e->getMessage());
echo 'syncGloV5Route 解析路径：' . $val . ', 错误信息：' . $e->getMessage() . PHP_EOL;
$this->log('syncGloV5Route 未获取到路由信息：' . $project->id . ', 路由获取地址：' . $route_domain . 'k_u_api.php' . ', 错误信息：' . $e->getMessage());
echo 'syncGloV5Route 未获取到路由信息：' . $project->id . ', 路由获取地址：' . $route_domain . 'k_u_api.php' . ', 错误信息：' . $e->getMessage() . PHP_EOL;
return false;
}
}
$ids = InquiryProject::where('date', '<', $date)->where('version', '<', InquiryProject::VERSION_SIX)->pluck('id')->toArray();
$this->deleteExpire($ids);
return true;
}
... ... @@ -148,6 +124,7 @@ class SyncInquiryProject extends Command
*/
public function syncGloV6()
{
$date = intval(date('Ymd'));
// 获取优化中台项目
$project = Project::leftJoin('gl_project_online_check', 'gl_project.id', '=', 'gl_project_online_check.project_id')
->where('gl_project.type', Project::TYPE_TWO)
... ... @@ -165,55 +142,40 @@ class SyncInquiryProject extends Command
// 过滤暂停优化项目, 映射类型2， model没有定义常量
if (in_array(2, $val->level))
continue;
$project = InquiryProject::saveProject(InquiryProject::VERSION_SIX, $val->id, $val->title, $val->channel['channel_id'], $domains[$val->id]);
$this->syncGloV6Route($project->id, $val->id);
$project = InquiryProject::saveProject($date,InquiryProject::VERSION_SIX, $val->id, $val->title, $val->channel['channel_id'], $domains[$val->id]);
$this->pushTask($project->id);
}
$ids = InquiryProject::where('date', '<', $date)->where('version', '=', InquiryProject::VERSION_SIX)->pluck('id')->toArraty();
$this->deleteExpire($ids);
return true;
}
/**
* 同步v6项目路由
* @param $project_id
* @param $origin_project_id
* 删除过期数据， 非当日更新数据， 都删除， 误删第二天再重新同步更新
* @param $ids
* @return bool
*/
public function syncGloV6Route($project_id, $origin_project_id)
public function deleteExpire($ids)
{
ProjectServer::useProject($origin_project_id);
// TODO 产品分类标题、路由， 产品标题、路由， 同步到路由表
$category = Category::where('status', Category::STATUS_ACTIVE)->get(['title', 'route']);
foreach ($category as $key=>$val) {
InquiryProjectRoute::saveProjectRoute($project_id, $val->title, $val->route);
}
// 产品数量会比较多， 所以使用分页 同步数据
$id = 0;
while (true) {
echo '同步项目路由：' . $id . PHP_EOL;
$product = Product::where('status', Product::STATUS_ON)->where('id', '>', $id)->orderBy('id', 'asc')->limit(1000)->get(['id', 'title', 'route']);
if ($product->isEmpty())
break;
foreach ($product as $key=>$val) {
$id = $val->id;
InquiryProjectRoute::saveProjectRoute($project_id, $val->title, $val->route);
}
}
DB::disconnect('custom_mysql');
if (empty($ids))
return true;
$project_num = InquiryProject::whereIn('id', $ids)->delete();
$project_route_num = InquiryProjectRoute::whereIn('project_id', $ids)->delete();
$this->log('删除过期项目数量：' . $project_num . ', 删除过期项目路由数量：' . $project_route_num);
return true;
}
/**
* 删除过期数据， 非当日更新数据， 都删除， 误删第二天再重新同步更新
* 往任务队列中生产数据
* @param $id
* @return bool
*/
public function deleteExpire()
public function pushTask($id)
{
$date = intval(date('Ymd'));
$project_num = InquiryProject::where('date', '<', $date)->delete();
$project_route_num = InquiryProjectRoute::where('date', '<', $date)->delete();
$this->log('删除过期项目数量：' . $project_num . ', 删除过期路由数量：' . $project_route_num);
Redis::lPush('sync_inquiry_project_route_task', $id);
return true;
}
... ...
<?php
/**
* Created by PhpStorm.
* User: zhl
* Date: 2025/2/18
* Time: 17:10
*/
namespace App\Console\Commands\Inquiry;
use App\Models\Channel\Channel;
use App\Models\Domain\DomainInfo;
use App\Models\Inquiry\InquiryProject;
use App\Models\Inquiry\InquiryProjectRoute;
use App\Models\Product\Category;
use App\Models\Product\Product;
use App\Models\Project\OnlineCheck;
use App\Models\Project\Project;
use App\Services\ProjectServer;
use Illuminate\Console\Command;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Log;
use Illuminate\Support\Facades\Redis;
/**
* Class SyncInquiryProjectRoute
* @package App\Console\Commands\Inquiry
*/
class SyncInquiryProjectRoute extends Command
{
/**
* The name and signature of the console command.
*
* @var string
*/
protected $signature = 'sync_inquiry_project_route';
/**
* The console command description.
*
* @var string
*/
protected $description = '同步询盘信息：项目对应路由，';
/**
* Create a new command instance.
*
* @return void
*/
public function __construct()
{
parent::__construct();
}
/**
* 同步优化项目及路由
* TODO 同步v4 v5 v6项目以及路由， 删除过期项目及路由
* @return bool
*/
public function handle()
{
while (true) {
$sync_id = Redis::rpop('sync_inquiry_project_route_task');
if (empty($sync_id)) {
sleep(60);
continue;
}
$task = InquiryProject::where(['id' => $sync_id])->first();
if (empty($task))
continue;
// 同步对应项目路由， 以及删除过期路由
if ($task->version == InquiryProject::VERSION_SIX){
$this->syncGloV6Route($task);
} else {
$this->syncGloV5Route($task);
}
}
return true;
}
/**
* 同步v4 v5项目路由
* @param $task
* @return bool
*/
public function syncGloV5Route($task)
{
$date = intval(date('Ymd'));
$result = file_get_contents(storage_path('logs/sync_inquiry_project_route/' . $task->id . '.json'));
$result = json_decode($result, true);
if (empty($result)) {
// 未获取到数据 删除当前项目过期路由
$this->deleteExpire($task->id, $date);
$this->log('syncGloV5Route 未获取到路由信息：' . $task->id . ', 路由获取地址：' . ($task->is_split && $task->test_url ? $task->test_url : $task->main_url) . 'k_u_api.php');
return false;
}
foreach ($result as $key=>$val) {
try {
$tmp = explode('|', $val);
$url_tmp = parse_url($tmp[0]);
$route = trim($url_tmp['path'], '/');
$title = str_replace('+', ' ', $tmp[1]);
if (strlen($title) > 200 || strlen($route) > 200) {
$this->log('syncGloV5Route 路由或标题过长，无效记录');
continue;
}
InquiryProjectRoute::saveProjectRoute($task->id, $title, $route, $date);
} catch (\Exception $e) {
$this->log('syncGloV5Route 解析路径：' . $val . ', 错误信息：' . $e->getMessage());
echo 'syncGloV5Route 解析路径：' . $val . ', 错误信息：' . $e->getMessage() . PHP_EOL;
}
}
// 删除当前项目过期路由
$this->deleteExpire($task->id, $date);
return true;
}
/**
* 同步v6项目路由
* @param $task
* @return bool
*/
public function syncGloV6Route($task)
{
$date = intval(date('Ymd'));
ProjectServer::useProject($task->primary_id);
// TODO 产品分类标题、路由， 产品标题、路由， 同步到路由表
$category = Category::where('status', Category::STATUS_ACTIVE)->get(['title', 'route']);
foreach ($category as $key=>$val) {
InquiryProjectRoute::saveProjectRoute($task->id, $val->title, $val->route, $date);
}
// 产品数量会比较多， 所以使用分页 同步数据
$id = 0;
while (true) {
echo '同步项目路由：' . $id . PHP_EOL;
$product = Product::where('status', Product::STATUS_ON)->where('id', '>', $id)->orderBy('id', 'asc')->limit(1000)->get(['id', 'title', 'route']);
if ($product->isEmpty())
break;
foreach ($product as $key=>$val) {
$id = $val->id;
InquiryProjectRoute::saveProjectRoute($task->id, $val->title, $val->route, $date);
}
}
DB::disconnect('custom_mysql');
// 删除当前项目过期路由
$this->deleteExpire($task->id, $date);
return true;
}
/**
* 删除过期数据， 非当前更新数据， 都删除， 误删第二天再重新同步更新
* @param $project_id
* @param $date
* @return bool
*/
public function deleteExpire($project_id, $date)
{
$project_route_num = InquiryProjectRoute::where(['project_id' => $project_id])->where('date', '<', $date)->delete();
$this->log('项目ID：' . $project_id . ', 删除过期路由数量：' . $project_route_num);
return true;
}
/**
* 输出日志到特定的文件内， 这个文件需要定时排除内容
* @param $message
* @return bool
*/
public function log($message)
{
$message = date('Y-m-d H:i:s') . ' ' . $message . PHP_EOL;
file_put_contents(storage_path('logs/zhl/output') . date('Y-m-d') . '.log', $message, FILE_APPEND);
return true;
}
}
\ No newline at end of file
... ...
... ... @@ -27,6 +27,7 @@ class InquiryProject extends Base
/**
* 保存项目信息
* @param $date
* @param $version
* @param $primary_id
* @param $project
... ... @@ -36,9 +37,8 @@ class InquiryProject extends Base
* @param string $test_domain
* @return InquiryProject
*/
public static function saveProject($version, $primary_id, $project, $channel, $domain, $is_split = 0, $test_domain = '')
public static function saveProject($date, $version, $primary_id, $project, $channel, $domain, $is_split = 0, $test_domain = '')
{
$date = date('Ymd');
$log = self::where(compact('version', 'primary_id'))->first();
if ($log) {
$log->date = $date;
... ...
... ... @@ -25,11 +25,11 @@ class InquiryProjectRoute extends Base
* @param int $project_id gl_inquiry_project 表ID
* @param $title
* @param $route
* @param $date
* @return InquiryProjectRoute
*/
public static function saveProjectRoute($project_id, $title, $route)
public static function saveProjectRoute($project_id, $title, $route, $date)
{
$date = date('Ymd');
$log = self::where(compact('project_id', 'title'))->first();
if ($log) {
$log->date = $date;
... ...