周海龙 / globalso · 提交

转到一个项目

作者 周海龙
提交 541b36f09ad8ee9207610a081480a1ae3e320e92 2 个父辈 2b34f538 8f9ba861

合并分支 'zhl' 到 'master' 

Zhl



查看合并请求 !1172
内嵌 并排对比
正在显示 4 个修改的文件 包含 230 行增加 和 84 行删除
@@ -19,6 +19,7 @@ use App\Services\ProjectServer; @@ -19,6 +19,7 @@ use App\Services\ProjectServer;
19 use Illuminate\Console\Command; 19 use Illuminate\Console\Command;
20 use Illuminate\Support\Facades\DB; 20 use Illuminate\Support\Facades\DB;
21 use Illuminate\Support\Facades\Log; 21 use Illuminate\Support\Facades\Log;
  22 +use Illuminate\Support\Facades\Redis;
22 23
23 /** 24 /**
24 * Class SyncInquiryProject 25 * Class SyncInquiryProject
@@ -38,7 +39,7 @@ class SyncInquiryProject extends Command @@ -38,7 +39,7 @@ class SyncInquiryProject extends Command
38 * 39 *
39 * @var string 40 * @var string
40 */ 41 */
41 - protected $description = '同步询盘信息：项目以及对应路由，'; 42 + protected $description = '同步询盘信息：项目';
42 43
43 /** 44 /**
44 * Create a new command instance. 45 * Create a new command instance.
@@ -61,8 +62,6 @@ class SyncInquiryProject extends Command @@ -61,8 +62,6 @@ class SyncInquiryProject extends Command
61 $this->syncGloV5(); 62 $this->syncGloV5();
62 echo '开始同步v6' . PHP_EOL; 63 echo '开始同步v6' . PHP_EOL;
63 $this->syncGloV6(); 64 $this->syncGloV6();
64 - echo '删除过期数据' . PHP_EOL;  
65 - $this->deleteExpire();  
66 return true; 65 return true;
67 } 66 }
68 67
@@ -72,73 +71,50 @@ class SyncInquiryProject extends Command @@ -72,73 +71,50 @@ class SyncInquiryProject extends Command
72 */ 71 */
73 public function syncGloV5() 72 public function syncGloV5()
74 { 73 {
  74 + $date = intval(date('Ymd'));
75 $result = http_get('https://www.quanqiusou.cn/extend_api/webs/globalso_v5_tg.php'); 75 $result = http_get('https://www.quanqiusou.cn/extend_api/webs/globalso_v5_tg.php');
76 $channel = Channel::pluck('contact_mobile', 'id')->toArray(); 76 $channel = Channel::pluck('contact_mobile', 'id')->toArray();
77 foreach ($result as $key=>$val) 77 foreach ($result as $key=>$val)
78 { 78 {
79 echo '同步项目：' . $val['postid'] . ' - ' . $val['company'] . PHP_EOL; 79 echo '同步项目：' . $val['postid'] . ' - ' . $val['company'] . PHP_EOL;
80 // 记录渠道商, 如果渠道有问题， 记录日志， 跳过当前项目， 处理数据后， 第二天再重新同步 80 // 记录渠道商, 如果渠道有问题， 记录日志， 跳过当前项目， 处理数据后， 第二天再重新同步
81 - $channel_flag = false;  
82 $channel_tmp = []; 81 $channel_tmp = [];
83 foreach ($val['agent'] as $v) { 82 foreach ($val['agent'] as $v) {
84 $channel_id = array_search($v, $channel); 83 $channel_id = array_search($v, $channel);
  84 + // 代理信息 只要找到一个， 就需要继续记录
85 if (empty($channel_id)) { 85 if (empty($channel_id)) {
86 $this->log('ERROR 代理商找不到不记录当前项目, 代理商手机号码：' . $v); 86 $this->log('ERROR 代理商找不到不记录当前项目, 代理商手机号码：' . $v);
87 - $channel_flag = true;  
88 - break; 87 + continue;
89 } 88 }
90 array_push($channel_tmp, $channel_id); 89 array_push($channel_tmp, $channel_id);
91 } 90 }
92 91
93 - if ($channel_flag || empty($channel_tmp)) { 92 + if (empty($channel_tmp)) {
94 $this->log('ERROR 代理商找不到不记录当前项目, 项目ID：' . $val['postid']); 93 $this->log('ERROR 代理商找不到不记录当前项目, 项目ID：' . $val['postid']);
95 continue; 94 continue;
96 } 95 }
97 96
98 - // 保存项目 以及路由  
99 - $project = InquiryProject::saveProject($val['v'], $val['postid'], $val['company'], implode(',', $channel_tmp), $val['main_url'], $val['is_split'], $val['test_url']); 97 + // 保存项目 以及 获取路由信息
  98 + $project = InquiryProject::saveProject($date, $val['v'], $val['postid'], $val['company'], implode(',', $channel_tmp), $val['main_url'], $val['is_split'], $val['test_url']);
100 $route_domain = $val['is_split'] && $val['test_url'] ? $val['test_url'] : $val['main_url']; 99 $route_domain = $val['is_split'] && $val['test_url'] ? $val['test_url'] : $val['main_url'];
101 - $this->syncGloV5Route($project->id, $route_domain);  
102 - }  
103 - return true;  
104 - }  
105 -  
106 - /**  
107 - * 同步v4 v5项目路由  
108 - * @param $project_id  
109 - * @param $route_domain  
110 - * @return bool  
111 - */  
112 - public function syncGloV5Route($project_id, $route_domain)  
113 - {  
114 - try{  
115 - $result = http_get($route_domain . 'k_u_api.php');  
116 - if (empty($result)) {  
117 - $this->log('syncGloV5Route 未获取到路由信息：' . $project_id . ', 路由获取地址：' . $route_domain . 'k_u_api.php');  
118 - return false;  
119 - }  
120 - } catch (\Exception $e) {  
121 - $this->log('syncGloV5Route 未获取到路由信息：' . $project_id . ', 路由获取地址：' . $route_domain . 'k_u_api.php' . ', 错误信息：' . $e->getMessage());  
122 - echo 'syncGloV5Route 未获取到路由信息：' . $project_id . ', 路由获取地址：' . $route_domain . 'k_u_api.php' . ', 错误信息：' . $e->getMessage() . PHP_EOL;  
123 - return false;  
124 - }  
125 -  
126 - foreach ($result as $key=>$val) {  
127 - try {  
128 - $tmp = explode('|', $val);  
129 - $url_tmp = parse_url($tmp[0]);  
130 - $route = trim($url_tmp['path'], '/');  
131 - $title = str_replace('+', ' ', $tmp[1]);  
132 - if (strlen($title) > 200 || strlen($route) > 200) {  
133 - $this->log('syncGloV5Route 路由或标题过长，无效记录');  
134 - continue; 100 + try{
  101 + $result = http_get($route_domain . 'k_u_api.php');
  102 + if (empty($result)) {
  103 + $this->log('syncGloV5Route 未获取到路由信息：' . $project->id . ', 路由获取地址：' . $route_domain . 'k_u_api.php');
  104 + return false;
135 } 105 }
136 - InquiryProjectRoute::saveProjectRoute($project_id, $title, $route); 106 + file_put_contents(storage_path('logs/sync_inquiry_project_route/' . $project->id . '.json'), json_encode($result));
  107 + $this->pushTask($project->id);
137 } catch (\Exception $e) { 108 } catch (\Exception $e) {
138 - $this->log('syncGloV5Route 解析路径：' . $val . ', 错误信息：' . $e->getMessage());  
139 - echo 'syncGloV5Route 解析路径：' . $val . ', 错误信息：' . $e->getMessage() . PHP_EOL; 109 + $this->log('syncGloV5Route 未获取到路由信息：' . $project->id . ', 路由获取地址：' . $route_domain . 'k_u_api.php' . ', 错误信息：' . $e->getMessage());
  110 + echo 'syncGloV5Route 未获取到路由信息：' . $project->id . ', 路由获取地址：' . $route_domain . 'k_u_api.php' . ', 错误信息：' . $e->getMessage() . PHP_EOL;
  111 + return false;
140 } 112 }
141 } 113 }
  114 +
  115 + $ids = InquiryProject::where('date', '<', $date)->where('version', '<', InquiryProject::VERSION_SIX)->pluck('id')->toArray();
  116 + $this->deleteExpire($ids);
  117 +
142 return true; 118 return true;
143 } 119 }
144 120
@@ -148,6 +124,7 @@ class SyncInquiryProject extends Command @@ -148,6 +124,7 @@ class SyncInquiryProject extends Command
148 */ 124 */
149 public function syncGloV6() 125 public function syncGloV6()
150 { 126 {
  127 + $date = intval(date('Ymd'));
151 // 获取优化中台项目 128 // 获取优化中台项目
152 $project = Project::leftJoin('gl_project_online_check', 'gl_project.id', '=', 'gl_project_online_check.project_id') 129 $project = Project::leftJoin('gl_project_online_check', 'gl_project.id', '=', 'gl_project_online_check.project_id')
153 ->where('gl_project.type', Project::TYPE_TWO) 130 ->where('gl_project.type', Project::TYPE_TWO)
@@ -165,55 +142,40 @@ class SyncInquiryProject extends Command @@ -165,55 +142,40 @@ class SyncInquiryProject extends Command
165 // 过滤暂停优化项目, 映射类型2， model没有定义常量 142 // 过滤暂停优化项目, 映射类型2， model没有定义常量
166 if (in_array(2, $val->level)) 143 if (in_array(2, $val->level))
167 continue; 144 continue;
168 - $project = InquiryProject::saveProject(InquiryProject::VERSION_SIX, $val->id, $val->title, $val->channel['channel_id'], $domains[$val->id]);  
169 - $this->syncGloV6Route($project->id, $val->id); 145 + $project = InquiryProject::saveProject($date,InquiryProject::VERSION_SIX, $val->id, $val->title, $val->channel['channel_id'], $domains[$val->id]);
  146 + $this->pushTask($project->id);
170 } 147 }
  148 +
  149 + $ids = InquiryProject::where('date', '<', $date)->where('version', '=', InquiryProject::VERSION_SIX)->pluck('id')->toArraty();
  150 + $this->deleteExpire($ids);
  151 +
171 return true; 152 return true;
172 } 153 }
173 154
  155 +
174 /** 156 /**
175 - * 同步v6项目路由  
176 - * @param $project_id  
177 - * @param $origin_project_id 157 + * 删除过期数据， 非当日更新数据， 都删除， 误删第二天再重新同步更新
  158 + * @param $ids
178 * @return bool 159 * @return bool
179 */ 160 */
180 - public function syncGloV6Route($project_id, $origin_project_id) 161 + public function deleteExpire($ids)
181 { 162 {
182 - ProjectServer::useProject($origin_project_id);  
183 - // TODO 产品分类标题、路由， 产品标题、路由， 同步到路由表  
184 - $category = Category::where('status', Category::STATUS_ACTIVE)->get(['title', 'route']);  
185 - foreach ($category as $key=>$val) {  
186 - InquiryProjectRoute::saveProjectRoute($project_id, $val->title, $val->route);  
187 - }  
188 -  
189 - // 产品数量会比较多， 所以使用分页 同步数据  
190 - $id = 0;  
191 - while (true) {  
192 - echo '同步项目路由：' . $id . PHP_EOL;  
193 - $product = Product::where('status', Product::STATUS_ON)->where('id', '>', $id)->orderBy('id', 'asc')->limit(1000)->get(['id', 'title', 'route']);  
194 - if ($product->isEmpty())  
195 - break;  
196 -  
197 - foreach ($product as $key=>$val) {  
198 - $id = $val->id;  
199 - InquiryProjectRoute::saveProjectRoute($project_id, $val->title, $val->route);  
200 - }  
201 - }  
202 -  
203 - DB::disconnect('custom_mysql'); 163 + if (empty($ids))
  164 + return true;
  165 + $project_num = InquiryProject::whereIn('id', $ids)->delete();
  166 + $project_route_num = InquiryProjectRoute::whereIn('project_id', $ids)->delete();
  167 + $this->log('删除过期项目数量：' . $project_num . ', 删除过期项目路由数量：' . $project_route_num);
204 return true; 168 return true;
205 } 169 }
206 170
207 /** 171 /**
208 - * 删除过期数据， 非当日更新数据， 都删除， 误删第二天再重新同步更新 172 + * 往任务队列中生产数据
  173 + * @param $id
209 * @return bool 174 * @return bool
210 */ 175 */
211 - public function deleteExpire() 176 + public function pushTask($id)
212 { 177 {
213 - $date = intval(date('Ymd'));  
214 - $project_num = InquiryProject::where('date', '<', $date)->delete();  
215 - $project_route_num = InquiryProjectRoute::where('date', '<', $date)->delete();  
216 - $this->log('删除过期项目数量：' . $project_num . ', 删除过期路由数量：' . $project_route_num); 178 + Redis::lPush('sync_inquiry_project_route_task', $id);
217 return true; 179 return true;
218 } 180 }
219 181
  1 +<?php
  2 +/**
  3 + * Created by PhpStorm.
  4 + * User: zhl
  5 + * Date: 2025/2/18
  6 + * Time: 17:10
  7 + */
  8 +namespace App\Console\Commands\Inquiry;
  9 +
  10 +use App\Models\Channel\Channel;
  11 +use App\Models\Domain\DomainInfo;
  12 +use App\Models\Inquiry\InquiryProject;
  13 +use App\Models\Inquiry\InquiryProjectRoute;
  14 +use App\Models\Product\Category;
  15 +use App\Models\Product\Product;
  16 +use App\Models\Project\OnlineCheck;
  17 +use App\Models\Project\Project;
  18 +use App\Services\ProjectServer;
  19 +use Illuminate\Console\Command;
  20 +use Illuminate\Support\Facades\DB;
  21 +use Illuminate\Support\Facades\Log;
  22 +use Illuminate\Support\Facades\Redis;
  23 +
  24 +/**
  25 + * Class SyncInquiryProjectRoute
  26 + * @package App\Console\Commands\Inquiry
  27 + */
  28 +class SyncInquiryProjectRoute extends Command
  29 +{
  30 + /**
  31 + * The name and signature of the console command.
  32 + *
  33 + * @var string
  34 + */
  35 + protected $signature = 'sync_inquiry_project_route';
  36 +
  37 + /**
  38 + * The console command description.
  39 + *
  40 + * @var string
  41 + */
  42 + protected $description = '同步询盘信息：项目对应路由，';
  43 +
  44 + /**
  45 + * Create a new command instance.
  46 + *
  47 + * @return void
  48 + */
  49 + public function __construct()
  50 + {
  51 + parent::__construct();
  52 + }
  53 +
  54 + /**
  55 + * 同步优化项目及路由
  56 + * TODO 同步v4 v5 v6项目以及路由， 删除过期项目及路由
  57 + * @return bool
  58 + */
  59 + public function handle()
  60 + {
  61 + while (true) {
  62 + $sync_id = Redis::rpop('sync_inquiry_project_route_task');
  63 + if (empty($sync_id)) {
  64 + sleep(60);
  65 + continue;
  66 + }
  67 +
  68 + $task = InquiryProject::where(['id' => $sync_id])->first();
  69 + if (empty($task))
  70 + continue;
  71 +
  72 + // 同步对应项目路由， 以及删除过期路由
  73 + if ($task->version == InquiryProject::VERSION_SIX){
  74 + $this->syncGloV6Route($task);
  75 + } else {
  76 + $this->syncGloV5Route($task);
  77 + }
  78 +
  79 + }
  80 + return true;
  81 + }
  82 +
  83 + /**
  84 + * 同步v4 v5项目路由
  85 + * @param $task
  86 + * @return bool
  87 + */
  88 + public function syncGloV5Route($task)
  89 + {
  90 + $date = intval(date('Ymd'));
  91 +
  92 + $result = file_get_contents(storage_path('logs/sync_inquiry_project_route/' . $task->id . '.json'));
  93 + $result = json_decode($result, true);
  94 + if (empty($result)) {
  95 + // 未获取到数据 删除当前项目过期路由
  96 + $this->deleteExpire($task->id, $date);
  97 + $this->log('syncGloV5Route 未获取到路由信息：' . $task->id . ', 路由获取地址：' . ($task->is_split && $task->test_url ? $task->test_url : $task->main_url) . 'k_u_api.php');
  98 + return false;
  99 + }
  100 +
  101 + foreach ($result as $key=>$val) {
  102 + try {
  103 + $tmp = explode('|', $val);
  104 + $url_tmp = parse_url($tmp[0]);
  105 + $route = trim($url_tmp['path'], '/');
  106 + $title = str_replace('+', ' ', $tmp[1]);
  107 + if (strlen($title) > 200 || strlen($route) > 200) {
  108 + $this->log('syncGloV5Route 路由或标题过长，无效记录');
  109 + continue;
  110 + }
  111 + InquiryProjectRoute::saveProjectRoute($task->id, $title, $route, $date);
  112 + } catch (\Exception $e) {
  113 + $this->log('syncGloV5Route 解析路径：' . $val . ', 错误信息：' . $e->getMessage());
  114 + echo 'syncGloV5Route 解析路径：' . $val . ', 错误信息：' . $e->getMessage() . PHP_EOL;
  115 + }
  116 + }
  117 +
  118 + // 删除当前项目过期路由
  119 + $this->deleteExpire($task->id, $date);
  120 + return true;
  121 + }
  122 +
  123 + /**
  124 + * 同步v6项目路由
  125 + * @param $task
  126 + * @return bool
  127 + */
  128 + public function syncGloV6Route($task)
  129 + {
  130 + $date = intval(date('Ymd'));
  131 + ProjectServer::useProject($task->primary_id);
  132 + // TODO 产品分类标题、路由， 产品标题、路由， 同步到路由表
  133 + $category = Category::where('status', Category::STATUS_ACTIVE)->get(['title', 'route']);
  134 + foreach ($category as $key=>$val) {
  135 + InquiryProjectRoute::saveProjectRoute($task->id, $val->title, $val->route, $date);
  136 + }
  137 +
  138 + // 产品数量会比较多， 所以使用分页 同步数据
  139 + $id = 0;
  140 + while (true) {
  141 + echo '同步项目路由：' . $id . PHP_EOL;
  142 + $product = Product::where('status', Product::STATUS_ON)->where('id', '>', $id)->orderBy('id', 'asc')->limit(1000)->get(['id', 'title', 'route']);
  143 + if ($product->isEmpty())
  144 + break;
  145 +
  146 + foreach ($product as $key=>$val) {
  147 + $id = $val->id;
  148 + InquiryProjectRoute::saveProjectRoute($task->id, $val->title, $val->route, $date);
  149 + }
  150 + }
  151 +
  152 + DB::disconnect('custom_mysql');
  153 +
  154 + // 删除当前项目过期路由
  155 + $this->deleteExpire($task->id, $date);
  156 +
  157 + return true;
  158 + }
  159 +
  160 + /**
  161 + * 删除过期数据， 非当前更新数据， 都删除， 误删第二天再重新同步更新
  162 + * @param $project_id
  163 + * @param $date
  164 + * @return bool
  165 + */
  166 + public function deleteExpire($project_id, $date)
  167 + {
  168 + $project_route_num = InquiryProjectRoute::where(['project_id' => $project_id])->where('date', '<', $date)->delete();
  169 + $this->log('项目ID：' . $project_id . ', 删除过期路由数量：' . $project_route_num);
  170 + return true;
  171 + }
  172 +
  173 + /**
  174 + * 输出日志到特定的文件内， 这个文件需要定时排除内容
  175 + * @param $message
  176 + * @return bool
  177 + */
  178 + public function log($message)
  179 + {
  180 + $message = date('Y-m-d H:i:s') . ' ' . $message . PHP_EOL;
  181 + file_put_contents(storage_path('logs/zhl/output') . date('Y-m-d') . '.log', $message, FILE_APPEND);
  182 + return true;
  183 + }
  184 +}
@@ -27,6 +27,7 @@ class InquiryProject extends Base @@ -27,6 +27,7 @@ class InquiryProject extends Base
27 27
28 /** 28 /**
29 * 保存项目信息 29 * 保存项目信息
  30 + * @param $date
30 * @param $version 31 * @param $version
31 * @param $primary_id 32 * @param $primary_id
32 * @param $project 33 * @param $project
@@ -36,9 +37,8 @@ class InquiryProject extends Base @@ -36,9 +37,8 @@ class InquiryProject extends Base
36 * @param string $test_domain 37 * @param string $test_domain
37 * @return InquiryProject 38 * @return InquiryProject
38 */ 39 */
39 - public static function saveProject($version, $primary_id, $project, $channel, $domain, $is_split = 0, $test_domain = '') 40 + public static function saveProject($date, $version, $primary_id, $project, $channel, $domain, $is_split = 0, $test_domain = '')
40 { 41 {
41 - $date = date('Ymd');  
42 $log = self::where(compact('version', 'primary_id'))->first(); 42 $log = self::where(compact('version', 'primary_id'))->first();
43 if ($log) { 43 if ($log) {
44 $log->date = $date; 44 $log->date = $date;
@@ -25,11 +25,11 @@ class InquiryProjectRoute extends Base @@ -25,11 +25,11 @@ class InquiryProjectRoute extends Base
25 * @param int $project_id gl_inquiry_project 表ID 25 * @param int $project_id gl_inquiry_project 表ID
26 * @param $title 26 * @param $title
27 * @param $route 27 * @param $route
  28 + * @param $date
28 * @return InquiryProjectRoute 29 * @return InquiryProjectRoute
29 */ 30 */
30 - public static function saveProjectRoute($project_id, $title, $route) 31 + public static function saveProjectRoute($project_id, $title, $route, $date)
31 { 32 {
32 - $date = date('Ymd');  
33 $log = self::where(compact('project_id', 'title'))->first(); 33 $log = self::where(compact('project_id', 'title'))->first();
34 if ($log) { 34 if ($log) {
35 $log->date = $date; 35 $log->date = $date;