拆分同步脚本

zhl
app/Console/Commands/Inquiry/SyncInquiryProject.php
--- a/app/Console/Commands/Inquiry/SyncInquiryProject.php
查看文件 @285a89d
+++ b/app/Console/Commands/Inquiry/SyncInquiryProject.php
查看文件 @285a89d
@@ -19,6 +19,7 @@ use App\Services\ProjectServer;
 use Illuminate\Console\Command;
 use Illuminate\Support\Facades\DB;
 use Illuminate\Support\Facades\Log;
+use Illuminate\Support\Facades\Redis;
 /**
  * Class SyncInquiryProject
@@ -38,7 +39,7 @@ class SyncInquiryProject extends Command
      *
      * @var string
      */
-    protected $description = '同步询盘信息：项目以及对应路由，';
+    protected $description = '同步询盘信息：项目';
     /**
      * Create a new command instance.
@@ -61,8 +62,6 @@ class SyncInquiryProject extends Command
         $this->syncGloV5();
         echo '开始同步v6' . PHP_EOL;
         $this->syncGloV6();
-        echo '删除过期数据' . PHP_EOL;
-        $this->deleteExpire();
         return true;
     }
@@ -72,73 +71,47 @@ class SyncInquiryProject extends Command
      */
     public function syncGloV5()
     {
+        $date = intval(date('Ymd'));
         $result = http_get('https://www.quanqiusou.cn/extend_api/webs/globalso_v5_tg.php');
         $channel = Channel::pluck('contact_mobile', 'id')->toArray();
         foreach ($result as $key=>$val)
         {
             echo '同步项目：' . $val['postid'] . ' - ' . $val['company'] . PHP_EOL;
             // 记录渠道商, 如果渠道有问题， 记录日志， 跳过当前项目， 处理数据后， 第二天再重新同步
-            $channel_flag = false;
             $channel_tmp = [];
             foreach ($val['agent'] as $v) {
                 $channel_id = array_search($v, $channel);
+                // 代理信息 只要找到一个， 就需要继续记录
                 if (empty($channel_id)) {
                     $this->log('ERROR 代理商找不到不记录当前项目, 代理商手机号码：' . $v);
-                    $channel_flag = true;
-                    break;
+                    continue;
                 }
                 array_push($channel_tmp, $channel_id);
             }
-            if ($channel_flag || empty($channel_tmp)) {
+            if (empty($channel_tmp)) {
                 $this->log('ERROR 代理商找不到不记录当前项目, 项目ID：' . $val['postid']);
                 continue;
             }
-            // 保存项目 以及路由
-            $project = InquiryProject::saveProject($val['v'], $val['postid'], $val['company'], implode(',', $channel_tmp), $val['main_url'], $val['is_split'], $val['test_url']);
+            // 保存项目 以及 获取路由信息
+            $project = InquiryProject::saveProject($date, $val['v'], $val['postid'], $val['company'], implode(',', $channel_tmp), $val['main_url'], $val['is_split'], $val['test_url']);
             $route_domain = $val['is_split'] && $val['test_url'] ? $val['test_url'] : $val['main_url'];
-            $this->syncGloV5Route($project->id, $route_domain);
-        }
-        return true;
-    }
-
-    /**
-     * 同步v4 v5项目路由
-     * @param $project_id
-     * @param $route_domain
-     * @return bool
-     */
-    public function syncGloV5Route($project_id, $route_domain)
-    {
             try{
                 $result = http_get($route_domain . 'k_u_api.php');
                 if (empty($result)) {
-                $this->log('syncGloV5Route 未获取到路由信息：' . $project_id . ', 路由获取地址：' . $route_domain . 'k_u_api.php');
+                    $this->log('syncGloV5Route 未获取到路由信息：' . $project->id . ', 路由获取地址：' . $route_domain . 'k_u_api.php');
                     return false;
                 }
+                file_put_contents(storage_path('logs/sync_inquiry_project_route/' . $project->id . '.json'), json_encode($result));
+                $this->pushTask($project->id);
             } catch (\Exception $e) {
-            $this->log('syncGloV5Route 未获取到路由信息：' . $project_id . ', 路由获取地址：' . $route_domain . 'k_u_api.php' . ', 错误信息：' . $e->getMessage());
-            echo 'syncGloV5Route 未获取到路由信息：' . $project_id . ', 路由获取地址：' . $route_domain . 'k_u_api.php' . ', 错误信息：' . $e->getMessage() . PHP_EOL;
+                $this->log('syncGloV5Route 未获取到路由信息：' . $project->id . ', 路由获取地址：' . $route_domain . 'k_u_api.php' . ', 错误信息：' . $e->getMessage());
+                echo 'syncGloV5Route 未获取到路由信息：' . $project->id . ', 路由获取地址：' . $route_domain . 'k_u_api.php' . ', 错误信息：' . $e->getMessage() . PHP_EOL;
                 return false;
             }
-
-        foreach ($result as $key=>$val) {
-            try {
-                $tmp = explode('|', $val);
-                $url_tmp = parse_url($tmp[0]);
-                $route = trim($url_tmp['path'], '/');
-                $title = str_replace('+', ' ', $tmp[1]);
-                if (strlen($title) > 200 || strlen($route) > 200) {
-                    $this->log('syncGloV5Route 路由或标题过长，无效记录');
-                    continue;
-                }
-                InquiryProjectRoute::saveProjectRoute($project_id, $title, $route);
-            } catch (\Exception $e) {
-                $this->log('syncGloV5Route 解析路径：' . $val . ', 错误信息：' . $e->getMessage());
-                echo 'syncGloV5Route 解析路径：' . $val . ', 错误信息：' . $e->getMessage() . PHP_EOL;
-            }
         }
+        $this->deleteExpire($date);
         return true;
     }
@@ -148,6 +121,7 @@ class SyncInquiryProject extends Command
      */
     public function syncGloV6()
     {
+        $date = intval(date('Ymd'));
         // 获取优化中台项目
         $project = Project::leftJoin('gl_project_online_check', 'gl_project.id', '=', 'gl_project_online_check.project_id')
             ->where('gl_project.type', Project::TYPE_TWO)
@@ -165,55 +139,34 @@ class SyncInquiryProject extends Command
             // 过滤暂停优化项目, 映射类型2， model没有定义常量
             if (in_array(2, $val->level))
                 continue;
-            $project = InquiryProject::saveProject(InquiryProject::VERSION_SIX, $val->id, $val->title, $val->channel['channel_id'], $domains[$val->id]);
-            $this->syncGloV6Route($project->id, $val->id);
+            $project = InquiryProject::saveProject($date,InquiryProject::VERSION_SIX, $val->id, $val->title, $val->channel['channel_id'], $domains[$val->id]);
+            $this->pushTask($project->id);
         }
+        $this->deleteExpire($date);
         return true;
     }
+
     /**
-     * 同步v6项目路由
-     * @param $project_id
-     * @param $origin_project_id
+     * 删除过期数据， 非当日更新数据， 都删除， 误删第二天再重新同步更新
+     * @param $date
      * @return bool
      */
-    public function syncGloV6Route($project_id, $origin_project_id)
+    public function deleteExpire($date)
     {
-        ProjectServer::useProject($origin_project_id);
-        // TODO 产品分类标题、路由， 产品标题、路由， 同步到路由表
-        $category = Category::where('status', Category::STATUS_ACTIVE)->get(['title', 'route']);
-        foreach ($category as $key=>$val) {
-            InquiryProjectRoute::saveProjectRoute($project_id, $val->title, $val->route);
-        }
-
-        // 产品数量会比较多， 所以使用分页 同步数据
-        $id = 0;
-        while (true) {
-            echo '同步项目路由：' . $id . PHP_EOL;
-            $product = Product::where('status', Product::STATUS_ON)->where('id', '>', $id)->orderBy('id', 'asc')->limit(1000)->get(['id', 'title', 'route']);
-            if ($product->isEmpty())
-                break;
-
-            foreach ($product as $key=>$val) {
-                $id = $val->id;
-                InquiryProjectRoute::saveProjectRoute($project_id, $val->title, $val->route);
-            }
-        }
-
-        DB::disconnect('custom_mysql');
+        $project_num = InquiryProject::where('date', '<', $date)->delete();
+        $this->log('删除过期项目数量：' . $project_num);
         return true;
     }
     /**
-     * 删除过期数据， 非当日更新数据， 都删除， 误删第二天再重新同步更新
+     * 往任务队列中生产数据
+     * @param $id
      * @return bool
      */
-    public function deleteExpire()
+    public function pushTask($id)
     {
-        $date = intval(date('Ymd'));
-        $project_num = InquiryProject::where('date', '<', $date)->delete();
-        $project_route_num = InquiryProjectRoute::where('date', '<', $date)->delete();
-        $this->log('删除过期项目数量：' . $project_num . ', 删除过期路由数量：' . $project_route_num);
+        Redis::lPush('sync_inquiry_project_route_task', $id);
         return true;
     }