作者 lyh

Merge branch 'master' of http://47.244.231.31:8099/zhl/globalso-v6 into develop

  1 +<?php
  2 +
  3 +namespace App\Console\Commands\Update;
  4 +
  5 +use App\Models\Collect\CollectSource;
  6 +use App\Models\Collect\CollectTask;
  7 +use App\Models\Com\UpdateLog;
  8 +use App\Services\CosService;
  9 +use App\Services\ProjectServer;
  10 +use Illuminate\Console\Command;
  11 +use Illuminate\Support\Facades\DB;
  12 +use Illuminate\Support\Facades\Redis;
  13 +
  14 +/**
  15 + * 4.0,5.0升级到6.0,页面采集
  16 + * Class ProjectImport
  17 + * @package App\Console\Commands
  18 + * @author Akun
  19 + * @date 2023/11/10 16:04
  20 + */
  21 +class HtmlCollect extends Command
  22 +{
  23 + /**
  24 + * The name and signature of the console command.
  25 + *
  26 + * @var string
  27 + */
  28 + protected $signature = 'project_html_collect';
  29 +
  30 + /**
  31 + * The console command description.
  32 + *
  33 + * @var string
  34 + */
  35 + protected $description = '执行项目html页面采集';
  36 +
  37 +
  38 + public function handle()
  39 + {
  40 +// while (true) {
  41 + $this->start_update();
  42 +// }
  43 + }
  44 +
  45 + protected function start_update()
  46 + {
  47 +// $task_id = $this->get_task();
  48 + $task_id = '298_1';
  49 + if ($task_id === false) {
  50 + //所有项目采集完成
  51 + sleep(60);
  52 + return true;
  53 + } elseif ($task_id === 0) {
  54 + //当前项目采集完成
  55 + sleep(2);
  56 + return true;
  57 + }
  58 +
  59 + $task_arr = explode('_', $task_id);
  60 + $project_id = $task_arr[0];
  61 + $collect_id = $task_arr[1];
  62 +
  63 + //设置数据库
  64 + $project = ProjectServer::useProject($project_id);
  65 + if ($project) {
  66 + $collect_info = CollectTask::select(['id', 'domain', 'route'])->where('id', $collect_id)->where('status', CollectTask::STATUS_UN)->first();
  67 +
  68 + if (!$collect_info) {
  69 + sleep(2);
  70 + return true;
  71 + }
  72 +
  73 + echo 'date:' . date('Y-m-d H:i:s') . ', project_id: ' . $project_id . ', task_type: ' . $collect_id . ', collect start' . PHP_EOL;
  74 +
  75 + $collect_info->status = CollectTask::STATUS_ING;
  76 + $collect_info->save();
  77 +
  78 + //采集html页面,下载资源到本地并替换
  79 + try {
  80 + $html = file_get_contents('https://' . $collect_info->domain . $collect_info->route);
  81 + $source_list = $this->html_preg($html, $project_id, $collect_info->domain);
  82 +
  83 + if ($source_list) {
  84 + $html = $this->upload_source($html, $source_list, $project_id);
  85 + }
  86 + } catch (\Exception $e) {
  87 + echo 'date:' . date('Y-m-d H:i:s') . ', project_id: ' . $project_id . ', task_type: ' . $collect_id . ', error: ' . $e->getMessage() . PHP_EOL;
  88 + return true;
  89 + }
  90 +
  91 + $collect_info->html = $html;
  92 + $collect_info->status = CollectTask::STATUS_COM;
  93 + $collect_info->save();
  94 +
  95 + echo 'date:' . date('Y-m-d H:i:s') . ', project_id: ' . $project_id . ', task_type: ' . $collect_id . ', collect end' . PHP_EOL;
  96 + }
  97 + //关闭数据库
  98 + DB::disconnect('custom_mysql');
  99 +
  100 + sleep(2);
  101 + }
  102 +
  103 + //获取任务
  104 + protected function get_task()
  105 + {
  106 + $key = 'console_html_collect_task';
  107 + $task_id = Redis::rpop($key);
  108 + if ($task_id) {
  109 + return $task_id;
  110 + }
  111 +
  112 +
  113 + $update_log = UpdateLog::where('status', UpdateLog::STATUS_COM)->where('collect_status', UpdateLog::COLLECT_STATUS_UN)->orderBy('project_id', 'asc')->first();
  114 + if (!$update_log) {
  115 + return false;
  116 + }
  117 +
  118 + $complete = false;
  119 + //设置数据库
  120 + $project = ProjectServer::useProject($update_log->project_id);
  121 + if ($project) {
  122 + $collect_list = CollectTask::select(['id', 'project_id'])->where('project_id', $update_log['project_id'])->where('status', CollectTask::STATUS_UN)->limit(50)->get();
  123 +
  124 + if ($collect_list->count() == 0) {
  125 + $complete = true;
  126 + } else {
  127 + foreach ($collect_list as $collect) {
  128 + Redis::lpush($key, $collect['project_id'] . '_' . $collect['id']);
  129 + }
  130 + }
  131 + }
  132 + //关闭数据库
  133 + DB::disconnect('custom_mysql');
  134 +
  135 + if ($complete) {
  136 + $update_log->collect_status = UpdateLog::COLLECT_STATUS_COM;
  137 + return 0;
  138 + }
  139 +
  140 + $task_id = Redis::rpop($key);
  141 + return $task_id;
  142 + }
  143 +
  144 + //正则匹配html资源
  145 + protected function html_preg($html, $project_id, $domain)
  146 + {
  147 + $source = [];
  148 +
  149 + if (!$html) {
  150 + return $source;
  151 + }
  152 +
  153 + //图片
  154 + preg_match_all('/<img\s+[^>]*?src\s*=\s*(\'|\")(.*?)\\1[^>]*?\/?\s*>/i', $html, $result_img);
  155 + $img = $result_img[2] ?? [];
  156 + foreach ($img as $vi) {
  157 + $check_vi = $this->url_check($vi, $project_id, $domain);
  158 + $check_vi && $source[] = $check_vi;
  159 + }
  160 +
  161 + //js
  162 + preg_match_all('/<script\s+[^>]*?src\s*=\s*(\'|\")(.*?)\\1[^>]*?\/?\s*>/i', $html, $result_js);
  163 + $js = $result_js[2] ?? [];
  164 + foreach ($js as $vj) {
  165 + $check_vj = $this->url_check($vj, $project_id, $domain);
  166 + $check_vj && $source[] = $check_vj;
  167 + }
  168 +
  169 + //video
  170 + preg_match_all('/<source\s+[^>]*?src\s*=\s*(\'|\")(.*?)\\1[^>]*?\/?\s*>/i', $html, $result_video);
  171 + $video = $result_video[2] ?? [];
  172 + foreach ($video as $vv) {
  173 + $check_vv = $this->url_check($vv, $project_id, $domain);
  174 + $check_vv && $source[] = $check_vv;
  175 + }
  176 +
  177 + //css
  178 + preg_match_all('/<link\s+[^>]*?href\s*=\s*(\'|\")(.*?)\\1[^>]*?\/?\s*>/i', $html, $result_css);
  179 + $css = $result_css[2] ?? [];
  180 + foreach ($css as $vc) {
  181 + $check_vc = $this->url_check($vc, $project_id, $domain);
  182 + $check_vc && $source[] = $check_vc;
  183 + }
  184 +
  185 + return $source;
  186 + }
  187 +
  188 + //判断资源是否需要下载
  189 + protected function url_check($url, $project_id, $domain)
  190 + {
  191 + if ($url) {
  192 + $arr = parse_url($url);
  193 + $scheme = $arr['scheme'] ?? '';
  194 + $host = $arr['host'] ?? '';
  195 + $path = $arr['path'] ?? '';
  196 +
  197 + if ((strpos($host, '.globalso.') === false)
  198 + && (strpos($host, '.goodao.') === false)
  199 + && $path && (strpos($path, '.') !== false)) {
  200 +
  201 + $source = CollectSource::where('project_id', $project_id)->where('origin', $url)->first();
  202 + if (!$source) {
  203 + return [
  204 + 'download' => true,
  205 + 'url' => $url,
  206 + 'url_complete' => ($scheme ?: 'https') . '://' . ($host ?: $domain) . $path
  207 + ];
  208 + } else {
  209 + return [
  210 + 'download' => false,
  211 + 'url' => $url,
  212 + 'url_complete' => $source['target']
  213 + ];
  214 + }
  215 + } else {
  216 + return false;
  217 + }
  218 + } else {
  219 + return false;
  220 + }
  221 + }
  222 +
  223 + //下载并替换资源
  224 + protected function upload_source($html, $source, $project_id)
  225 + {
  226 + foreach ($source as $vs) {
  227 +
  228 + if ($vs['download']) {
  229 + $new_source = CosService::uploadRemote($project_id, 'source', $vs['url_complete']);
  230 + if ($new_source) {
  231 + CollectSource::insert([
  232 + 'project_id' => $project_id,
  233 + 'origin' => $vs['url'],
  234 + 'target' => $new_source,
  235 + 'created_at' => date('Y-m-d H:i:s'),
  236 + 'updated_at' => date('Y-m-d H:i:s'),
  237 + ]);
  238 + $html = str_replace($vs['url'], getImageUrl($new_source), $html);
  239 + }
  240 + } else {
  241 + $html = str_replace($vs['url'], getImageUrl($vs['url_complete']), $html);
  242 + }
  243 + }
  244 +
  245 + return $html;
  246 + }
  247 +}
@@ -4,8 +4,8 @@ namespace App\Console\Commands\Update; @@ -4,8 +4,8 @@ namespace App\Console\Commands\Update;
4 4
5 use App\Helper\Arr; 5 use App\Helper\Arr;
6 use App\Http\Logic\Bside\Product\CategoryLogic; 6 use App\Http\Logic\Bside\Product\CategoryLogic;
7 -use App\Http\Logic\Bside\Product\KeywordLogic;  
8 use App\Models\Blog\Blog; 7 use App\Models\Blog\Blog;
  8 +use App\Models\Collect\CollectTask;
9 use App\Models\Com\UpdateLog; 9 use App\Models\Com\UpdateLog;
10 use App\Models\News\News; 10 use App\Models\News\News;
11 use App\Models\Product\Category; 11 use App\Models\Product\Category;
@@ -77,7 +77,7 @@ class ProjectUpdate extends Command @@ -77,7 +77,7 @@ class ProjectUpdate extends Command
77 $task->save(); 77 $task->save();
78 78
79 //设置数据库 79 //设置数据库
80 - $project = ProjectServer::useProject($task->project_id); 80 + $project = ProjectServer::useProject($project_id);
81 if ($project) { 81 if ($project) {
82 if ($api_type == 'category') { 82 if ($api_type == 'category') {
83 //分类 83 //分类
@@ -86,48 +86,6 @@ class ProjectUpdate extends Command @@ -86,48 +86,6 @@ class ProjectUpdate extends Command
86 if (isset($data['code']) && $data['code'] == 200) { 86 if (isset($data['code']) && $data['code'] == 200) {
87 $items = $data['data'] ?? []; 87 $items = $data['data'] ?? [];
88 $this->category_insert($project_id, $items, 0); 88 $this->category_insert($project_id, $items, 0);
89 -// $model = new Category();  
90 -// foreach ($items as $item) {  
91 -// $parent = $model->read(['pid' => 0, 'title' => $item['name']], 'id');  
92 -// if (!$parent) {  
93 -// try {  
94 -// $parent_id = $model->addReturnId([  
95 -// 'project_id' => $project_id,  
96 -// 'title' => $item['name'],  
97 -// 'pid' => 0,  
98 -// 'keywords' => $item['keywords'],  
99 -// 'describe' => $item['description']  
100 -// ]);  
101 -// $route = RouteMap::setRoute($item['url'] ? $this->get_url_route($item['url']) : $item['name'], RouteMap::SOURCE_PRODUCT_CATE, $parent_id, $project_id);  
102 -// $model->edit(['route' => $route], ['id' => $parent_id]);  
103 -// } catch (\Exception $e) {  
104 -// echo 'date:' . date('Y-m-d H:i:s') . ', task_id: ' . $task->id . ', error: ' . $e->getMessage() . PHP_EOL;  
105 -// continue;  
106 -// }  
107 -// } else {  
108 -// $parent_id = $parent['id'];  
109 -// }  
110 -//  
111 -// foreach ($item['children'] as $child) {  
112 -// $child_info = $model->read(['pid' => $parent_id, 'title' => $child['name']]);  
113 -// if (!$child_info) {  
114 -// try {  
115 -// $child_id = $model->addReturnId([  
116 -// 'project_id' => $project_id,  
117 -// 'title' => $child['name'],  
118 -// 'pid' => $parent_id,  
119 -// 'keywords' => $child['keywords'],  
120 -// 'describe' => $child['description']  
121 -// ]);  
122 -// $route = RouteMap::setRoute($child['url'] ? $this->get_url_route($child['url']) : $child['name'], RouteMap::SOURCE_PRODUCT_CATE, $child_id, $project_id);  
123 -// $model->edit(['route' => $route], ['id' => $child_id]);  
124 -// } catch (\Exception $e) {  
125 -// echo 'date:' . date('Y-m-d H:i:s') . ', task_id: ' . $task->id . ', error: ' . $e->getMessage() . PHP_EOL;  
126 -// continue;  
127 -// }  
128 -// }  
129 -// }  
130 -// }  
131 } else { 89 } else {
132 return true; 90 return true;
133 } 91 }
@@ -164,7 +122,11 @@ class ProjectUpdate extends Command @@ -164,7 +122,11 @@ class ProjectUpdate extends Command
164 echo 'date:' . date('Y-m-d H:i:s') . ', task_id: ' . $task->id . ', error: ' . $e->getMessage() . PHP_EOL; 122 echo 'date:' . date('Y-m-d H:i:s') . ', task_id: ' . $task->id . ', error: ' . $e->getMessage() . PHP_EOL;
165 continue; 123 continue;
166 } 124 }
  125 + } else {
  126 + $id = $keyword['id'];
167 } 127 }
  128 +
  129 + CollectTask::_insert($item['url'], $project_id, RouteMap::SOURCE_PRODUCT_KEYWORD, $id);
168 } 130 }
169 } 131 }
170 } 132 }
@@ -279,7 +241,11 @@ class ProjectUpdate extends Command @@ -279,7 +241,11 @@ class ProjectUpdate extends Command
279 echo 'date:' . date('Y-m-d H:i:s') . ', task_id: ' . $task->id . ', error: ' . $e->getMessage() . PHP_EOL; 241 echo 'date:' . date('Y-m-d H:i:s') . ', task_id: ' . $task->id . ', error: ' . $e->getMessage() . PHP_EOL;
280 continue; 242 continue;
281 } 243 }
  244 + } else {
  245 + $id = $product['id'];
282 } 246 }
  247 +
  248 + CollectTask::_insert($item['url'], $project_id, RouteMap::SOURCE_PRODUCT, $id);
283 } 249 }
284 } 250 }
285 } 251 }
@@ -329,7 +295,11 @@ class ProjectUpdate extends Command @@ -329,7 +295,11 @@ class ProjectUpdate extends Command
329 echo 'date:' . date('Y-m-d H:i:s') . ', task_id: ' . $task->id . ', error: ' . $e->getMessage() . PHP_EOL; 295 echo 'date:' . date('Y-m-d H:i:s') . ', task_id: ' . $task->id . ', error: ' . $e->getMessage() . PHP_EOL;
330 continue; 296 continue;
331 } 297 }
  298 + } else {
  299 + $id = $news['id'];
332 } 300 }
  301 +
  302 + CollectTask::_insert($item['url'], $project_id, $api_type == 'news' ? RouteMap::SOURCE_NEWS : RouteMap::SOURCE_BLOG, $id);
333 } 303 }
334 } 304 }
335 } 305 }
@@ -373,7 +343,11 @@ class ProjectUpdate extends Command @@ -373,7 +343,11 @@ class ProjectUpdate extends Command
373 echo 'date:' . date('Y-m-d H:i:s') . ', task_id: ' . $task->id . ', error: ' . $e->getMessage() . PHP_EOL; 343 echo 'date:' . date('Y-m-d H:i:s') . ', task_id: ' . $task->id . ', error: ' . $e->getMessage() . PHP_EOL;
374 continue; 344 continue;
375 } 345 }
  346 + } else {
  347 + $id = $custom['id'];
376 } 348 }
  349 +
  350 + CollectTask::_insert($item['url'], $project_id, RouteMap::SOURCE_PAGE, $id);
377 } 351 }
378 } 352 }
379 } 353 }
@@ -423,6 +397,7 @@ class ProjectUpdate extends Command @@ -423,6 +397,7 @@ class ProjectUpdate extends Command
423 return $arr[count($arr) - 2]; 397 return $arr[count($arr) - 2];
424 } 398 }
425 399
  400 + //多级分类入库
426 protected function category_insert($project_id, $items, $pid = 0) 401 protected function category_insert($project_id, $items, $pid = 0)
427 { 402 {
428 $model = new Category(); 403 $model = new Category();
  1 +<?php
  2 +
  3 +namespace App\Http\Controllers\Aside;
  4 +
  5 +use App\Http\Logic\Aside\CollectLogic;
  6 +
  7 +
  8 +/**
  9 + * 提供给AICC采集
  10 + * Class CollectController
  11 + * @package App\Http\Controllers\Aside
  12 + * @author zbj
  13 + * @date 2023/11/10
  14 + */
  15 +class CollectController extends BaseController
  16 +{
  17 + /**
  18 + * @author zbj
  19 + * @date 2023/11/10
  20 + */
  21 + public function index(CollectLogic $collectLogic)
  22 + {
  23 + $data = $collectLogic->collect_data();
  24 + return $this->success($data);
  25 + }
  26 +}
@@ -801,7 +801,7 @@ class ProjectController extends BaseController @@ -801,7 +801,7 @@ class ProjectController extends BaseController
801 'project_id.required' => 'project_id不能为空', 801 'project_id.required' => 'project_id不能为空',
802 ]); 802 ]);
803 803
804 - $token = $logic->getAiccToken($this->map); 804 + $token = $logic->getSiteToken($this->map);
805 805
806 $this->response('success',Code::SUCCESS,['site_token' => $token]); 806 $this->response('success',Code::SUCCESS,['site_token' => $token]);
807 807
  1 +<?php
  2 +
  3 +namespace App\Http\Logic\Aside;
  4 +
  5 +use App\Helper\Arr;
  6 +use App\Http\Logic\Logic;
  7 +use App\Models\Blog\Blog;
  8 +use App\Models\Domain\DomainInfo;
  9 +use App\Models\News\News;
  10 +use App\Models\Product\Keyword;
  11 +use App\Models\Product\Product;
  12 +use App\Models\Project\Project;
  13 +use App\Services\ProjectServer;
  14 +
  15 +
  16 +/**
  17 + * Class CollectLogic
  18 + * @package App\Http\Logic\Aside
  19 + * @author zbj
  20 + * @date 2023/11/10
  21 + */
  22 +class CollectLogic extends Logic
  23 +{
  24 + protected $project;
  25 + protected $domain;
  26 + protected $type;
  27 + protected $page_size = 100;
  28 +
  29 + public function __construct()
  30 + {
  31 + $this->checkAuth();
  32 + }
  33 +
  34 + /**
  35 + * 校验权限
  36 + * @throws \App\Exceptions\AsideGlobalException
  37 + * @throws \App\Exceptions\BsideGlobalException
  38 + * @author zbj
  39 + * @date 2023/11/10
  40 + */
  41 + public function checkAuth()
  42 + {
  43 + $request = request();
  44 + $site_token = $request->header('site-token');
  45 + $domain = $request->input('domain');
  46 + if (!$site_token) {
  47 + $this->fail('参数异常');
  48 + }
  49 + $this->project = Project::where('site_token', $site_token)->first();
  50 + if (!$this->project) {
  51 + $this->fail('授权码无效');
  52 + }
  53 + $domain_info = DomainInfo::where('project_id', $this->project->id)->where('domain', $domain)->first();
  54 + if (!$domain_info) {
  55 + $this->fail('域名不匹配');
  56 + }
  57 + $this->domain = 'https://' . $domain_info['domain'] . '/';
  58 + $this->type = $request->input('type', '');
  59 + }
  60 +
  61 + public function collect_data()
  62 + {
  63 + ProjectServer::useProject($this->project->id);
  64 + $action = $this->type;
  65 + return $this->$action();
  66 + }
  67 +
  68 + public function __call($name, $param)
  69 + {
  70 + return [];
  71 + }
  72 +
  73 +
  74 + public function product()
  75 + {
  76 + $this->model = new Product();
  77 + $where[] = ['status' => Product::STATUS_ON];
  78 + $sort = ['sort' => 'desc'];
  79 + $columns = ['title', 'content', 'gallery', 'seo_mate', 'intro', 'route', 'keyword_id'];
  80 + $list = self::getList($where,$sort, $columns, $this->page_size);
  81 + $data =[];
  82 + foreach ($list['list'] as $item){
  83 + //关键词标签 没有就取seo 键词
  84 + if($item['keyword_id']){
  85 + $keyword = Keyword::whereIn('id', $item['keyword_id'])->pluck('title')->toArray();
  86 + if($keyword){
  87 + $keyword = implode(',', $keyword);
  88 + }
  89 + }
  90 + $keyword = $keyword ?: ($item['seo_mate']['keyword'] ?? '');
  91 + $data[] = [
  92 + 'title' => $item['title'],
  93 + 'url' => $this->domain . $item['route'],
  94 + 'keywords' => $keyword,
  95 + 'description' => strip_tags($item['intro']?:''),
  96 + 'content' => strip_tags($item['content'] ?: ''),
  97 + 'img' => array_column($item['gallery'] ?: [], 'url')
  98 + ];
  99 + }
  100 + $list['list'] = $data;
  101 + return $list;
  102 + }
  103 +
  104 + public function news()
  105 + {
  106 + $this->model = new News();
  107 + $where[] = ['status' => News::STATUS_ONE];
  108 + $sort = ['sort' => 'desc'];
  109 + $columns = ['name', 'text', 'image', 'seo_keywords', 'remark', 'url'];
  110 + $list = self::getList($where,$sort, $columns, $this->page_size);
  111 + $data =[];
  112 + foreach ($list['list'] as $item){
  113 + $data[] = [
  114 + 'title' => $item['name'],
  115 + 'url' => $this->domain . $item['url'],
  116 + 'keywords' => $item['seo_keywords'],
  117 + 'description' => strip_tags($item['remark']?:''),
  118 + 'content' => strip_tags($item['text'] ?: ''),
  119 + 'img' => $item['image'] ?:''
  120 + ];
  121 + }
  122 + $list['list'] = $data;
  123 + return $list;
  124 + }
  125 +
  126 + public function blog()
  127 + {
  128 + $this->model = new Blog();
  129 + $where[] = ['status' => Blog::STATUS_ONE];
  130 + $sort = ['sort' => 'desc'];
  131 + $columns = ['name', 'text', 'image', 'seo_keywords', 'remark', 'url'];
  132 + $list = self::getList($where,$sort, $columns, $this->page_size);
  133 + $data =[];
  134 + foreach ($list['list'] as $item){
  135 + $data[] = [
  136 + 'title' => $item['name'],
  137 + 'url' => $this->domain . $item['url'],
  138 + 'keywords' => $item['seo_keywords'],
  139 + 'description' => strip_tags($item['remark']?:''),
  140 + 'content' => strip_tags($item['text'] ?: ''),
  141 + 'img' => $item['image'] ?:''
  142 + ];
  143 + }
  144 + $list['list'] = $data;
  145 + return $list;
  146 + }
  147 +}
@@ -651,6 +651,7 @@ class ProjectLogic extends BaseLogic @@ -651,6 +651,7 @@ class ProjectLogic extends BaseLogic
651 $query->select('*')->from("{$name}"); 651 $query->select('*')->from("{$name}");
652 } 652 }
653 ); 653 );
  654 +
654 if (Schema::connection('custom_mysql')->hasColumn($table, 'project_id')) { 655 if (Schema::connection('custom_mysql')->hasColumn($table, 'project_id')) {
655 DB::connection('custom_mysql')->table($table)->update(['project_id' => $news_project_id]); 656 DB::connection('custom_mysql')->table($table)->update(['project_id' => $news_project_id]);
656 } 657 }
@@ -659,16 +660,16 @@ class ProjectLogic extends BaseLogic @@ -659,16 +660,16 @@ class ProjectLogic extends BaseLogic
659 } 660 }
660 661
661 /** 662 /**
662 - * 获取AICC采集数据接口token 663 + * 对外接口token
663 * @param $data 664 * @param $data
664 * @return string 665 * @return string
665 * @author zbj 666 * @author zbj
666 * @date 2023/11/10 667 * @date 2023/11/10
667 */ 668 */
668 - public function getAiccToken($data){ 669 + public function getSiteToken($data){
669 $project = $this->getCacheInfo($data['project_id']); 670 $project = $this->getCacheInfo($data['project_id']);
670 if(empty($project['site_token']) || !empty($data['refresh'])){ 671 if(empty($project['site_token']) || !empty($data['refresh'])){
671 - $token = strtolower(Str::random() . base64_encode("globalso_v6")); 672 + $token = strtolower(base64_encode("6.0") . md5('project_' . $data['project_id'] . '_' . time()));
672 $project->site_token = $token; 673 $project->site_token = $token;
673 $project->save(); 674 $project->save();
674 } 675 }
@@ -69,8 +69,6 @@ class CustomTemplateLogic extends BaseLogic @@ -69,8 +69,6 @@ class CustomTemplateLogic extends BaseLogic
69 }else{ 69 }else{
70 if($this->param['url'] == $this->model::NOT_FOUND_PAGE_URL){ 70 if($this->param['url'] == $this->model::NOT_FOUND_PAGE_URL){
71 $this->fail('404页面已存在'); 71 $this->fail('404页面已存在');
72 - }else{  
73 - $this->param['url'] = $this->param['url'].'-tag';  
74 } 72 }
75 $this->param['project_id'] = $this->user['project_id']; 73 $this->param['project_id'] = $this->user['project_id'];
76 $id = $this->model->addReturnId($this->param); 74 $id = $this->model->addReturnId($this->param);
  1 +<?php
  2 +
  3 +namespace App\Models\Collect;
  4 +
  5 +use App\Models\Base;
  6 +
  7 +class CollectSource extends Base
  8 +{
  9 + //设置关联表名
  10 + protected $table = 'gl_collect_source';
  11 +
  12 + //连接数据库
  13 + protected $connection = 'custom_mysql';
  14 +
  15 +
  16 +}
  1 +<?php
  2 +
  3 +namespace App\Models\Collect;
  4 +
  5 +use App\Models\Base;
  6 +
  7 +class CollectTask extends Base
  8 +{
  9 + //设置关联表名
  10 + protected $table = 'gl_collect_task';
  11 +
  12 + //连接数据库
  13 + protected $connection = 'custom_mysql';
  14 +
  15 + const STATUS_UN = 0;
  16 + const STATUS_ING = 1;
  17 + const STATUS_COM= 2;
  18 +
  19 + public static function _insert($url, $project_id, $source, $source_id)
  20 + {
  21 + if(!$url){
  22 + return;
  23 + }
  24 +
  25 + $url_arr = parse_url($url);
  26 +
  27 + $data = [
  28 + 'project_id' => $project_id,
  29 + 'source' => $source,
  30 + 'source_id' => $source_id,
  31 + 'domain' => $url_arr['host'],
  32 + 'route' => $url_arr['path']
  33 + ];
  34 +
  35 + $task = self::where($data)->first();
  36 + if(!$task){
  37 + $data['created_at'] = $data['updated_at'] = date('Y-m-d H:i:s');
  38 + self::insert($data);
  39 + }
  40 + }
  41 +}
@@ -13,6 +13,9 @@ class UpdateLog extends Model @@ -13,6 +13,9 @@ class UpdateLog extends Model
13 const STATUS_ING = 1;//导入中 13 const STATUS_ING = 1;//导入中
14 const STATUS_COM = 2;//导入完成 14 const STATUS_COM = 2;//导入完成
15 15
  16 + const COLLECT_STATUS_UN = 0;//未开始
  17 + const COLLECT_STATUS_COM = 1;//采集完成
  18 +
16 /** 19 /**
17 * 创建更新日志 20 * 创建更新日志
18 * @param $project_id 21 * @param $project_id
@@ -30,6 +33,7 @@ class UpdateLog extends Model @@ -30,6 +33,7 @@ class UpdateLog extends Model
30 $log->api_type = $type; 33 $log->api_type = $type;
31 $log->api_url = $url; 34 $log->api_url = $url;
32 $log->sort = $type == 'category' ? 0 :1; 35 $log->sort = $type == 'category' ? 0 :1;
  36 + $log->collect_status = in_array($type, ['website_info', 'category']) ? 1 : 0;
33 return $log->save(); 37 return $log->save();
34 } 38 }
35 return true; 39 return true;
@@ -340,6 +340,8 @@ Route::group([], function () { @@ -340,6 +340,8 @@ Route::group([], function () {
340 // 提供模板 提单后台查看 340 // 提供模板 提单后台查看
341 Route::any('get_template_list', [Aside\Template\ATemplateController::class, 'getTemplateList'])->name('admin.get_template_list'); 341 Route::any('get_template_list', [Aside\Template\ATemplateController::class, 'getTemplateList'])->name('admin.get_template_list');
342 Route::any('get_template_detail', [Aside\Template\ATemplateController::class, 'getTemplateDetail'])->name('admin.get_template_detail'); 342 Route::any('get_template_detail', [Aside\Template\ATemplateController::class, 'getTemplateDetail'])->name('admin.get_template_detail');
  343 +
  344 + Route::any('/collect', [Aside\CollectController::class, 'index'])->name('admin.collect');
343 }); 345 });
344 346
345 347