Merge branch 'master' of http://47.244.231.31:8099/zhl/globalso-v6 into develop
正在显示
11 个修改的文件
包含
507 行增加
和
50 行删除
app/Console/Commands/Update/HtmlCollect.php
0 → 100644
| 1 | +<?php | ||
| 2 | + | ||
| 3 | +namespace App\Console\Commands\Update; | ||
| 4 | + | ||
| 5 | +use App\Models\Collect\CollectSource; | ||
| 6 | +use App\Models\Collect\CollectTask; | ||
| 7 | +use App\Models\Com\UpdateLog; | ||
| 8 | +use App\Services\CosService; | ||
| 9 | +use App\Services\ProjectServer; | ||
| 10 | +use Illuminate\Console\Command; | ||
| 11 | +use Illuminate\Support\Facades\DB; | ||
| 12 | +use Illuminate\Support\Facades\Redis; | ||
| 13 | + | ||
| 14 | +/** | ||
| 15 | + * 4.0,5.0升级到6.0,页面采集 | ||
| 16 | + * Class ProjectImport | ||
| 17 | + * @package App\Console\Commands | ||
| 18 | + * @author Akun | ||
| 19 | + * @date 2023/11/10 16:04 | ||
| 20 | + */ | ||
| 21 | +class HtmlCollect extends Command | ||
| 22 | +{ | ||
| 23 | + /** | ||
| 24 | + * The name and signature of the console command. | ||
| 25 | + * | ||
| 26 | + * @var string | ||
| 27 | + */ | ||
| 28 | + protected $signature = 'project_html_collect'; | ||
| 29 | + | ||
| 30 | + /** | ||
| 31 | + * The console command description. | ||
| 32 | + * | ||
| 33 | + * @var string | ||
| 34 | + */ | ||
| 35 | + protected $description = '执行项目html页面采集'; | ||
| 36 | + | ||
| 37 | + | ||
| 38 | + public function handle() | ||
| 39 | + { | ||
| 40 | +// while (true) { | ||
| 41 | + $this->start_update(); | ||
| 42 | +// } | ||
| 43 | + } | ||
| 44 | + | ||
| 45 | + protected function start_update() | ||
| 46 | + { | ||
| 47 | +// $task_id = $this->get_task(); | ||
| 48 | + $task_id = '298_1'; | ||
| 49 | + if ($task_id === false) { | ||
| 50 | + //所有项目采集完成 | ||
| 51 | + sleep(60); | ||
| 52 | + return true; | ||
| 53 | + } elseif ($task_id === 0) { | ||
| 54 | + //当前项目采集完成 | ||
| 55 | + sleep(2); | ||
| 56 | + return true; | ||
| 57 | + } | ||
| 58 | + | ||
| 59 | + $task_arr = explode('_', $task_id); | ||
| 60 | + $project_id = $task_arr[0]; | ||
| 61 | + $collect_id = $task_arr[1]; | ||
| 62 | + | ||
| 63 | + //设置数据库 | ||
| 64 | + $project = ProjectServer::useProject($project_id); | ||
| 65 | + if ($project) { | ||
| 66 | + $collect_info = CollectTask::select(['id', 'domain', 'route'])->where('id', $collect_id)->where('status', CollectTask::STATUS_UN)->first(); | ||
| 67 | + | ||
| 68 | + if (!$collect_info) { | ||
| 69 | + sleep(2); | ||
| 70 | + return true; | ||
| 71 | + } | ||
| 72 | + | ||
| 73 | + echo 'date:' . date('Y-m-d H:i:s') . ', project_id: ' . $project_id . ', task_type: ' . $collect_id . ', collect start' . PHP_EOL; | ||
| 74 | + | ||
| 75 | + $collect_info->status = CollectTask::STATUS_ING; | ||
| 76 | + $collect_info->save(); | ||
| 77 | + | ||
| 78 | + //采集html页面,下载资源到本地并替换 | ||
| 79 | + try { | ||
| 80 | + $html = file_get_contents('https://' . $collect_info->domain . $collect_info->route); | ||
| 81 | + $source_list = $this->html_preg($html, $project_id, $collect_info->domain); | ||
| 82 | + | ||
| 83 | + if ($source_list) { | ||
| 84 | + $html = $this->upload_source($html, $source_list, $project_id); | ||
| 85 | + } | ||
| 86 | + } catch (\Exception $e) { | ||
| 87 | + echo 'date:' . date('Y-m-d H:i:s') . ', project_id: ' . $project_id . ', task_type: ' . $collect_id . ', error: ' . $e->getMessage() . PHP_EOL; | ||
| 88 | + return true; | ||
| 89 | + } | ||
| 90 | + | ||
| 91 | + $collect_info->html = $html; | ||
| 92 | + $collect_info->status = CollectTask::STATUS_COM; | ||
| 93 | + $collect_info->save(); | ||
| 94 | + | ||
| 95 | + echo 'date:' . date('Y-m-d H:i:s') . ', project_id: ' . $project_id . ', task_type: ' . $collect_id . ', collect end' . PHP_EOL; | ||
| 96 | + } | ||
| 97 | + //关闭数据库 | ||
| 98 | + DB::disconnect('custom_mysql'); | ||
| 99 | + | ||
| 100 | + sleep(2); | ||
| 101 | + } | ||
| 102 | + | ||
| 103 | + //获取任务 | ||
| 104 | + protected function get_task() | ||
| 105 | + { | ||
| 106 | + $key = 'console_html_collect_task'; | ||
| 107 | + $task_id = Redis::rpop($key); | ||
| 108 | + if ($task_id) { | ||
| 109 | + return $task_id; | ||
| 110 | + } | ||
| 111 | + | ||
| 112 | + | ||
| 113 | + $update_log = UpdateLog::where('status', UpdateLog::STATUS_COM)->where('collect_status', UpdateLog::COLLECT_STATUS_UN)->orderBy('project_id', 'asc')->first(); | ||
| 114 | + if (!$update_log) { | ||
| 115 | + return false; | ||
| 116 | + } | ||
| 117 | + | ||
| 118 | + $complete = false; | ||
| 119 | + //设置数据库 | ||
| 120 | + $project = ProjectServer::useProject($update_log->project_id); | ||
| 121 | + if ($project) { | ||
| 122 | + $collect_list = CollectTask::select(['id', 'project_id'])->where('project_id', $update_log['project_id'])->where('status', CollectTask::STATUS_UN)->limit(50)->get(); | ||
| 123 | + | ||
| 124 | + if ($collect_list->count() == 0) { | ||
| 125 | + $complete = true; | ||
| 126 | + } else { | ||
| 127 | + foreach ($collect_list as $collect) { | ||
| 128 | + Redis::lpush($key, $collect['project_id'] . '_' . $collect['id']); | ||
| 129 | + } | ||
| 130 | + } | ||
| 131 | + } | ||
| 132 | + //关闭数据库 | ||
| 133 | + DB::disconnect('custom_mysql'); | ||
| 134 | + | ||
| 135 | + if ($complete) { | ||
| 136 | + $update_log->collect_status = UpdateLog::COLLECT_STATUS_COM; | ||
| 137 | + return 0; | ||
| 138 | + } | ||
| 139 | + | ||
| 140 | + $task_id = Redis::rpop($key); | ||
| 141 | + return $task_id; | ||
| 142 | + } | ||
| 143 | + | ||
| 144 | + //正则匹配html资源 | ||
| 145 | + protected function html_preg($html, $project_id, $domain) | ||
| 146 | + { | ||
| 147 | + $source = []; | ||
| 148 | + | ||
| 149 | + if (!$html) { | ||
| 150 | + return $source; | ||
| 151 | + } | ||
| 152 | + | ||
| 153 | + //图片 | ||
| 154 | + preg_match_all('/<img\s+[^>]*?src\s*=\s*(\'|\")(.*?)\\1[^>]*?\/?\s*>/i', $html, $result_img); | ||
| 155 | + $img = $result_img[2] ?? []; | ||
| 156 | + foreach ($img as $vi) { | ||
| 157 | + $check_vi = $this->url_check($vi, $project_id, $domain); | ||
| 158 | + $check_vi && $source[] = $check_vi; | ||
| 159 | + } | ||
| 160 | + | ||
| 161 | + //js | ||
| 162 | + preg_match_all('/<script\s+[^>]*?src\s*=\s*(\'|\")(.*?)\\1[^>]*?\/?\s*>/i', $html, $result_js); | ||
| 163 | + $js = $result_js[2] ?? []; | ||
| 164 | + foreach ($js as $vj) { | ||
| 165 | + $check_vj = $this->url_check($vj, $project_id, $domain); | ||
| 166 | + $check_vj && $source[] = $check_vj; | ||
| 167 | + } | ||
| 168 | + | ||
| 169 | + //video | ||
| 170 | + preg_match_all('/<source\s+[^>]*?src\s*=\s*(\'|\")(.*?)\\1[^>]*?\/?\s*>/i', $html, $result_video); | ||
| 171 | + $video = $result_video[2] ?? []; | ||
| 172 | + foreach ($video as $vv) { | ||
| 173 | + $check_vv = $this->url_check($vv, $project_id, $domain); | ||
| 174 | + $check_vv && $source[] = $check_vv; | ||
| 175 | + } | ||
| 176 | + | ||
| 177 | + //css | ||
| 178 | + preg_match_all('/<link\s+[^>]*?href\s*=\s*(\'|\")(.*?)\\1[^>]*?\/?\s*>/i', $html, $result_css); | ||
| 179 | + $css = $result_css[2] ?? []; | ||
| 180 | + foreach ($css as $vc) { | ||
| 181 | + $check_vc = $this->url_check($vc, $project_id, $domain); | ||
| 182 | + $check_vc && $source[] = $check_vc; | ||
| 183 | + } | ||
| 184 | + | ||
| 185 | + return $source; | ||
| 186 | + } | ||
| 187 | + | ||
| 188 | + //判断资源是否需要下载 | ||
| 189 | + protected function url_check($url, $project_id, $domain) | ||
| 190 | + { | ||
| 191 | + if ($url) { | ||
| 192 | + $arr = parse_url($url); | ||
| 193 | + $scheme = $arr['scheme'] ?? ''; | ||
| 194 | + $host = $arr['host'] ?? ''; | ||
| 195 | + $path = $arr['path'] ?? ''; | ||
| 196 | + | ||
| 197 | + if ((strpos($host, '.globalso.') === false) | ||
| 198 | + && (strpos($host, '.goodao.') === false) | ||
| 199 | + && $path && (strpos($path, '.') !== false)) { | ||
| 200 | + | ||
| 201 | + $source = CollectSource::where('project_id', $project_id)->where('origin', $url)->first(); | ||
| 202 | + if (!$source) { | ||
| 203 | + return [ | ||
| 204 | + 'download' => true, | ||
| 205 | + 'url' => $url, | ||
| 206 | + 'url_complete' => ($scheme ?: 'https') . '://' . ($host ?: $domain) . $path | ||
| 207 | + ]; | ||
| 208 | + } else { | ||
| 209 | + return [ | ||
| 210 | + 'download' => false, | ||
| 211 | + 'url' => $url, | ||
| 212 | + 'url_complete' => $source['target'] | ||
| 213 | + ]; | ||
| 214 | + } | ||
| 215 | + } else { | ||
| 216 | + return false; | ||
| 217 | + } | ||
| 218 | + } else { | ||
| 219 | + return false; | ||
| 220 | + } | ||
| 221 | + } | ||
| 222 | + | ||
| 223 | + //下载并替换资源 | ||
| 224 | + protected function upload_source($html, $source, $project_id) | ||
| 225 | + { | ||
| 226 | + foreach ($source as $vs) { | ||
| 227 | + | ||
| 228 | + if ($vs['download']) { | ||
| 229 | + $new_source = CosService::uploadRemote($project_id, 'source', $vs['url_complete']); | ||
| 230 | + if ($new_source) { | ||
| 231 | + CollectSource::insert([ | ||
| 232 | + 'project_id' => $project_id, | ||
| 233 | + 'origin' => $vs['url'], | ||
| 234 | + 'target' => $new_source, | ||
| 235 | + 'created_at' => date('Y-m-d H:i:s'), | ||
| 236 | + 'updated_at' => date('Y-m-d H:i:s'), | ||
| 237 | + ]); | ||
| 238 | + $html = str_replace($vs['url'], getImageUrl($new_source), $html); | ||
| 239 | + } | ||
| 240 | + } else { | ||
| 241 | + $html = str_replace($vs['url'], getImageUrl($vs['url_complete']), $html); | ||
| 242 | + } | ||
| 243 | + } | ||
| 244 | + | ||
| 245 | + return $html; | ||
| 246 | + } | ||
| 247 | +} |
| @@ -4,8 +4,8 @@ namespace App\Console\Commands\Update; | @@ -4,8 +4,8 @@ namespace App\Console\Commands\Update; | ||
| 4 | 4 | ||
| 5 | use App\Helper\Arr; | 5 | use App\Helper\Arr; |
| 6 | use App\Http\Logic\Bside\Product\CategoryLogic; | 6 | use App\Http\Logic\Bside\Product\CategoryLogic; |
| 7 | -use App\Http\Logic\Bside\Product\KeywordLogic; | ||
| 8 | use App\Models\Blog\Blog; | 7 | use App\Models\Blog\Blog; |
| 8 | +use App\Models\Collect\CollectTask; | ||
| 9 | use App\Models\Com\UpdateLog; | 9 | use App\Models\Com\UpdateLog; |
| 10 | use App\Models\News\News; | 10 | use App\Models\News\News; |
| 11 | use App\Models\Product\Category; | 11 | use App\Models\Product\Category; |
| @@ -77,7 +77,7 @@ class ProjectUpdate extends Command | @@ -77,7 +77,7 @@ class ProjectUpdate extends Command | ||
| 77 | $task->save(); | 77 | $task->save(); |
| 78 | 78 | ||
| 79 | //设置数据库 | 79 | //设置数据库 |
| 80 | - $project = ProjectServer::useProject($task->project_id); | 80 | + $project = ProjectServer::useProject($project_id); |
| 81 | if ($project) { | 81 | if ($project) { |
| 82 | if ($api_type == 'category') { | 82 | if ($api_type == 'category') { |
| 83 | //分类 | 83 | //分类 |
| @@ -86,48 +86,6 @@ class ProjectUpdate extends Command | @@ -86,48 +86,6 @@ class ProjectUpdate extends Command | ||
| 86 | if (isset($data['code']) && $data['code'] == 200) { | 86 | if (isset($data['code']) && $data['code'] == 200) { |
| 87 | $items = $data['data'] ?? []; | 87 | $items = $data['data'] ?? []; |
| 88 | $this->category_insert($project_id, $items, 0); | 88 | $this->category_insert($project_id, $items, 0); |
| 89 | -// $model = new Category(); | ||
| 90 | -// foreach ($items as $item) { | ||
| 91 | -// $parent = $model->read(['pid' => 0, 'title' => $item['name']], 'id'); | ||
| 92 | -// if (!$parent) { | ||
| 93 | -// try { | ||
| 94 | -// $parent_id = $model->addReturnId([ | ||
| 95 | -// 'project_id' => $project_id, | ||
| 96 | -// 'title' => $item['name'], | ||
| 97 | -// 'pid' => 0, | ||
| 98 | -// 'keywords' => $item['keywords'], | ||
| 99 | -// 'describe' => $item['description'] | ||
| 100 | -// ]); | ||
| 101 | -// $route = RouteMap::setRoute($item['url'] ? $this->get_url_route($item['url']) : $item['name'], RouteMap::SOURCE_PRODUCT_CATE, $parent_id, $project_id); | ||
| 102 | -// $model->edit(['route' => $route], ['id' => $parent_id]); | ||
| 103 | -// } catch (\Exception $e) { | ||
| 104 | -// echo 'date:' . date('Y-m-d H:i:s') . ', task_id: ' . $task->id . ', error: ' . $e->getMessage() . PHP_EOL; | ||
| 105 | -// continue; | ||
| 106 | -// } | ||
| 107 | -// } else { | ||
| 108 | -// $parent_id = $parent['id']; | ||
| 109 | -// } | ||
| 110 | -// | ||
| 111 | -// foreach ($item['children'] as $child) { | ||
| 112 | -// $child_info = $model->read(['pid' => $parent_id, 'title' => $child['name']]); | ||
| 113 | -// if (!$child_info) { | ||
| 114 | -// try { | ||
| 115 | -// $child_id = $model->addReturnId([ | ||
| 116 | -// 'project_id' => $project_id, | ||
| 117 | -// 'title' => $child['name'], | ||
| 118 | -// 'pid' => $parent_id, | ||
| 119 | -// 'keywords' => $child['keywords'], | ||
| 120 | -// 'describe' => $child['description'] | ||
| 121 | -// ]); | ||
| 122 | -// $route = RouteMap::setRoute($child['url'] ? $this->get_url_route($child['url']) : $child['name'], RouteMap::SOURCE_PRODUCT_CATE, $child_id, $project_id); | ||
| 123 | -// $model->edit(['route' => $route], ['id' => $child_id]); | ||
| 124 | -// } catch (\Exception $e) { | ||
| 125 | -// echo 'date:' . date('Y-m-d H:i:s') . ', task_id: ' . $task->id . ', error: ' . $e->getMessage() . PHP_EOL; | ||
| 126 | -// continue; | ||
| 127 | -// } | ||
| 128 | -// } | ||
| 129 | -// } | ||
| 130 | -// } | ||
| 131 | } else { | 89 | } else { |
| 132 | return true; | 90 | return true; |
| 133 | } | 91 | } |
| @@ -164,7 +122,11 @@ class ProjectUpdate extends Command | @@ -164,7 +122,11 @@ class ProjectUpdate extends Command | ||
| 164 | echo 'date:' . date('Y-m-d H:i:s') . ', task_id: ' . $task->id . ', error: ' . $e->getMessage() . PHP_EOL; | 122 | echo 'date:' . date('Y-m-d H:i:s') . ', task_id: ' . $task->id . ', error: ' . $e->getMessage() . PHP_EOL; |
| 165 | continue; | 123 | continue; |
| 166 | } | 124 | } |
| 125 | + } else { | ||
| 126 | + $id = $keyword['id']; | ||
| 167 | } | 127 | } |
| 128 | + | ||
| 129 | + CollectTask::_insert($item['url'], $project_id, RouteMap::SOURCE_PRODUCT_KEYWORD, $id); | ||
| 168 | } | 130 | } |
| 169 | } | 131 | } |
| 170 | } | 132 | } |
| @@ -279,7 +241,11 @@ class ProjectUpdate extends Command | @@ -279,7 +241,11 @@ class ProjectUpdate extends Command | ||
| 279 | echo 'date:' . date('Y-m-d H:i:s') . ', task_id: ' . $task->id . ', error: ' . $e->getMessage() . PHP_EOL; | 241 | echo 'date:' . date('Y-m-d H:i:s') . ', task_id: ' . $task->id . ', error: ' . $e->getMessage() . PHP_EOL; |
| 280 | continue; | 242 | continue; |
| 281 | } | 243 | } |
| 244 | + } else { | ||
| 245 | + $id = $product['id']; | ||
| 282 | } | 246 | } |
| 247 | + | ||
| 248 | + CollectTask::_insert($item['url'], $project_id, RouteMap::SOURCE_PRODUCT, $id); | ||
| 283 | } | 249 | } |
| 284 | } | 250 | } |
| 285 | } | 251 | } |
| @@ -329,7 +295,11 @@ class ProjectUpdate extends Command | @@ -329,7 +295,11 @@ class ProjectUpdate extends Command | ||
| 329 | echo 'date:' . date('Y-m-d H:i:s') . ', task_id: ' . $task->id . ', error: ' . $e->getMessage() . PHP_EOL; | 295 | echo 'date:' . date('Y-m-d H:i:s') . ', task_id: ' . $task->id . ', error: ' . $e->getMessage() . PHP_EOL; |
| 330 | continue; | 296 | continue; |
| 331 | } | 297 | } |
| 298 | + } else { | ||
| 299 | + $id = $news['id']; | ||
| 332 | } | 300 | } |
| 301 | + | ||
| 302 | + CollectTask::_insert($item['url'], $project_id, $api_type == 'news' ? RouteMap::SOURCE_NEWS : RouteMap::SOURCE_BLOG, $id); | ||
| 333 | } | 303 | } |
| 334 | } | 304 | } |
| 335 | } | 305 | } |
| @@ -373,7 +343,11 @@ class ProjectUpdate extends Command | @@ -373,7 +343,11 @@ class ProjectUpdate extends Command | ||
| 373 | echo 'date:' . date('Y-m-d H:i:s') . ', task_id: ' . $task->id . ', error: ' . $e->getMessage() . PHP_EOL; | 343 | echo 'date:' . date('Y-m-d H:i:s') . ', task_id: ' . $task->id . ', error: ' . $e->getMessage() . PHP_EOL; |
| 374 | continue; | 344 | continue; |
| 375 | } | 345 | } |
| 346 | + } else { | ||
| 347 | + $id = $custom['id']; | ||
| 376 | } | 348 | } |
| 349 | + | ||
| 350 | + CollectTask::_insert($item['url'], $project_id, RouteMap::SOURCE_PAGE, $id); | ||
| 377 | } | 351 | } |
| 378 | } | 352 | } |
| 379 | } | 353 | } |
| @@ -423,6 +397,7 @@ class ProjectUpdate extends Command | @@ -423,6 +397,7 @@ class ProjectUpdate extends Command | ||
| 423 | return $arr[count($arr) - 2]; | 397 | return $arr[count($arr) - 2]; |
| 424 | } | 398 | } |
| 425 | 399 | ||
| 400 | + //多级分类入库 | ||
| 426 | protected function category_insert($project_id, $items, $pid = 0) | 401 | protected function category_insert($project_id, $items, $pid = 0) |
| 427 | { | 402 | { |
| 428 | $model = new Category(); | 403 | $model = new Category(); |
| 1 | +<?php | ||
| 2 | + | ||
| 3 | +namespace App\Http\Controllers\Aside; | ||
| 4 | + | ||
| 5 | +use App\Http\Logic\Aside\CollectLogic; | ||
| 6 | + | ||
| 7 | + | ||
| 8 | +/** | ||
| 9 | + * 提供给AICC采集 | ||
| 10 | + * Class CollectController | ||
| 11 | + * @package App\Http\Controllers\Aside | ||
| 12 | + * @author zbj | ||
| 13 | + * @date 2023/11/10 | ||
| 14 | + */ | ||
| 15 | +class CollectController extends BaseController | ||
| 16 | +{ | ||
| 17 | + /** | ||
| 18 | + * @author zbj | ||
| 19 | + * @date 2023/11/10 | ||
| 20 | + */ | ||
| 21 | + public function index(CollectLogic $collectLogic) | ||
| 22 | + { | ||
| 23 | + $data = $collectLogic->collect_data(); | ||
| 24 | + return $this->success($data); | ||
| 25 | + } | ||
| 26 | +} |
| @@ -801,7 +801,7 @@ class ProjectController extends BaseController | @@ -801,7 +801,7 @@ class ProjectController extends BaseController | ||
| 801 | 'project_id.required' => 'project_id不能为空', | 801 | 'project_id.required' => 'project_id不能为空', |
| 802 | ]); | 802 | ]); |
| 803 | 803 | ||
| 804 | - $token = $logic->getAiccToken($this->map); | 804 | + $token = $logic->getSiteToken($this->map); |
| 805 | 805 | ||
| 806 | $this->response('success',Code::SUCCESS,['site_token' => $token]); | 806 | $this->response('success',Code::SUCCESS,['site_token' => $token]); |
| 807 | 807 |
app/Http/Logic/Aside/CollectLogic.php
0 → 100644
| 1 | +<?php | ||
| 2 | + | ||
| 3 | +namespace App\Http\Logic\Aside; | ||
| 4 | + | ||
| 5 | +use App\Helper\Arr; | ||
| 6 | +use App\Http\Logic\Logic; | ||
| 7 | +use App\Models\Blog\Blog; | ||
| 8 | +use App\Models\Domain\DomainInfo; | ||
| 9 | +use App\Models\News\News; | ||
| 10 | +use App\Models\Product\Keyword; | ||
| 11 | +use App\Models\Product\Product; | ||
| 12 | +use App\Models\Project\Project; | ||
| 13 | +use App\Services\ProjectServer; | ||
| 14 | + | ||
| 15 | + | ||
| 16 | +/** | ||
| 17 | + * Class CollectLogic | ||
| 18 | + * @package App\Http\Logic\Aside | ||
| 19 | + * @author zbj | ||
| 20 | + * @date 2023/11/10 | ||
| 21 | + */ | ||
| 22 | +class CollectLogic extends Logic | ||
| 23 | +{ | ||
| 24 | + protected $project; | ||
| 25 | + protected $domain; | ||
| 26 | + protected $type; | ||
| 27 | + protected $page_size = 100; | ||
| 28 | + | ||
| 29 | + public function __construct() | ||
| 30 | + { | ||
| 31 | + $this->checkAuth(); | ||
| 32 | + } | ||
| 33 | + | ||
| 34 | + /** | ||
| 35 | + * 校验权限 | ||
| 36 | + * @throws \App\Exceptions\AsideGlobalException | ||
| 37 | + * @throws \App\Exceptions\BsideGlobalException | ||
| 38 | + * @author zbj | ||
| 39 | + * @date 2023/11/10 | ||
| 40 | + */ | ||
| 41 | + public function checkAuth() | ||
| 42 | + { | ||
| 43 | + $request = request(); | ||
| 44 | + $site_token = $request->header('site-token'); | ||
| 45 | + $domain = $request->input('domain'); | ||
| 46 | + if (!$site_token) { | ||
| 47 | + $this->fail('参数异常'); | ||
| 48 | + } | ||
| 49 | + $this->project = Project::where('site_token', $site_token)->first(); | ||
| 50 | + if (!$this->project) { | ||
| 51 | + $this->fail('授权码无效'); | ||
| 52 | + } | ||
| 53 | + $domain_info = DomainInfo::where('project_id', $this->project->id)->where('domain', $domain)->first(); | ||
| 54 | + if (!$domain_info) { | ||
| 55 | + $this->fail('域名不匹配'); | ||
| 56 | + } | ||
| 57 | + $this->domain = 'https://' . $domain_info['domain'] . '/'; | ||
| 58 | + $this->type = $request->input('type', ''); | ||
| 59 | + } | ||
| 60 | + | ||
| 61 | + public function collect_data() | ||
| 62 | + { | ||
| 63 | + ProjectServer::useProject($this->project->id); | ||
| 64 | + $action = $this->type; | ||
| 65 | + return $this->$action(); | ||
| 66 | + } | ||
| 67 | + | ||
| 68 | + public function __call($name, $param) | ||
| 69 | + { | ||
| 70 | + return []; | ||
| 71 | + } | ||
| 72 | + | ||
| 73 | + | ||
| 74 | + public function product() | ||
| 75 | + { | ||
| 76 | + $this->model = new Product(); | ||
| 77 | + $where[] = ['status' => Product::STATUS_ON]; | ||
| 78 | + $sort = ['sort' => 'desc']; | ||
| 79 | + $columns = ['title', 'content', 'gallery', 'seo_mate', 'intro', 'route', 'keyword_id']; | ||
| 80 | + $list = self::getList($where,$sort, $columns, $this->page_size); | ||
| 81 | + $data =[]; | ||
| 82 | + foreach ($list['list'] as $item){ | ||
| 83 | + //关键词标签 没有就取seo 键词 | ||
| 84 | + if($item['keyword_id']){ | ||
| 85 | + $keyword = Keyword::whereIn('id', $item['keyword_id'])->pluck('title')->toArray(); | ||
| 86 | + if($keyword){ | ||
| 87 | + $keyword = implode(',', $keyword); | ||
| 88 | + } | ||
| 89 | + } | ||
| 90 | + $keyword = $keyword ?: ($item['seo_mate']['keyword'] ?? ''); | ||
| 91 | + $data[] = [ | ||
| 92 | + 'title' => $item['title'], | ||
| 93 | + 'url' => $this->domain . $item['route'], | ||
| 94 | + 'keywords' => $keyword, | ||
| 95 | + 'description' => strip_tags($item['intro']?:''), | ||
| 96 | + 'content' => strip_tags($item['content'] ?: ''), | ||
| 97 | + 'img' => array_column($item['gallery'] ?: [], 'url') | ||
| 98 | + ]; | ||
| 99 | + } | ||
| 100 | + $list['list'] = $data; | ||
| 101 | + return $list; | ||
| 102 | + } | ||
| 103 | + | ||
| 104 | + public function news() | ||
| 105 | + { | ||
| 106 | + $this->model = new News(); | ||
| 107 | + $where[] = ['status' => News::STATUS_ONE]; | ||
| 108 | + $sort = ['sort' => 'desc']; | ||
| 109 | + $columns = ['name', 'text', 'image', 'seo_keywords', 'remark', 'url']; | ||
| 110 | + $list = self::getList($where,$sort, $columns, $this->page_size); | ||
| 111 | + $data =[]; | ||
| 112 | + foreach ($list['list'] as $item){ | ||
| 113 | + $data[] = [ | ||
| 114 | + 'title' => $item['name'], | ||
| 115 | + 'url' => $this->domain . $item['url'], | ||
| 116 | + 'keywords' => $item['seo_keywords'], | ||
| 117 | + 'description' => strip_tags($item['remark']?:''), | ||
| 118 | + 'content' => strip_tags($item['text'] ?: ''), | ||
| 119 | + 'img' => $item['image'] ?:'' | ||
| 120 | + ]; | ||
| 121 | + } | ||
| 122 | + $list['list'] = $data; | ||
| 123 | + return $list; | ||
| 124 | + } | ||
| 125 | + | ||
| 126 | + public function blog() | ||
| 127 | + { | ||
| 128 | + $this->model = new Blog(); | ||
| 129 | + $where[] = ['status' => Blog::STATUS_ONE]; | ||
| 130 | + $sort = ['sort' => 'desc']; | ||
| 131 | + $columns = ['name', 'text', 'image', 'seo_keywords', 'remark', 'url']; | ||
| 132 | + $list = self::getList($where,$sort, $columns, $this->page_size); | ||
| 133 | + $data =[]; | ||
| 134 | + foreach ($list['list'] as $item){ | ||
| 135 | + $data[] = [ | ||
| 136 | + 'title' => $item['name'], | ||
| 137 | + 'url' => $this->domain . $item['url'], | ||
| 138 | + 'keywords' => $item['seo_keywords'], | ||
| 139 | + 'description' => strip_tags($item['remark']?:''), | ||
| 140 | + 'content' => strip_tags($item['text'] ?: ''), | ||
| 141 | + 'img' => $item['image'] ?:'' | ||
| 142 | + ]; | ||
| 143 | + } | ||
| 144 | + $list['list'] = $data; | ||
| 145 | + return $list; | ||
| 146 | + } | ||
| 147 | +} |
| @@ -651,6 +651,7 @@ class ProjectLogic extends BaseLogic | @@ -651,6 +651,7 @@ class ProjectLogic extends BaseLogic | ||
| 651 | $query->select('*')->from("{$name}"); | 651 | $query->select('*')->from("{$name}"); |
| 652 | } | 652 | } |
| 653 | ); | 653 | ); |
| 654 | + | ||
| 654 | if (Schema::connection('custom_mysql')->hasColumn($table, 'project_id')) { | 655 | if (Schema::connection('custom_mysql')->hasColumn($table, 'project_id')) { |
| 655 | DB::connection('custom_mysql')->table($table)->update(['project_id' => $news_project_id]); | 656 | DB::connection('custom_mysql')->table($table)->update(['project_id' => $news_project_id]); |
| 656 | } | 657 | } |
| @@ -659,16 +660,16 @@ class ProjectLogic extends BaseLogic | @@ -659,16 +660,16 @@ class ProjectLogic extends BaseLogic | ||
| 659 | } | 660 | } |
| 660 | 661 | ||
| 661 | /** | 662 | /** |
| 662 | - * 获取AICC采集数据接口token | 663 | + * 对外接口token |
| 663 | * @param $data | 664 | * @param $data |
| 664 | * @return string | 665 | * @return string |
| 665 | * @author zbj | 666 | * @author zbj |
| 666 | * @date 2023/11/10 | 667 | * @date 2023/11/10 |
| 667 | */ | 668 | */ |
| 668 | - public function getAiccToken($data){ | 669 | + public function getSiteToken($data){ |
| 669 | $project = $this->getCacheInfo($data['project_id']); | 670 | $project = $this->getCacheInfo($data['project_id']); |
| 670 | if(empty($project['site_token']) || !empty($data['refresh'])){ | 671 | if(empty($project['site_token']) || !empty($data['refresh'])){ |
| 671 | - $token = strtolower(Str::random() . base64_encode("globalso_v6")); | 672 | + $token = strtolower(base64_encode("6.0") . md5('project_' . $data['project_id'] . '_' . time())); |
| 672 | $project->site_token = $token; | 673 | $project->site_token = $token; |
| 673 | $project->save(); | 674 | $project->save(); |
| 674 | } | 675 | } |
| @@ -69,8 +69,6 @@ class CustomTemplateLogic extends BaseLogic | @@ -69,8 +69,6 @@ class CustomTemplateLogic extends BaseLogic | ||
| 69 | }else{ | 69 | }else{ |
| 70 | if($this->param['url'] == $this->model::NOT_FOUND_PAGE_URL){ | 70 | if($this->param['url'] == $this->model::NOT_FOUND_PAGE_URL){ |
| 71 | $this->fail('404页面已存在'); | 71 | $this->fail('404页面已存在'); |
| 72 | - }else{ | ||
| 73 | - $this->param['url'] = $this->param['url'].'-tag'; | ||
| 74 | } | 72 | } |
| 75 | $this->param['project_id'] = $this->user['project_id']; | 73 | $this->param['project_id'] = $this->user['project_id']; |
| 76 | $id = $this->model->addReturnId($this->param); | 74 | $id = $this->model->addReturnId($this->param); |
app/Models/Collect/CollectSource.php
0 → 100644
app/Models/Collect/CollectTask.php
0 → 100644
| 1 | +<?php | ||
| 2 | + | ||
| 3 | +namespace App\Models\Collect; | ||
| 4 | + | ||
| 5 | +use App\Models\Base; | ||
| 6 | + | ||
| 7 | +class CollectTask extends Base | ||
| 8 | +{ | ||
| 9 | + //设置关联表名 | ||
| 10 | + protected $table = 'gl_collect_task'; | ||
| 11 | + | ||
| 12 | + //连接数据库 | ||
| 13 | + protected $connection = 'custom_mysql'; | ||
| 14 | + | ||
| 15 | + const STATUS_UN = 0; | ||
| 16 | + const STATUS_ING = 1; | ||
| 17 | + const STATUS_COM= 2; | ||
| 18 | + | ||
| 19 | + public static function _insert($url, $project_id, $source, $source_id) | ||
| 20 | + { | ||
| 21 | + if(!$url){ | ||
| 22 | + return; | ||
| 23 | + } | ||
| 24 | + | ||
| 25 | + $url_arr = parse_url($url); | ||
| 26 | + | ||
| 27 | + $data = [ | ||
| 28 | + 'project_id' => $project_id, | ||
| 29 | + 'source' => $source, | ||
| 30 | + 'source_id' => $source_id, | ||
| 31 | + 'domain' => $url_arr['host'], | ||
| 32 | + 'route' => $url_arr['path'] | ||
| 33 | + ]; | ||
| 34 | + | ||
| 35 | + $task = self::where($data)->first(); | ||
| 36 | + if(!$task){ | ||
| 37 | + $data['created_at'] = $data['updated_at'] = date('Y-m-d H:i:s'); | ||
| 38 | + self::insert($data); | ||
| 39 | + } | ||
| 40 | + } | ||
| 41 | +} |
| @@ -13,6 +13,9 @@ class UpdateLog extends Model | @@ -13,6 +13,9 @@ class UpdateLog extends Model | ||
| 13 | const STATUS_ING = 1;//导入中 | 13 | const STATUS_ING = 1;//导入中 |
| 14 | const STATUS_COM = 2;//导入完成 | 14 | const STATUS_COM = 2;//导入完成 |
| 15 | 15 | ||
| 16 | + const COLLECT_STATUS_UN = 0;//未开始 | ||
| 17 | + const COLLECT_STATUS_COM = 1;//采集完成 | ||
| 18 | + | ||
| 16 | /** | 19 | /** |
| 17 | * 创建更新日志 | 20 | * 创建更新日志 |
| 18 | * @param $project_id | 21 | * @param $project_id |
| @@ -30,6 +33,7 @@ class UpdateLog extends Model | @@ -30,6 +33,7 @@ class UpdateLog extends Model | ||
| 30 | $log->api_type = $type; | 33 | $log->api_type = $type; |
| 31 | $log->api_url = $url; | 34 | $log->api_url = $url; |
| 32 | $log->sort = $type == 'category' ? 0 :1; | 35 | $log->sort = $type == 'category' ? 0 :1; |
| 36 | + $log->collect_status = in_array($type, ['website_info', 'category']) ? 1 : 0; | ||
| 33 | return $log->save(); | 37 | return $log->save(); |
| 34 | } | 38 | } |
| 35 | return true; | 39 | return true; |
| @@ -340,6 +340,8 @@ Route::group([], function () { | @@ -340,6 +340,8 @@ Route::group([], function () { | ||
| 340 | // 提供模板 提单后台查看 | 340 | // 提供模板 提单后台查看 |
| 341 | Route::any('get_template_list', [Aside\Template\ATemplateController::class, 'getTemplateList'])->name('admin.get_template_list'); | 341 | Route::any('get_template_list', [Aside\Template\ATemplateController::class, 'getTemplateList'])->name('admin.get_template_list'); |
| 342 | Route::any('get_template_detail', [Aside\Template\ATemplateController::class, 'getTemplateDetail'])->name('admin.get_template_detail'); | 342 | Route::any('get_template_detail', [Aside\Template\ATemplateController::class, 'getTemplateDetail'])->name('admin.get_template_detail'); |
| 343 | + | ||
| 344 | + Route::any('/collect', [Aside\CollectController::class, 'index'])->name('admin.collect'); | ||
| 343 | }); | 345 | }); |
| 344 | 346 | ||
| 345 | 347 |
-
请 注册 或 登录 后发表评论