周海龙 / globalso · 提交

转到一个项目

作者 lyh
提交 915b8fa5c21283fd70b32a923a6bfc86b5693cc8 2 个父辈 f0c74120 5aa0d8f3

gx

内嵌 并排对比
正在显示 11 个修改的文件 包含 444 行增加 和 29 行删除
@@ -320,7 +320,7 @@ class SyncProject extends Command @@ -320,7 +320,7 @@ class SyncProject extends Command
320 $param['contract'] = Arr::a2s($param['contract']); 320 $param['contract'] = Arr::a2s($param['contract']);
321 $param['bill'] = Arr::a2s($param['bill']); 321 $param['bill'] = Arr::a2s($param['bill']);
322 if(!isset($param['renewal_record'])){ 322 if(!isset($param['renewal_record'])){
323 - $param['renewal_record'] = [["amount"=> null, "remark"=> null, "expire_at"=> null]]; 323 + $param['renewal_record'] = Arr::a2s([["amount"=> null, "remark"=> null, "expire_at"=> null]]);
324 } 324 }
325 $info = $paymentModel->read(['project_id'=>$id]); 325 $info = $paymentModel->read(['project_id'=>$id]);
326 if($info !== false){ 326 if($info !== false){
@@ -5,6 +5,7 @@ namespace App\Console\Commands\Update; @@ -5,6 +5,7 @@ namespace App\Console\Commands\Update;
5 use App\Models\Collect\CollectSource; 5 use App\Models\Collect\CollectSource;
6 use App\Models\Collect\CollectTask; 6 use App\Models\Collect\CollectTask;
7 use App\Models\Com\UpdateLog; 7 use App\Models\Com\UpdateLog;
  8 +use App\Models\RouteMap\RouteMap;
8 use App\Services\CosService; 9 use App\Services\CosService;
9 use App\Services\ProjectServer; 10 use App\Services\ProjectServer;
10 use Illuminate\Console\Command; 11 use Illuminate\Console\Command;
@@ -12,7 +13,7 @@ use Illuminate\Support\Facades\DB; @@ -12,7 +13,7 @@ use Illuminate\Support\Facades\DB;
12 use Illuminate\Support\Facades\Redis; 13 use Illuminate\Support\Facades\Redis;
13 14
14 /** 15 /**
15 - * 4.0,5.0升级到6.0，页面采集 16 + * 4.0,5.0升级到6.0，主站页面采集
16 * Class ProjectImport 17 * Class ProjectImport
17 * @package App\Console\Commands 18 * @package App\Console\Commands
18 * @author Akun 19 * @author Akun
@@ -37,9 +38,9 @@ class HtmlCollect extends Command @@ -37,9 +38,9 @@ class HtmlCollect extends Command
37 38
38 public function handle() 39 public function handle()
39 { 40 {
40 -// while (true) {  
41 - $this->start_collect();  
42 -// } 41 + while (true) {
  42 + $this->start_collect();
  43 + }
43 } 44 }
44 45
45 protected function start_collect() 46 protected function start_collect()
@@ -62,7 +63,7 @@ class HtmlCollect extends Command @@ -62,7 +63,7 @@ class HtmlCollect extends Command
62 //设置数据库 63 //设置数据库
63 $project = ProjectServer::useProject($project_id); 64 $project = ProjectServer::useProject($project_id);
64 if ($project) { 65 if ($project) {
65 - $collect_info = CollectTask::select(['id', 'domain', 'route'])->where('id', $collect_id)->where('status', CollectTask::STATUS_UN)->first(); 66 + $collect_info = CollectTask::select(['id', 'domain', 'route'])->where('id', $collect_id)->where('status', CollectTask::STATUS_UN)->where('language', '')->first();
66 67
67 if (!$collect_info) { 68 if (!$collect_info) {
68 sleep(2); 69 sleep(2);
@@ -83,6 +84,9 @@ class HtmlCollect extends Command @@ -83,6 +84,9 @@ class HtmlCollect extends Command
83 $html = $this->upload_source($html, $source_list, $project_id); 84 $html = $this->upload_source($html, $source_list, $project_id);
84 } 85 }
85 } catch (\Exception $e) { 86 } catch (\Exception $e) {
  87 + $collect_info->status = CollectTask::STATUS_FAIL;
  88 + $collect_info->save();
  89 +
86 echo 'date:' . date('Y-m-d H:i:s') . ', project_id: ' . $project_id . ', collect_id: ' . $collect_id . ', error: ' . $e->getMessage() . PHP_EOL; 90 echo 'date:' . date('Y-m-d H:i:s') . ', project_id: ' . $project_id . ', collect_id: ' . $collect_id . ', error: ' . $e->getMessage() . PHP_EOL;
87 return true; 91 return true;
88 } 92 }
@@ -114,11 +118,29 @@ class HtmlCollect extends Command @@ -114,11 +118,29 @@ class HtmlCollect extends Command
114 return false; 118 return false;
115 } 119 }
116 120
  121 + switch ($update_log->api_type) {
  122 + case 'page':
  123 + $source = RouteMap::SOURCE_PAGE;
  124 + break;
  125 + case 'news':
  126 + $source = RouteMap::SOURCE_NEWS;
  127 + break;
  128 + case 'blog':
  129 + $source = RouteMap::SOURCE_BLOG;
  130 + break;
  131 + case 'tag':
  132 + $source = RouteMap::SOURCE_PRODUCT_KEYWORD;
  133 + break;
  134 + default:
  135 + $source = RouteMap::SOURCE_PRODUCT;
  136 + break;
  137 + }
  138 +
117 $complete = false; 139 $complete = false;
118 //设置数据库 140 //设置数据库
119 $project = ProjectServer::useProject($update_log->project_id); 141 $project = ProjectServer::useProject($update_log->project_id);
120 if ($project) { 142 if ($project) {
121 - $collect_list = CollectTask::select(['id', 'project_id'])->where('project_id', $update_log['project_id'])->where('status', CollectTask::STATUS_UN)->limit(50)->get(); 143 + $collect_list = CollectTask::select(['id', 'project_id'])->where('project_id', $update_log['project_id'])->where('source', $source)->where('language', '')->where('status', CollectTask::STATUS_UN)->orderBy('id', 'asc')->limit(50)->get();
122 144
123 if ($collect_list->count() == 0) { 145 if ($collect_list->count() == 0) {
124 $complete = true; 146 $complete = true;
@@ -132,7 +154,7 @@ class HtmlCollect extends Command @@ -132,7 +154,7 @@ class HtmlCollect extends Command
132 DB::disconnect('custom_mysql'); 154 DB::disconnect('custom_mysql');
133 155
134 if ($complete) { 156 if ($complete) {
135 - $update_log->collect_status = UpdateLog::COLLECT_STATUS_COM; 157 + $update_log->collect_status = UpdateLog::COLLECT_STATUS_MAIN;
136 return 0; 158 return 0;
137 } 159 }
138 160
@@ -203,9 +225,9 @@ class HtmlCollect extends Command @@ -203,9 +225,9 @@ class HtmlCollect extends Command
203 $path = $arr['path'] ?? ''; 225 $path = $arr['path'] ?? '';
204 226
205 if ( 227 if (
206 - (strpos($host, '.globalso.') === false) &&  
207 - (strpos($host, '.goodao.') === false) &&  
208 - $path && (strpos($path, '.') !== false) 228 + (empty($host) || $host == $domain)
  229 + && $path
  230 + && (strpos($path, '.') !== false)
209 ) { 231 ) {
210 232
211 $source = CollectSource::where('project_id', $project_id)->where('origin', $url)->first(); 233 $source = CollectSource::where('project_id', $project_id)->where('origin', $url)->first();
  1 +<?php
  2 +
  3 +namespace App\Console\Commands\Update;
  4 +
  5 +use App\Models\Collect\CollectSource;
  6 +use App\Models\Collect\CollectTask;
  7 +use App\Models\Com\UpdateLog;
  8 +use App\Models\RouteMap\RouteMap;
  9 +use App\Services\CosService;
  10 +use App\Services\ProjectServer;
  11 +use Illuminate\Console\Command;
  12 +use Illuminate\Support\Facades\DB;
  13 +use Illuminate\Support\Facades\Redis;
  14 +
  15 +/**
  16 + * 4.0,5.0升级到6.0，小语种页面采集
  17 + * Class ProjectImport
  18 + * @package App\Console\Commands
  19 + * @author Akun
  20 + * @date 2023/11/20 14:04
  21 + */
  22 +class HtmlLanguageCollect extends Command
  23 +{
  24 + /**
  25 + * The name and signature of the console command.
  26 + *
  27 + * @var string
  28 + */
  29 + protected $signature = 'project_html_language_collect';
  30 +
  31 + /**
  32 + * The console command description.
  33 + *
  34 + * @var string
  35 + */
  36 + protected $description = '执行项目html页面采集';
  37 +
  38 +
  39 + public function handle()
  40 + {
  41 + while (true) {
  42 + $this->start_collect();
  43 + }
  44 + }
  45 +
  46 + protected function start_collect()
  47 + {
  48 + $task_id = $this->get_task();
  49 + if ($task_id === false) {
  50 + //所有项目采集完成
  51 + sleep(60);
  52 + return true;
  53 + } elseif ($task_id === 0) {
  54 + //当前项目采集完成
  55 + sleep(2);
  56 + return true;
  57 + }
  58 +
  59 + $task_arr = explode('_', $task_id);
  60 + $project_id = $task_arr[0];
  61 + $collect_id = $task_arr[1];
  62 +
  63 + //设置数据库
  64 + $project = ProjectServer::useProject($project_id);
  65 + if ($project) {
  66 + $collect_info = CollectTask::select(['id', 'domain', 'route'])->where('id', $collect_id)->where('status', CollectTask::STATUS_UN)->where('language', '!=', '')->first();
  67 +
  68 + if (!$collect_info) {
  69 + sleep(2);
  70 + return true;
  71 + }
  72 +
  73 + echo 'date:' . date('Y-m-d H:i:s') . ', project_id: ' . $project_id . ', collect_id: ' . $collect_id . ', collect start' . PHP_EOL;
  74 +
  75 + $collect_info->status = CollectTask::STATUS_ING;
  76 + $collect_info->save();
  77 +
  78 + //采集html页面，下载资源到本地并替换
  79 + try {
  80 + $html = file_get_contents('https://' . $collect_info->domain . $collect_info->route);
  81 + $source_list = $this->html_preg($html, $project_id, $collect_info->domain);
  82 +
  83 + if ($source_list) {
  84 + $html = $this->upload_source($html, $source_list, $project_id);
  85 + }
  86 + } catch (\Exception $e) {
  87 + $collect_info->status = CollectTask::STATUS_FAIL;
  88 + $collect_info->save();
  89 +
  90 + echo 'date:' . date('Y-m-d H:i:s') . ', project_id: ' . $project_id . ', collect_id: ' . $collect_id . ', error: ' . $e->getMessage() . PHP_EOL;
  91 + return true;
  92 + }
  93 +
  94 + $collect_info->html = $html;
  95 + $collect_info->status = CollectTask::STATUS_COM;
  96 + $collect_info->save();
  97 +
  98 + echo 'date:' . date('Y-m-d H:i:s') . ', project_id: ' . $project_id . ', collect_id: ' . $collect_id . ', collect end' . PHP_EOL;
  99 + }
  100 + //关闭数据库
  101 + DB::disconnect('custom_mysql');
  102 +
  103 + sleep(2);
  104 + }
  105 +
  106 + //获取任务
  107 + protected function get_task()
  108 + {
  109 + $key = 'console_html_language_collect_task';
  110 + $task_id = Redis::rpop($key);
  111 + if ($task_id) {
  112 + return $task_id;
  113 + }
  114 +
  115 +
  116 + $update_log = UpdateLog::where('status', UpdateLog::STATUS_COM)->where('collect_status', UpdateLog::COLLECT_STATUS_MAIN)->orderBy('project_id', 'asc')->first();
  117 + if (!$update_log) {
  118 + return false;
  119 + }
  120 +
  121 + switch ($update_log->api_type) {
  122 + case 'page':
  123 + $source = RouteMap::SOURCE_PAGE;
  124 + break;
  125 + case 'news':
  126 + $source = RouteMap::SOURCE_NEWS;
  127 + break;
  128 + case 'blog':
  129 + $source = RouteMap::SOURCE_BLOG;
  130 + break;
  131 + case 'tag':
  132 + $source = RouteMap::SOURCE_PRODUCT_KEYWORD;
  133 + break;
  134 + default:
  135 + $source = RouteMap::SOURCE_PRODUCT;
  136 + break;
  137 + }
  138 +
  139 + $complete = false;
  140 + //设置数据库
  141 + $project = ProjectServer::useProject($update_log->project_id);
  142 + if ($project) {
  143 + $collect_list = CollectTask::select(['id', 'project_id'])->where('project_id', $update_log['project_id'])->where('source', $source)->where('language', '!=', '')->where('status', CollectTask::STATUS_UN)->orderBy('id', 'asc')->limit(50)->get();
  144 +
  145 + if ($collect_list->count() == 0) {
  146 + $complete = true;
  147 + } else {
  148 + foreach ($collect_list as $collect) {
  149 + Redis::lpush($key, $collect['project_id'] . '_' . $collect['id']);
  150 + }
  151 + }
  152 + }
  153 + //关闭数据库
  154 + DB::disconnect('custom_mysql');
  155 +
  156 + if ($complete) {
  157 + $update_log->collect_status = UpdateLog::COLLECT_STATUS_COM;
  158 + return 0;
  159 + }
  160 +
  161 + $task_id = Redis::rpop($key);
  162 + return $task_id;
  163 + }
  164 +
  165 + //正则匹配html资源
  166 + protected function html_preg($html, $project_id, $domain)
  167 + {
  168 + $source = [];
  169 +
  170 + if (!$html) {
  171 + return $source;
  172 + }
  173 +
  174 + //image
  175 + preg_match_all('/<img\s+[^>]*?src\s*=\s*(\'|\")(.*?)\\1[^>]*?\/?\s*>/i', $html, $result_img);
  176 + $img = $result_img[2] ?? [];
  177 + foreach ($img as $vi) {
  178 + $check_vi = $this->url_check($vi, $project_id, $domain);
  179 + $check_vi && $source[] = $check_vi;
  180 + }
  181 +
  182 + //js
  183 + preg_match_all('/<script\s+[^>]*?src\s*=\s*(\'|\")(.*?)\\1[^>]*?\/?\s*>/i', $html, $result_js);
  184 + $js = $result_js[2] ?? [];
  185 + foreach ($js as $vj) {
  186 + $check_vj = $this->url_check($vj, $project_id, $domain);
  187 + $check_vj && $source[] = $check_vj;
  188 + }
  189 +
  190 + //video
  191 + preg_match_all('/<source\s+[^>]*?src\s*=\s*(\'|\")(.*?)\\1[^>]*?\/?\s*>/i', $html, $result_video);
  192 + $video = $result_video[2] ?? [];
  193 + foreach ($video as $vv) {
  194 + $check_vv = $this->url_check($vv, $project_id, $domain);
  195 + $check_vv && $source[] = $check_vv;
  196 + }
  197 +
  198 + //css
  199 + preg_match_all('/<link\s+[^>]*?href\s*=\s*(\'|\")(.*?)\\1[^>]*?\/?\s*>/i', $html, $result_css);
  200 + $css = $result_css[2] ?? [];
  201 + foreach ($css as $vc) {
  202 + $check_vc = $this->url_check($vc, $project_id, $domain);
  203 + $check_vc && $source[] = $check_vc;
  204 + }
  205 +
  206 + //css background
  207 + preg_match_all("/url\(['\"]?(\s*[^>]+?)['\"]?\)/i", $html, $result_css_b);
  208 + $css_b = $result_css_b[1] ?? [];
  209 + foreach ($css_b as $vc_b) {
  210 + $check_vc_b = $this->url_check($vc_b, $project_id, $domain);
  211 + $check_vc_b && $source[] = $check_vc_b;
  212 + }
  213 +
  214 +
  215 + return $source;
  216 + }
  217 +
  218 + //判断资源是否需要下载
  219 + protected function url_check($url, $project_id, $domain)
  220 + {
  221 + if ($url) {
  222 + $arr = parse_url($url);
  223 + $scheme = $arr['scheme'] ?? '';
  224 + $host = $arr['host'] ?? '';
  225 + $path = $arr['path'] ?? '';
  226 +
  227 + if (
  228 + (empty($host) || $host == $domain)
  229 + && $path
  230 + && (strpos($path, '.') !== false)
  231 + ) {
  232 +
  233 + $source = CollectSource::where('project_id', $project_id)->where('origin', $url)->first();
  234 + if (!$source) {
  235 + return [
  236 + 'download' => true,
  237 + 'url' => $url,
  238 + 'url_complete' => ($scheme ?: 'https') . '://' . ($host ?: $domain) . $path
  239 + ];
  240 + } else {
  241 + return [
  242 + 'download' => false,
  243 + 'url' => $url,
  244 + 'url_complete' => $source['target']
  245 + ];
  246 + }
  247 + } else {
  248 + return false;
  249 + }
  250 + } else {
  251 + return false;
  252 + }
  253 + }
  254 +
  255 + //下载并替换资源
  256 + protected function upload_source($html, $source, $project_id)
  257 + {
  258 + foreach ($source as $vs) {
  259 +
  260 + if ($vs['download']) {
  261 + $new_source = CosService::uploadRemote($project_id, 'source', $vs['url_complete']);
  262 + if ($new_source) {
  263 + CollectSource::insert([
  264 + 'project_id' => $project_id,
  265 + 'origin' => $vs['url'],
  266 + 'target' => $new_source,
  267 + 'created_at' => date('Y-m-d H:i:s'),
  268 + 'updated_at' => date('Y-m-d H:i:s'),
  269 + ]);
  270 + $html = str_replace($vs['url'], getImageUrl($new_source), $html);
  271 +
  272 + if (substr($new_source, -3, 3) == 'css') {
  273 + // 下载css文件中的资源
  274 + $css_html = file_get_contents($vs['url_complete']);
  275 + preg_match_all("/url\(['\"](\s*[^>]+?)['\"]\)/i", $css_html, $result_css_source);
  276 + $css_source = $result_css_source[1] ?? [];
  277 +
  278 + $url_arr = explode('/', $vs['url_complete']);
  279 + $target_arr = explode('/', $new_source);
  280 + foreach ($css_source as $vcs) {
  281 + $vcs_arr = parse_url($vcs);
  282 + if (isset($vcs_arr['domain'])) {
  283 + //不是相对路径，不下载
  284 + continue;
  285 + }
  286 +
  287 + $vcs = $vcs_arr['path'] ?? '';
  288 + if (!$vcs) {
  289 + continue;
  290 + }
  291 +
  292 + $source_info = CollectSource::where('project_id', $project_id)->where('origin', $vcs)->first();
  293 + if ($source_info) {
  294 + //已存在，不下载
  295 + continue;
  296 + }
  297 +
  298 + $url_arr[count($url_arr) - 1] = $vcs;
  299 + $url_css_complete = implode('/', $url_arr);
  300 + $target_arr[count($target_arr) - 1] = $vcs;
  301 + $path = implode('/', $target_arr);
  302 +
  303 + $new_source_css = CosService::uploadRemote($project_id, 'source', $url_css_complete, $path);
  304 + if ($new_source_css) {
  305 + CollectSource::insert([
  306 + 'project_id' => $project_id,
  307 + 'origin' => $vcs,
  308 + 'target' => $new_source_css,
  309 + 'created_at' => date('Y-m-d H:i:s'),
  310 + 'updated_at' => date('Y-m-d H:i:s'),
  311 + ]);
  312 + }
  313 + }
  314 + }
  315 + }
  316 + } else {
  317 + $html = str_replace($vs['url'], getImageUrl($vs['url_complete']), $html);
  318 + }
  319 + }
  320 +
  321 + return $html;
  322 + }
  323 +}
@@ -76,6 +76,22 @@ class ProjectUpdate extends Command @@ -76,6 +76,22 @@ class ProjectUpdate extends Command
76 $task->status = UpdateLog::STATUS_ING;//同步中 76 $task->status = UpdateLog::STATUS_ING;//同步中
77 $task->save(); 77 $task->save();
78 78
  79 + $domain_arr = parse_url($api_url);
  80 + //获取网站配置
  81 + $link_type = 0;
  82 + $url_web_config = 'https://' . $domain_arr['host'] . '/wp-content/cache/user_config.text';
  83 + $data_config = http_get($url_web_config, ['charset' => 'UTF-8']);
  84 + if ($data_config) {
  85 + $link_type = $data_config['link_type'];
  86 + }
  87 + //获取所有语种
  88 + $language_list = [];
  89 + $url_language = 'https://' . $domain_arr['host'] . '/wp-content/plugins/proofreading/json/user_language.json';
  90 + $data_language = http_get($url_language, ['charset' => 'UTF-8']);
  91 + if ($data_language) {
  92 + $language_list = array_column($data_language, 'short');
  93 + }
  94 +
79 //设置数据库 95 //设置数据库
80 $project = ProjectServer::useProject($project_id); 96 $project = ProjectServer::useProject($project_id);
81 if ($project) { 97 if ($project) {
@@ -126,7 +142,7 @@ class ProjectUpdate extends Command @@ -126,7 +142,7 @@ class ProjectUpdate extends Command
126 $id = $keyword['id']; 142 $id = $keyword['id'];
127 } 143 }
128 144
129 - CollectTask::_insert($item['url'], $project_id, RouteMap::SOURCE_PRODUCT_KEYWORD, $id); 145 + CollectTask::_insert($item['url'], $project_id, RouteMap::SOURCE_PRODUCT_KEYWORD, $id, $link_type, $language_list);
130 } 146 }
131 } 147 }
132 } 148 }
@@ -258,7 +274,7 @@ class ProjectUpdate extends Command @@ -258,7 +274,7 @@ class ProjectUpdate extends Command
258 $id = $product['id']; 274 $id = $product['id'];
259 } 275 }
260 276
261 - CollectTask::_insert($item['url'], $project_id, RouteMap::SOURCE_PRODUCT, $id); 277 + CollectTask::_insert($item['url'], $project_id, RouteMap::SOURCE_PRODUCT, $id, $link_type, $language_list);
262 } 278 }
263 } 279 }
264 } 280 }
@@ -312,7 +328,7 @@ class ProjectUpdate extends Command @@ -312,7 +328,7 @@ class ProjectUpdate extends Command
312 $id = $news['id']; 328 $id = $news['id'];
313 } 329 }
314 330
315 - CollectTask::_insert($item['url'], $project_id, $api_type == 'news' ? RouteMap::SOURCE_NEWS : RouteMap::SOURCE_BLOG, $id); 331 + CollectTask::_insert($item['url'], $project_id, $api_type == 'news' ? RouteMap::SOURCE_NEWS : RouteMap::SOURCE_BLOG, $id, $link_type, $language_list);
316 } 332 }
317 } 333 }
318 } 334 }
@@ -360,7 +376,7 @@ class ProjectUpdate extends Command @@ -360,7 +376,7 @@ class ProjectUpdate extends Command
360 $id = $custom['id']; 376 $id = $custom['id'];
361 } 377 }
362 378
363 - CollectTask::_insert($item['url'], $project_id, RouteMap::SOURCE_PAGE, $id); 379 + CollectTask::_insert($item['url'], $project_id, RouteMap::SOURCE_PAGE, $id, $link_type, $language_list);
364 } 380 }
365 } 381 }
366 } 382 }
@@ -67,6 +67,7 @@ class UpdateProgress extends Command @@ -67,6 +67,7 @@ class UpdateProgress extends Command
67 public function getUpdateProgress($project_id){ 67 public function getUpdateProgress($project_id){
68 $info = DB::connection('custom_mysql')->table('gl_update_progress')->whereRaw('total_num > current_num')->first(); 68 $info = DB::connection('custom_mysql')->table('gl_update_progress')->whereRaw('total_num > current_num')->first();
69 if(!empty($info)){ 69 if(!empty($info)){
  70 + $info = (array)$info;
70 //超时时间 71 //超时时间
71 $time = date("Y-m-d H:i:s",strtotime($info['created_at']) + $info['total_num'] * 60); 72 $time = date("Y-m-d H:i:s",strtotime($info['created_at']) + $info['total_num'] * 60);
72 if($time > date("Y-m-d H:i:s")){ 73 if($time > date("Y-m-d H:i:s")){
@@ -84,7 +84,7 @@ if (!function_exists('http_get')) { @@ -84,7 +84,7 @@ if (!function_exists('http_get')) {
84 /** 84 /**
85 * 发送http get请求 85 * 发送http get请求
86 * @param type $url 86 * @param type $url
87 - * @return type 87 + * @return []
88 */ 88 */
89 function http_get($url, $header = []) 89 function http_get($url, $header = [])
90 { 90 {
@@ -215,7 +215,7 @@ class DomainInfoLogic extends BaseLogic @@ -215,7 +215,7 @@ class DomainInfoLogic extends BaseLogic
215 } 215 }
216 216
217 //域名是否都已经解析 217 //域名是否都已经解析
218 - if($info['domain'] && !$this->check_cname($info['domain'], $server_info)){ 218 + if(!empty($info['domain']) && !$this->check_cname($info['domain'], $server_info)){
219 $this->fail('域名' . $info['domain'] . '未解析至目标服务器'); 219 $this->fail('域名' . $info['domain'] . '未解析至目标服务器');
220 } 220 }
221 foreach ($this->param['other_domain']??[] as $other_domain){ 221 foreach ($this->param['other_domain']??[] as $other_domain){
@@ -533,7 +533,7 @@ class ProjectLogic extends BaseLogic @@ -533,7 +533,7 @@ class ProjectLogic extends BaseLogic
533 533
534 if($server_info && $domain_info){ 534 if($server_info && $domain_info){
535 //验证解析 535 //验证解析
536 - if ($domain_info['domain'] && !DomainInfoLogic::instance()->check_cname($domain_info['domain'], $server_info)) { 536 + if (!empty($domain_info['domain']) && !DomainInfoLogic::instance()->check_cname($domain_info['domain'], $server_info)) {
537 throw new AsideGlobalException(Code::SYSTEM_ERROR,'域名' . $domain_info['domain'] . '未解析至目标服务器'); 537 throw new AsideGlobalException(Code::SYSTEM_ERROR,'域名' . $domain_info['domain'] . '未解析至目标服务器');
538 } 538 }
539 $api_url = 'http://'.$server_info['init_domain'].'/api/createSite'; 539 $api_url = 'http://'.$server_info['init_domain'].'/api/createSite';
@@ -14,27 +14,67 @@ class CollectTask extends Base @@ -14,27 +14,67 @@ class CollectTask extends Base
14 14
15 const STATUS_UN = 0; 15 const STATUS_UN = 0;
16 const STATUS_ING = 1; 16 const STATUS_ING = 1;
17 - const STATUS_COM= 2; 17 + const STATUS_COM = 2;
  18 + const STATUS_FAIL = 3;
18 19
19 - public static function _insert($url, $project_id, $source, $source_id) 20 + public static function _insert($url, $project_id, $source, $source_id, $link_type = 0, $language_list = [])
20 { 21 {
21 - if(!$url){ 22 + if (!$url) {
22 return; 23 return;
23 } 24 }
24 25
25 $url_arr = parse_url($url); 26 $url_arr = parse_url($url);
26 27
27 - $data = [ 28 + $where = [
28 'project_id' => $project_id, 29 'project_id' => $project_id,
29 'source' => $source, 30 'source' => $source,
30 'source_id' => $source_id, 31 'source_id' => $source_id,
31 'domain' => $url_arr['host'], 32 'domain' => $url_arr['host'],
32 - 'route' => $url_arr['path'] 33 + 'route' => $url_arr['path'],
  34 + 'language' => ''
33 ]; 35 ];
34 36
35 - $task = self::where($data)->first();  
36 - if(!$task){  
37 - $data['created_at'] = $data['updated_at'] = date('Y-m-d H:i:s'); 37 + $task = self::where($where)->first();
  38 + if (!$task) {
  39 + $now = date('Y-m-d H:i:s');
  40 + $data = [
  41 + [
  42 + 'project_id' => $project_id,
  43 + 'source' => $source,
  44 + 'source_id' => $source_id,
  45 + 'domain' => $url_arr['host'],
  46 + 'route' => $url_arr['path'],
  47 + 'language' => '',
  48 + 'created_at' => $now,
  49 + 'updated_at' => $now,
  50 + ]
  51 + ];
  52 +
  53 + if ($link_type > 0 && $language_list) {
  54 + $domain_arr = explode('.', $url_arr['host']);
  55 + foreach ($language_list as $v_lan) {
  56 + if ($link_type == 1) {
  57 + //二级域名
  58 + $domain_arr[0] = $v_lan;
  59 + $new_domain = implode('.', $domain_arr);
  60 + } else {
  61 + //二级目录
  62 + $new_domain = $url_arr['host'] . '/' . $v_lan;
  63 + }
  64 +
  65 + $data[] = [
  66 + 'project_id' => $project_id,
  67 + 'source' => $source,
  68 + 'source_id' => $source_id,
  69 + 'domain' => $new_domain,
  70 + 'route' => $url_arr['path'],
  71 + 'language' => $v_lan,
  72 + 'created_at' => $now,
  73 + 'updated_at' => $now,
  74 + ];
  75 + }
  76 + }
  77 +
38 self::insert($data); 78 self::insert($data);
39 } 79 }
40 } 80 }
@@ -14,7 +14,8 @@ class UpdateLog extends Model @@ -14,7 +14,8 @@ class UpdateLog extends Model
14 const STATUS_COM = 2;//导入完成 14 const STATUS_COM = 2;//导入完成
15 15
16 const COLLECT_STATUS_UN = 0;//未开始 16 const COLLECT_STATUS_UN = 0;//未开始
17 - const COLLECT_STATUS_COM = 1;//采集完成 17 + const COLLECT_STATUS_COM = 1;//全站小语种采集完成
  18 + const COLLECT_STATUS_MAIN = 2;//英语主站采集完成
18 19
19 /** 20 /**
20 * 创建更新日志 21 * 创建更新日志
@@ -33,7 +34,7 @@ class UpdateLog extends Model @@ -33,7 +34,7 @@ class UpdateLog extends Model
33 $log->api_type = $type; 34 $log->api_type = $type;
34 $log->api_url = $url; 35 $log->api_url = $url;
35 $log->sort = $type == 'category' ? 0 :1; 36 $log->sort = $type == 'category' ? 0 :1;
36 - $log->collect_status = in_array($type, ['website_info', 'category']) ? 1 : 0; 37 + $log->collect_status = ($type == 'category' || $type == 'website_info') ? 1 : 0;
37 return $log->save(); 38 return $log->save();
38 } 39 }
39 return true; 40 return true;
@@ -93,11 +93,23 @@ class CosService @@ -93,11 +93,23 @@ class CosService
93 'secretKey' => $cos['credentials']['secretKey'], 93 'secretKey' => $cos['credentials']['secretKey'],
94 ], 94 ],
95 ]); 95 ]);
  96 +
  97 + $opts = [
  98 + 'http' => [
  99 + 'header' => 'User-Agent:Mozilla/5.0 (Windows NT 6.2; WOW64; rv:32.0) Gecko/20100101 Firefox/32.0'
  100 + ],
  101 + 'ssl' => [
  102 + 'verify_peer' => false,
  103 + 'verify_peer_name' => false,
  104 + ]
  105 + ];
  106 + $body = file_get_contents($file_url,false,stream_context_create($opts));
  107 +
96 try { 108 try {
97 $cosClient->putObject([ 109 $cosClient->putObject([
98 'Bucket' => $cos['bucket'], 110 'Bucket' => $cos['bucket'],
99 'Key' => $key, 111 'Key' => $key,
100 - 'Body' => fopen($file_url, 'r'), 112 + 'Body' => $body,
101 ]); 113 ]);
102 return $key; 114 return $key;
103 }catch (\Exception $e){ 115 }catch (\Exception $e){