周海龙 / globalso · 提交

转到一个项目

作者 刘锟
提交 a2d4fd8cf8018e20a3cde07083391e43c49c84c3 2 个父辈 f5442603 feb03f3f

合并分支 'akun' 到 'master' 

Akun



查看合并请求 !172
内嵌 并排对比
正在显示 2 个修改的文件 包含 67 行增加 和 60 行删除
@@ -39,15 +39,14 @@ class HtmlCollect extends Command @@ -39,15 +39,14 @@ class HtmlCollect extends Command
39 39
40 public function handle() 40 public function handle()
41 { 41 {
42 -// while (true) { 42 + while (true) {
43 $this->start_collect(); 43 $this->start_collect();
44 -// } 44 + }
45 } 45 }
46 46
47 protected function start_collect() 47 protected function start_collect()
48 { 48 {
49 -// $task_id = $this->get_task();  
50 - $task_id = '595_41517'; 49 + $task_id = $this->get_task();
51 if ($task_id === false) { 50 if ($task_id === false) {
52 //所有项目采集完成 51 //所有项目采集完成
53 sleep(60); 52 sleep(60);
@@ -336,62 +336,25 @@ class HtmlLanguageCollect extends Command @@ -336,62 +336,25 @@ class HtmlLanguageCollect extends Command
336 ]); 336 ]);
337 $html = str_replace($vs['url'], getImageUrl($new_source), $html); 337 $html = str_replace($vs['url'], getImageUrl($new_source), $html);
338 338
339 - if (substr($new_source, -3, 3) == 'css') {  
340 - // 下载css文件中的资源  
341 - $css_html = curl_c($vs['url_complete'], false);  
342 - preg_match_all("/url\(['\"](\s*[^>]+?)['\"]\)/i", $css_html, $result_css_source);  
343 - $css_source = $result_css_source[1] ?? [];  
344 -  
345 - $url_arr = explode('/', $vs['url_complete']);  
346 - $target_arr = explode('/', $new_source);  
347 - foreach ($css_source as $vcs) {  
348 - $vcs = str_replace('"', '', $vcs);  
349 - $vcs_arr = parse_url($vcs);  
350 - if (isset($vcs_arr['domain'])) {  
351 - //不是相对路径，不下载  
352 - continue;  
353 - } 339 + if (substr($new_source, -3, 3) == 'css' || substr($new_source, -2, 2) == 'js') {
354 340
355 - $vcs = $vcs_arr['path'] ?? '';  
356 - if (!$vcs) {  
357 - continue;  
358 - }  
359 - if (strpos($vcs, '.') === false) {  
360 - continue;  
361 - }  
362 - $path_arr = explode('.', $vcs);  
363 - if (in_array(end($path_arr), ['html', 'php', 'com', 'xml'])) {  
364 - continue;  
365 - } 341 + $source_html = curl_c(getImageUrl($new_source), false);
366 342
367 - $source_info = CollectSource::where('project_id', $project_id)->where('origin', $vcs)->first();  
368 - if ($source_info) {  
369 - //已存在，不下载  
370 - continue;  
371 - }  
372 -  
373 - $url_arr[count($url_arr) - 1] = $vcs;  
374 - $url_css_complete = implode('/', $url_arr);  
375 - $target_arr[count($target_arr) - 1] = $vcs;  
376 - $path = implode('/', $target_arr);  
377 -  
378 - $new_source_css = CosService::uploadRemote($project_id, 'source', $url_css_complete, $path);  
379 - if ($new_source_css) {  
380 - CollectSource::insert([  
381 - 'project_id' => $project_id,  
382 - 'origin' => $vcs,  
383 - 'target' => $new_source_css,  
384 - 'created_at' => date('Y-m-d H:i:s'),  
385 - 'updated_at' => date('Y-m-d H:i:s'),  
386 - ]);  
387 - } 343 + if (substr($new_source, -3, 3) == 'css') {
  344 + preg_match_all("/url\(['\"]?(\s*[^>]+?)['\"]?\)/i", $source_html, $result_source);
  345 + } else {
  346 + preg_match_all("/[large|thumb]+URL:['\"]+(\s*[^>]+?)['\"]+,/i", $source_html, $result_source);
388 } 347 }
389 - } elseif (substr($new_source, -2, 2) == 'js') {  
390 - $js_html = curl_c(getImageUrl($new_source), false);  
391 - preg_match_all("/[large|thumb]+URL:['\"]+(\s*[^>]+?)['\"]+,/i", $js_html, $result_js_source);  
392 - $js_source = $result_js_source[1] ?? [];  
393 - if ($js_source) {  
394 - foreach ($js_source as $vjs) { 348 +
  349 + $js_css_source = $result_source[1] ?? [];
  350 + if ($js_css_source) {
  351 + foreach ($js_css_source as $vjs) {
  352 + if (substr($vjs, 0, 2) == './') {
  353 + //相对路径
  354 + $url_arr = explode('/', $vs['url']);
  355 + $url_arr[count($url_arr) - 1] = substr($vjs, 2);
  356 + $vjs = implode('/', $url_arr);
  357 + }
395 $vjs_result = $this->url_check($vjs, $project_id, $domain, $web_url_domain, $home_url); 358 $vjs_result = $this->url_check($vjs, $project_id, $domain, $web_url_domain, $home_url);
396 if (!$vjs_result) { 359 if (!$vjs_result) {
397 continue; 360 continue;
@@ -407,19 +370,64 @@ class HtmlLanguageCollect extends Command @@ -407,19 +370,64 @@ class HtmlLanguageCollect extends Command
407 'created_at' => date('Y-m-d H:i:s'), 370 'created_at' => date('Y-m-d H:i:s'),
408 'updated_at' => date('Y-m-d H:i:s'), 371 'updated_at' => date('Y-m-d H:i:s'),
409 ]); 372 ]);
410 - $js_html = str_replace($vjs, getImageUrl($new_vjs), $js_html); 373 + $source_html = str_replace($vjs, getImageUrl($new_vjs), $source_html);
411 } 374 }
412 } else { 375 } else {
413 - $js_html = str_replace($vjs, getImageUrl($vjs_result['url_complete']), $js_html); 376 + $source_html = str_replace($vjs, getImageUrl($vjs_result['url_complete']), $source_html);
414 } 377 }
415 } 378 }
416 379
417 - CosService::uploadRemote($project_id, 'source', $new_source, $new_source, $js_html); 380 + CosService::uploadRemote($project_id, 'source', $new_source, $new_source, $source_html);
418 } 381 }
419 } 382 }
420 } 383 }
421 } else { 384 } else {
422 $html = str_replace($vs['url'], getImageUrl($vs['url_complete']), $html); 385 $html = str_replace($vs['url'], getImageUrl($vs['url_complete']), $html);
  386 +
  387 + if (substr($vs['url_complete'], -3, 3) == 'css' || substr($vs['url_complete'], -2, 2) == 'js') {
  388 +
  389 + $source_html = curl_c(getImageUrl($vs['url_complete']), false);
  390 +
  391 + if (substr($vs['url_complete'], -3, 3) == 'css') {
  392 + preg_match_all("/url\(['\"]?(\s*[^>]+?)['\"]?\)/i", $source_html, $result_source);
  393 + } else {
  394 + preg_match_all("/[large|thumb]+URL:['\"]+(\s*[^>]+?)['\"]+,/i", $source_html, $result_source);
  395 + }
  396 +
  397 + $js_css_source = $result_source[1] ?? [];
  398 + if ($js_css_source) {
  399 + foreach ($js_css_source as $vjs) {
  400 + if (substr($vjs, 0, 2) == './') {
  401 + //相对路径
  402 + $url_arr = explode('/', $vs['url']);
  403 + $url_arr[count($url_arr) - 1] = substr($vjs, 2);
  404 + $vjs = implode('/', $url_arr);
  405 + }
  406 + $vjs_result = $this->url_check($vjs, $project_id, $domain, $web_url_domain, $home_url);
  407 + if (!$vjs_result) {
  408 + continue;
  409 + }
  410 +
  411 + if ($vjs_result['download']) {
  412 + $new_vjs = CosService::uploadRemote($project_id, 'source', $vjs_result['url_complete']);
  413 + if ($new_vjs) {
  414 + CollectSource::insert([
  415 + 'project_id' => $project_id,
  416 + 'origin' => $vjs_result['url'],
  417 + 'target' => $new_vjs,
  418 + 'created_at' => date('Y-m-d H:i:s'),
  419 + 'updated_at' => date('Y-m-d H:i:s'),
  420 + ]);
  421 + $source_html = str_replace($vjs, getImageUrl($new_vjs), $source_html);
  422 + }
  423 + } else {
  424 + $source_html = str_replace($vjs, getImageUrl($vjs_result['url_complete']), $source_html);
  425 + }
  426 + }
  427 +
  428 + CosService::uploadRemote($project_id, 'source', $vs['url_complete'], $vs['url_complete'], $source_html);
  429 + }
  430 + }
423 } 431 }
424 } 432 }
425 433