<?php

namespace App\Console\Commands\Update;

use App\Helper\Arr;
use App\Models\Blog\Blog;
use App\Models\Collect\CollectSource;
use App\Models\Collect\CollectTask;
use App\Models\Com\UpdateLog;
use App\Models\Com\UpdateOldInfo;
use App\Models\CustomModule\CustomModule;
use App\Models\CustomModule\CustomModuleContent;
use App\Models\News\News;
use App\Models\Product\Product;
use App\Models\RouteMap\RouteMap;
use App\Models\Template\BCustomTemplate;
use App\Services\CosService;
use App\Services\ProjectServer;
use Illuminate\Console\Command;
use Illuminate\Support\Facades\Cache;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Redis;

/**
 * 4.0,5.0升级到6.0，主站页面采集
 * Class ProjectImport
 * @package App\Console\Commands
 * @author Akun
 * @date 2023/11/10 16:04
 */
class HtmlCollect extends Command
{
    /**
     * The name and signature of the console command.
     *
     * @var string
     */
    protected $signature = 'project_html_collect';

    /**
     * The console command description.
     *
     * @var string
     */
    protected $description = '执行项目html页面采集';


    public function handle()
    {
        ini_set('memory_limit', '1024M');
        while (true) {
            $this->start_collect();
        }
    }

    protected function start_collect()
    {
        $task_id = $this->get_task();
        if ($task_id === false) {
            //所有项目采集完成
            sleep(60);
            return true;
        } elseif ($task_id === 0) {
            //队列任务已处理完，有进程正在查询数据库，等待2秒后执行
            sleep(2);
            return true;
        }

        $task_arr = explode('_', $task_id);
        $project_id = $task_arr[0];
        $collect_id = $task_arr[1];

        //设置数据库
        $project = ProjectServer::useProject($project_id);
        if ($project) {
            $collect_info = CollectTask::select(['id', 'domain', 'route', 'source', 'source_id'])->where('id', $collect_id)->where('status', CollectTask::STATUS_UN)->where('language', '')->first();

            if (!$collect_info) {
                sleep(2);
                return true;
            }

            echo 'date:' . date('Y-m-d H:i:s') . ', project_id: ' . $project_id . ', collect_id: ' . $collect_id . ', collect start' . PHP_EOL;

            $collect_info->status = CollectTask::STATUS_ING;
            $collect_info->save();

            //获取站点原始域名信息
            $old_info = UpdateOldInfo::getOldDomain($project_id, $collect_info->domain);

            //采集html页面，下载资源到本地并替换
            try {
                $html = curl_c('https://' . $collect_info->domain . $collect_info->route, false);
                if (strlen($html) < 4) {

                    if ($html == 404) {
                        //原数据页面404，需要将6.0数据存入草稿箱
                        switch ($collect_info->source) {
                            //产品
                            case RouteMap::SOURCE_PRODUCT:
                                $model = new Product();
                                $status_draft = Product::STATUS_DRAFT;
                                break;
                            //博客
                            case RouteMap::SOURCE_BLOG:
                                $model = new Blog();
                                $status_draft = Blog::STATUS_DRAFT;
                                break;
                            //新闻
                            case RouteMap::SOURCE_NEWS:
                                $model = new News();
                                $status_draft = News::STATUS_DRAFT;
                                break;
                            //自定义模块详情
                            case RouteMap::SOURCE_MODULE:
                                $model = new CustomModule();
                                $status_draft = CustomModule::STATUS_DRAFT;
                                break;
                            default:
                                //单页详情
                                $model = new BCustomTemplate();
                                $status_draft = BCustomTemplate::STATUS_DRAFT;
                        }
                        $model->edit(['status' => $status_draft], ['project_id' => $project_id, 'id' => $collect_info->source_id, 'six_read' => 1]);

                        CollectTask::where('source', $collect_info->source)->where('source_id', $collect_info->source_id)->delete();
                    }

                    echo 'date:' . date('Y-m-d H:i:s') . ', project_id: ' . $project_id . ', collect_id: ' . $collect_id . ', error: ' . $html . PHP_EOL;
                    sleep(2);
                    return true;
                }

                //如果有base64图片，先替换掉，再进行资源匹配
                $new_html = $html;
                preg_match_all("/data:([^;]*);base64,(.*)?\"/", $new_html, $result_img);
                $img_base64 = $result_img[2] ?? [];
                foreach ($img_base64 as $v64) {
                    $new_html = str_replace($v64, '', $new_html);
                }

                //提取页面tdk
                $this->get_site_meta($new_html, $collect_info);

                $source_list = $this->html_preg($new_html, $project_id, $collect_info->domain, $old_info['web_url_domain'], $old_info['home_url']);

                if ($source_list) {
                    $html = $this->upload_source($html, $source_list, $project_id, $collect_info->domain, $old_info['web_url_domain'], $old_info['home_url']);
                }
            } catch (\Exception $e) {
                $collect_info->status = CollectTask::STATUS_FAIL;
                $collect_info->save();

                echo 'date:' . date('Y-m-d H:i:s') . ', project_id: ' . $project_id . ', collect_id: ' . $collect_id . ', error: ' . $e->getMessage() . PHP_EOL;
                sleep(2);
                return true;
            }

            $collect_info->html = $html;
            $collect_info->status = CollectTask::STATUS_COM;
            $collect_info->save();

            echo 'date:' . date('Y-m-d H:i:s') . ', project_id: ' . $project_id . ', collect_id: ' . $collect_id . ', collect end' . PHP_EOL;
        }
        //关闭数据库
        DB::disconnect('custom_mysql');

//        sleep(2);
        return true;
    }

    //获取任务
    protected function get_task()
    {
        $key = 'console_html_collect_task';
        $task_id = Redis::rpop($key);
        if ($task_id) {
            return $task_id;
        }

        $update_log_list = UpdateLog::where('status', UpdateLog::STATUS_COM)->where('collect_status', UpdateLog::COLLECT_STATUS_UN)->orderBy('updated_at', 'asc')->limit(20)->get();
        if ($update_log_list->count() == 0) {
            return false;
        }

        if (!Cache::add('html_collect_select_mysql', true, 10)) {
            //如果存在数据锁，表示有其他进程已往下执行，当前进程直接返回
            return 0;
        }

        foreach ($update_log_list as $update_log) {
            switch ($update_log->api_type) {
                case 'page':
                    $source = RouteMap::SOURCE_PAGE;
                    break;
                case 'news':
                    $source = RouteMap::SOURCE_NEWS;
                    break;
                case 'blog':
                    $source = RouteMap::SOURCE_BLOG;
                    break;
                case 'post':
                    $source = RouteMap::SOURCE_PRODUCT;
                    break;
                default:
                    $source = RouteMap::SOURCE_MODULE;
                    break;
            }

            $complete = false;
            //设置数据库
            $project = ProjectServer::useProject($update_log->project_id);
            if ($project) {
                $collect_list = CollectTask::select(['id', 'project_id'])->where('project_id', $update_log['project_id'])->where('source', $source)->where('language', '')->where('status', CollectTask::STATUS_UN)->orderBy('id', 'asc')->limit(100)->get();

                if ($collect_list->count() == 0) {
                    $complete = true;
                } else {
                    foreach ($collect_list as $collect) {
                        Redis::lpush($key, $collect['project_id'] . '_' . $collect['id']);
                    }
                }
            }
            //关闭数据库
            DB::disconnect('custom_mysql');

            if ($complete) {
                $update_log->collect_status = UpdateLog::COLLECT_STATUS_MAIN;
                $update_log->save();
            }
        }

        $task_id = Redis::rpop($key);
        return $task_id;
    }

    //获取META信息
    private function get_site_meta($html, $collect_info)
    {
        $meta = [];
        if (!empty($html)) {
            #Title
            preg_match_all('/<title>([\w\W]*?)<\/title>/', $html, $matches);
            if (!empty($matches[1])) {
                $meta['title'] = substr($matches[1][0], 0, 255);
            }

            #Keywords
            preg_match_all('/<meta\s+[^>]*?name=[\'|\"]keywords[\'|\"]\s+[^>]*?content=[\'|\"]([\w\W]*?)[\'|\"]/', $html, $matches);
            if (!empty($matches[1])) {
                $meta['keyword'] = substr($matches[1][0], 0, 255);
            }

            #Description
            preg_match_all('/<meta\s+[^>]*?name=[\'|\"]description[\'|\"]\s+[^>]*?content=[\'|\"]([\w\W]*?)[\'|\"]/', $html, $matches);
            if (!empty($matches[1])) {
                $meta['description'] = substr($matches[1][0], 0, 255);
            }
        }

        if (!empty($meta)) {
            $id = $collect_info->source_id;

            switch ($collect_info->source) {
                case RouteMap::SOURCE_PRODUCT:
                    $seo_mate = [
                        'title' => $meta['title'] ?? '',
                        'keyword' => $meta['keyword'] ?? '',
                        'description' => $meta['description'] ?? ''
                    ];
                    Product::where('id', $id)->update(['seo_mate' => Arr::a2s($seo_mate)]);
                    break;
                case RouteMap::SOURCE_NEWS:
                    News::where('id', $id)->update([
                        'seo_title' => $meta['title'] ?? '',
                        'seo_keywords' => $meta['keyword'] ?? '',
                        'seo_description' => $meta['description'] ?? ''
                    ]);
                    break;
                case RouteMap::SOURCE_BLOG:
                    Blog::where('id', $id)->update([
                        'seo_title' => $meta['title'] ?? '',
                        'seo_keywords' => $meta['keyword'] ?? '',
                        'seo_description' => $meta['description'] ?? ''
                    ]);
                    break;
                case RouteMap::SOURCE_PAGE:
                    BCustomTemplate::where('id', $id)->update([
                        'title' => $meta['title'] ?? '',
                        'keywords' => $meta['keyword'] ?? '',
                        'description' => $meta['description'] ?? ''
                    ]);
                    break;
                default:
                    CustomModuleContent::where('id', $id)->update([
                        'seo_title' => $meta['title'] ?? '',
                        'seo_keywords' => $meta['keyword'] ?? '',
                        'seo_description' => $meta['description'] ?? ''
                    ]);
            }
        }
    }

    //正则匹配html资源
    protected function html_preg($html, $project_id, $domain, $web_url_domain, $home_url)
    {
        $source = [];

        if (!$html) {
            return $source;
        }

        //image
        preg_match_all('/<img\s+[^>]*?src\s*=\s*(\'|\")(.*?)\\1[^>]*?\/?\s*>/i', $html, $result_img);
        $img = $result_img[2] ?? [];
        foreach ($img as $vi) {
            $check_vi = $this->url_check($vi, $project_id, $domain, $web_url_domain, $home_url);
            if ($check_vi && (!in_array($check_vi, $source))) {
                $source[] = $check_vi;
            }
        }

        //js
        preg_match_all('/<script\s+[^>]*?src\s*=\s*(\'|\")(.*?)\\1[^>]*?\/?\s*>/i', $html, $result_js);
        $js = $result_js[2] ?? [];
        foreach ($js as $vj) {
            $check_vj = $this->url_check($vj, $project_id, $domain, $web_url_domain, $home_url);
            if ($check_vj && (!in_array($check_vj, $source))) {
                $source[] = $check_vj;
            }
        }

        //video
        preg_match_all('/<source\s+[^>]*?src\s*=\s*(\'|\")(.*?)\\1[^>]*?\/?\s*>/i', $html, $result_video);
        $video = $result_video[2] ?? [];
        foreach ($video as $vv) {
            $check_vv = $this->url_check($vv, $project_id, $domain, $web_url_domain, $home_url);
            if ($check_vv && (!in_array($check_vv, $source))) {
                $source[] = $check_vv;
            }
        }
        preg_match_all('/<video\s+[^>]*?src\s*=\s*(\'|\")(.*?)\\1[^>]*?\/?\s*>/i', $html, $result_video_2);
        $video_2 = $result_video_2[2] ?? [];
        foreach ($video_2 as $vv2) {
            $check_vv2 = $this->url_check($vv2, $project_id, $domain, $web_url_domain, $home_url);
            if ($check_vv2 && (!in_array($check_vv2, $source))) {
                $source[] = $check_vv2;
            }
        }
        preg_match_all('/<iframe\s+[^>]*?src\s*=\s*(\'|\")(.*?)\\1[^>]*?\/?\s*>/i', $html, $result_video_3);
        $video_3 = $result_video_3[2] ?? [];
        foreach ($video_3 as $vv3) {
            $check_vv3 = $this->url_check($vv3, $project_id, $domain, $web_url_domain, $home_url);
            if ($check_vv3 && (!in_array($check_vv3, $source))) {
                $source[] = $check_vv3;
            }
        }

        //css
        preg_match_all('/<link\s+[^>]*?href\s*=\s*(\'|\")(.*?)\\1[^>]*?\/?\s*>/i', $html, $result_css);
        $css = $result_css[2] ?? [];
        foreach ($css as $vc) {
            $check_vc = $this->url_check($vc, $project_id, $domain, $web_url_domain, $home_url);
            if ($check_vc && (!in_array($check_vc, $source))) {
                $source[] = $check_vc;
            }
        }

        //css background
        preg_match_all("/url\(['\"]?(\s*[^>]+?)['\"]?\)/i", $html, $result_css_b);
        $css_b = $result_css_b[1] ?? [];
        foreach ($css_b as $vc_b) {
            $check_vc_b = $this->url_check($vc_b, $project_id, $domain, $web_url_domain, $home_url);
            if ($check_vc_b && (!in_array($check_vc_b, $source))) {
                $source[] = $check_vc_b;
            }
        }

        //a标签下载资源
        preg_match_all('/<a\s+[^>]*?href\s*=\s*(\'|\")(.*?)\\1[^>]*?\/?\s*>/i', $html, $result_a);
        $down = $result_a[2] ?? [];
        foreach ($down as $vd) {
            $check_vd = $this->url_check($vd, $project_id, $domain, $web_url_domain, $home_url);
            if ($check_vd && (!in_array($check_vd, $source))) {
                $source[] = $check_vd;
            }
        }

        return $source;
    }

    //判断资源是否需要下载
    protected function url_check($url, $project_id, $domain, $web_url_domain, $home_url)
    {
        $url = trim($url);
        if ($url) {
            $url = str_replace('&quot;', '', $url);
            $arr = parse_url($url);
            $scheme = $arr['scheme'] ?? '';
            $host = $arr['host'] ?? '';
            $path = $arr['path'] ?? '';
            $query = $arr['query'] ?? '';

            $path_arr = explode('.', $path);
            $path_end = end($path_arr);
            if (
                (empty($scheme) || $scheme == 'https' || $scheme == 'http')
                && (empty($host) || (strpos($web_url_domain, $host) !== false) || (strpos($home_url, $host) !== false))
                && $path
                && (substr($path, 0, 1) == '/')
                && (strpos($path, '.') !== false)
                && (strpos($path_end, 'html') === false)
                && (strpos($path_end, 'php') === false)
                && (strpos($path_end, 'com') === false)
                && (strpos($path_end, 'xml') === false)
            ) {
                $source = CollectSource::where('project_id', $project_id)->where('origin', $url)->first();
                if (!$source) {
                    return [
                        'download' => true,
                        'url' => $url,
                        'url_complete' => ($scheme ?: 'https') . '://' . $domain . $path . ($query ? '?' . $query : '')
                    ];
                } else {
                    return [
                        'download' => false,
                        'url' => $url,
                        'url_complete' => $source['target']
                    ];
                }
            } else {
                return false;
            }
        } else {
            return false;
        }
    }

    //下载并替换资源
    protected function upload_source($html, $source, $project_id, $domain, $web_url_domain, $home_url)
    {
        foreach ($source as $vs) {

            if ($vs['download']) {
                if ($project_id == 1355 && in_array(substr($vs['url_complete'], -3), ['pdf', 'zip', 'rar', '.gz'])) {
                    //项目id为1355的项目，可下载类资源要保持原名称，做特殊处理
                    $new_source = CosService::uploadRemote($project_id, 'source', $vs['url_complete'], '', '', 1);
                } else {
                    $new_source = CosService::uploadRemote($project_id, 'source', $vs['url_complete']);
                }
                if ($new_source) {
                    CollectSource::insert([
                        'project_id' => $project_id,
                        'origin' => $vs['url'],
                        'target' => $new_source,
                        'created_at' => date('Y-m-d H:i:s'),
                        'updated_at' => date('Y-m-d H:i:s'),
                    ]);
                    $html = str_replace($vs['url'], getImageUrl($new_source), $html);

                    if (substr($new_source, -3, 3) == 'css' || substr($new_source, -2, 2) == 'js') {

                        $source_html = curl_c(getImageUrl($new_source), false);

                        if (substr($new_source, -3, 3) == 'css') {
                            preg_match_all("/url\(['\"]?(\s*[^>]+?)['\"]?\)/i", $source_html, $result_source);
                        } else {
                            preg_match_all("/[large|thumb]+URL:['\"]+(\s*[^>]+?)['\"]+,/i", $source_html, $result_source);
                        }

                        $js_css_source = $result_source[1] ?? [];
                        if ($js_css_source) {
                            foreach ($js_css_source as $vjs) {
                                if (strpos($vjs, 'URL:"') !== false) {
                                    $vjs = substr($vjs, strpos($vjs, 'URL:"') + 5);
                                }

                                $vjs_down = str_replace('&quot;', '', $vjs);
                                if (strpos($vjs_down, 'data:') !== false) {
                                    //过滤二进制文件
                                    continue;
                                }
                                if (strlen($vjs_down) > 255) {
                                    //过滤太长文件
                                    continue;
                                }

                                $vjs_down_arr = parse_url($vjs_down);
                                $vjs_down_host = $vjs_down_arr['host'] ?? '';

                                $cos = config('filesystems.disks.cos');
                                $cosCdn = $cos['cdn'];

                                if ($vjs_down_host && $vjs_down_host == $cosCdn) {
                                    //过滤已经下载的
                                    continue;
                                }

                                if (empty($vjs_down_host) && substr($vjs_down, 0, 1) != '/') {
                                    //相对路径
                                    $url_arr = explode('/', $vs['url']);
                                    $url_arr[count($url_arr) - 1] = $vjs_down;
                                    $vjs_down = implode('/', $url_arr);
                                }

                                $vjs_result = $this->url_check($vjs_down, $project_id, $domain, $web_url_domain, $home_url);
                                if (!$vjs_result) {
                                    continue;
                                }

                                if ($vjs_result['download']) {
                                    $new_vjs = CosService::uploadRemote($project_id, 'source', $vjs_result['url_complete']);
                                    if ($new_vjs) {
                                        CollectSource::insert([
                                            'project_id' => $project_id,
                                            'origin' => $vjs_result['url'],
                                            'target' => $new_vjs,
                                            'created_at' => date('Y-m-d H:i:s'),
                                            'updated_at' => date('Y-m-d H:i:s'),
                                        ]);
                                        $source_html = str_replace($vjs, getImageUrl($new_vjs), $source_html);
                                    }
                                } else {
                                    $source_html = str_replace($vjs, getImageUrl($vjs_result['url_complete']), $source_html);
                                }
                            }

                            CosService::uploadRemote($project_id, 'source', $new_source, $new_source, $source_html);
                        }
                    }
                }
            } else {
                $html = str_replace($vs['url'], getImageUrl($vs['url_complete']), $html);
            }
        }

        return $html;
    }
}