<?php

namespace App\Console\Commands\Update;

use App\Helper\Arr;
use App\Http\Logic\Bside\News\NewsLogic;
use App\Http\Logic\Bside\Product\CategoryLogic;
use App\Models\Blog\Blog;
use App\Models\Collect\CollectSource;
use App\Models\Collect\CollectTask;
use App\Models\Com\UpdateLog;
use App\Models\Com\UpdateOldInfo;
use App\Models\CustomModule\CustomModule;
use App\Models\CustomModule\CustomModuleCategory;
use App\Models\CustomModule\CustomModuleContent;
use App\Models\CustomModule\CustomModuleExtend;
use App\Models\CustomModule\CustomModuleExtentContent;
use App\Models\File\ErrorFile;
use App\Models\News\News;
use App\Models\News\NewsCategory;
use App\Models\Product\Category;
use App\Models\Product\CategoryRelated;
use App\Models\Product\Extend;
use App\Models\Product\ExtendInfo;
use App\Models\Product\Keyword;
use App\Models\Product\Product;
use App\Models\RouteMap\RouteMap;
use App\Models\Template\BCustomTemplate;
use App\Models\WebSetting\WebSettingReceiving;
use App\Services\CosService;
use App\Services\ProjectServer;
use Illuminate\Console\Command;
use Illuminate\Support\Facades\Cache;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Redis;

/**
 * 4.0,5.0升级到6.0，内容同步
 * Class ProjectImport
 * @package App\Console\Commands
 * @author Akun
 * @date 2023/10/9 15:04
 */
class ProjectUpdate extends Command
{
    /**
     * The name and signature of the console command.
     *
     * @var string
     */
    protected $signature = 'project_update';

    /**
     * The console command description.
     *
     * @var string
     */
    protected $description = '执行项目升级任务';


    public function handle()
    {
        ini_set('memory_limit', '512M');
        while (true) {
            $this->start_update();
        }
    }

    protected function start_update()
    {
        $task_id = $this->get_task();
        if ($task_id === false) {
            //所有任务执行完成
            sleep(60);
            return true;
        } elseif ($task_id === 0) {
            //队列任务已处理完，有进程正在查询数据库，等待5秒后执行
            sleep(5);
            return true;
        }

        $task = UpdateLog::where('id', $task_id)->where('status', UpdateLog::STATUS_UN)->first();
        if (!$task) {
            sleep(2);
            return true;
        }

        $project_id = $task->project_id;
        $api_type = $task->api_type;
        $api_url_arr = explode('?', $task->api_url);
        $api_url = $api_url_arr[0];
        $domain_arr = parse_url($api_url);

        $page_size = 20;

        echo 'date:' . date('Y-m-d H:i:s') . ', task_id: ' . $task->id . ', task_type: ' . $api_type . ', update start' . PHP_EOL;

        $task->status = UpdateLog::STATUS_ING;//同步中
        $task->save();

        //获取站点原始域名信息
        $old_info = UpdateOldInfo::getOldDomain($project_id, $domain_arr['host']);
        $link_type = $old_info['link_type'];
        $web_url_domain = $old_info['web_url_domain'];
        $home_url = $old_info['home_url'];

        //获取所有语种
        $language_list = [];
        $url_language = 'https://' . $domain_arr['host'] . '/wp-content/plugins/proofreading/json/user_language.json';
        $data_language = curl_c($url_language);
        if (is_array($data_language)) {
            $language_list = array_column($data_language, 'short');
        }

        //获取所有页面
        $page_list = [];
        $url_page = 'https://' . $domain_arr['host'] . '/wp-content/cache/pages_list.json';
        $data_page = curl_c($url_page);
        if (is_array($data_page)) {
            $page_list = array_column($data_page, 'path');
        }

        //设置数据库
        $project = ProjectServer::useProject($project_id);
        if ($project) {
            if ($api_type == 'category') {
                //产品分类
                $url = $api_url . '?' . http_build_query(['w' => 'category']);
                $data = curl_c($url);
                if (isset($data['code']) && $data['code'] == 200) {
                    $items = $data['data'] ?? [];
                    $this->category_insert($project_id, $items, 0, $domain_arr['host'], $web_url_domain, $home_url);
                } else {
                    return true;
                }
            } elseif ($api_type == 'category_news') {
                // 新闻分类
                $url = $api_url . '?' . http_build_query(['w' => 'category_news']);
                $data = curl_c($url);
                if (isset($data['code']) && $data['code'] == 200) {
                    $items = $data['data'] ?? [];
                    $this->category_news_insert($project_id, $items, 0);
                } else {
                    return true;
                }
            } elseif ($api_type == 'tag') {
                //关键词
                $url = $api_url . '?' . http_build_query(['w' => 'tag', 'page' => 1, 'pagesize' => 0]);
                $data = curl_c($url);
                if (isset($data['code']) && $data['code'] == 200) {
                    $count = $data['data']['count'] ?? 0;

                    $total_page = ceil($count / $page_size);
                    for ($page = 1; $page <= $total_page; $page++) {
                        $url_page = $api_url . '?' . http_build_query(['w' => 'tag', 'page' => $page, 'pagesize' => $page_size]);
                        $data_page = curl_c($url_page);
                        if (isset($data_page['code']) && $data_page['code'] == 200) {
                            $items = $data_page['data']['data'] ?? [];

                            $model = new Keyword();
                            foreach ($items as $item) {
                                $route = $this->get_url_route($item['url'] ?? '');
                                if ($route) {
                                    //判断路由是否存在
                                    $route_info = RouteMap::where('project_id', $project_id)->where('route', $route)->where('path', '')->first();
                                    if ($route_info) {
                                        continue;
                                    }
                                    $keyword = $model->read(['route' => $route], 'id');
                                    if (!$keyword) {
                                        try {
                                            $item['name'] = $this->special2str($item['name'] ?? '');
                                            $id = $model->addReturnId([
                                                'project_id' => $project_id,
                                                'title' => $item['name'],
                                                'seo_title' => $item['seo_title'] ?? '',
                                                'seo_keywords' => $item['seo_keywords'] ?? '',
                                                'seo_description' => $item['seo_description'] ?? '',
                                                'is_upgrade' => 1,
                                                'route' => $route
                                            ]);
                                            $this->set_map($route, RouteMap::SOURCE_PRODUCT_KEYWORD, $id, $project_id);
                                        } catch (\Exception $e) {
                                            echo 'date:' . date('Y-m-d H:i:s') . ', task_id: ' . $task->id . ', error: ' . $e->getMessage() . PHP_EOL;
                                            continue;
                                        }
                                    }
                                }
                            }
                        }
                    }
                } else {
                    return true;
                }
            } elseif ($api_type == 'website_info') {
                //网站信息
                $url = $api_url . '?' . http_build_query(['w' => 'website_info']);
                $data = curl_c($url);
                if (isset($data['code']) && $data['code'] == 200) {
                    $phones = $data['data']['phones'] ?? '';
                    $emails = $data['data']['emails'] ?? '';
                    $custom_types = $data['data']['customposttypes'] ?? '';

                    $model = new WebSettingReceiving();
                    $has_phone = $model->read(['type' => 2]);
                    if (!$has_phone && $phones) {
                        $phone_arr = explode(',', $phones);
                        foreach ($phone_arr as $v_phone) {
                            if ($v_phone) {
                                $receiving_phones = $model->read(['type' => 2, 'values' => $v_phone]);
                                if (!$receiving_phones) {
                                    try {
                                        $model->add([
                                            'type' => 2,
                                            'values' => $v_phone,
                                            'project_id' => $project_id
                                        ]);
                                    } catch (\Exception $e) {
                                        echo 'date:' . date('Y-m-d H:i:s') . ', task_id: ' . $task->id . ', error: ' . $e->getMessage() . PHP_EOL;
                                        continue;
                                    }
                                }
                            }
                        }
                    }
                    $has_email = $model->read(['type' => 1]);
                    if (!$has_email && $emails) {
                        $email_arr = explode(',', $emails);
                        foreach ($email_arr as $v_email) {
                            if ($v_email) {
                                $receiving_emails = $model->read(['type' => 1, 'values' => $v_email]);
                                if (!$receiving_emails) {
                                    try {
                                        $model->add([
                                            'type' => 1,
                                            'values' => $v_email,
                                            'project_id' => $project_id
                                        ]);
                                    } catch (\Exception $e) {
                                        echo 'date:' . date('Y-m-d H:i:s') . ', task_id: ' . $task->id . ', error: ' . $e->getMessage() . PHP_EOL;
                                        continue;
                                    }
                                }
                            }
                        }
                    }
                    if ($custom_types) {
                        $custom_model = new CustomModule();
                        foreach ($custom_types as $v_custom) {
                            $log = UpdateLog::where('project_id', $project_id)->where('api_type', $v_custom)->first();
                            if (empty($log)) {
                                try {
                                    $custom_info = $custom_model->read(['route' => $v_custom]);
                                    if (!$custom_info) {
                                        $custom_model->add([
                                            'name' => $v_custom,
                                            'project_id' => $project_id,
                                            'route' => $v_custom
                                        ]);
                                    }
                                    UpdateLog::createLog($project_id, $v_custom, $task->api_url);
                                } catch (\Exception $e) {
                                    echo 'date:' . date('Y-m-d H:i:s') . ', task_id: ' . $task->id . ', error: ' . $e->getMessage() . PHP_EOL;
                                    continue;
                                }
                            }
                        }
                    }
                } else {
                    return true;
                }
            } elseif ($api_type == 'post') {
                //产品
                $url = $api_url . '?' . http_build_query(['w' => 'post', 'page' => 1, 'pagesize' => 0]);
                $data = curl_c($url);
                if (isset($data['code']) && $data['code'] == 200) {
                    $count = $data['data']['count'] ?? 0;

                    $total_page = ceil($count / $page_size);
                    for ($page = 1; $page <= $total_page; $page++) {
                        $url_page = $api_url . '?' . http_build_query(['w' => 'post', 'page' => $page, 'pagesize' => $page_size]);
                        $data_page = curl_c($url_page);
                        if (isset($data_page['code']) && $data_page['code'] == 200) {
                            $items = $data_page['data']['data'] ?? [];

                            $model = new Product();
                            $category_model = new Category();
                            $keyword_model = new Keyword();
                            $extend_model = new Extend();
                            $extend_info_model = new ExtendInfo();
                            $logic = new CategoryLogic();

                            foreach ($items as $item) {
                                $route = $this->get_url_route($item['url'] ?? '');
                                if ($route) {
                                    //图片
                                    $gallery = [];
                                    if ($item['images'] ?? []) {
                                        foreach ($item['images'] as $k_img => $img) {
                                            $gallery[] = ['alt' => '', 'url' => $this->source_download($img, $project_id, $domain_arr['host'], $web_url_domain, $home_url)];
                                        }
                                    }
                                    //分类
                                    $category_id = '';
                                    $category_arr = [];
                                    if ($item['category'] ?? []) {
                                        $category_arr = $category_model->list(['original_id' => ['in', array_column($item['category'], 'id')]]);
                                        $category_id = $logic->getLastCategory(array_column($category_arr, 'id'));
                                    }
                                    //关键词
                                    $keyword_id = '';
                                    if ($item['tags'] ?? []) {
                                        $keyword_arr = $keyword_model->list(['title' => ['in', array_column($item['tags'], 'name')]]);
                                        if ($keyword_arr) {
                                            $keyword_id = ',' . implode(',', array_column($keyword_arr, 'id')) . ',';
                                        }
                                    }
                                    //产品参数
                                    $attrs = [];
                                    if ($item['attr'] ?? []) {
                                        foreach ($item['attr'] as $k_attr => $attr) {
                                            $attrs[] = ['key' => $k_attr, 'value' => $attr];
                                        }
                                    }
                                    //下载文件
                                    $files = '';
                                    if ($item['file'] ?? '') {
                                        $files = json_encode(['url' => $this->source_download($item['file'], $project_id, $domain_arr['host'], $web_url_domain, $home_url, 1), 'text' => 'Download']);
                                    }
                                    //名称去掉特殊符号
                                    $item['ttile'] = $this->special2str($item['ttile'] ?? '');

                                    try {
                                        $product = $model->read(['route' => $route], ['id', 'six_read']);
                                        if (!$product) {
                                            $id = $model->insertGetId([
                                                'project_id' => $project_id,
                                                'title' => $item['ttile'],
                                                'intro' => $item['short_description'] ?? '',
                                                'content' => $item['content'] ?? '',
                                                'category_id' => $category_id,
                                                'keyword_id' => $keyword_id,
                                                'thumb' => isset($gallery[0]) ? Arr::a2s($gallery[0]) : '',
                                                'gallery' => Arr::a2s($gallery),
                                                'attrs' => Arr::a2s($attrs),
                                                'seo_mate' => Arr::a2s([
                                                    'title' => $item['ttile'],
                                                    'keyword' => $item['keywords'] ?? '',
                                                    'description' => $item['description'] ?? ''
                                                ]),
                                                'status' => Product::STATUS_ON,
                                                'created_at' => $item['post_date'] ?? date('Y-m-d H:i:s'),
                                                'updated_at' => $item['post_date'] ?? date('Y-m-d H:i:s'),
                                                'send_time' => $item['post_date'] ?? date('Y-m-d H:i:s'),
                                                'sort' => $item['sort'] ?? 0,
                                                'files' => $files,
                                                'is_upgrade' => 1,
                                                'six_read' => 1,
                                                'route' => $route
                                            ]);
                                            $this->set_map($route, RouteMap::SOURCE_PRODUCT, $id, $project_id);
                                            CollectTask::_insert($item['url'], $project_id, RouteMap::SOURCE_PRODUCT, $id, $domain_arr['host'], $link_type, $language_list, $page_list);
                                            $six_read = 1;
                                        } else {
                                            $id = $product['id'];
                                            $six_read = $product['six_read'];
                                            if ($six_read) {
                                                //按5.0展示才需要更新数据
                                                $model->edit([
                                                    'title' => $item['ttile'],
                                                    'intro' => $item['short_description'] ?? '',
                                                    'content' => $item['content'] ?? '',
                                                    'category_id' => $category_id,
                                                    'keyword_id' => $keyword_id,
                                                    'thumb' => isset($gallery[0]) ? Arr::a2s($gallery[0]) : '',
                                                    'gallery' => Arr::a2s($gallery),
                                                    'attrs' => Arr::a2s($attrs),
                                                    'seo_mate' => Arr::a2s([
                                                        'title' => $item['ttile'],
                                                        'keyword' => $item['keywords'] ?? '',
                                                        'description' => $item['description'] ?? ''
                                                    ]),
                                                    'send_time' => $item['post_date'] ?? date('Y-m-d H:i:s'),
                                                    'sort' => $item['sort'] ?? 0,
                                                    'files' => $files,
                                                ], ['id' => $id]);
                                            }
                                        }

                                        if ($six_read) {
                                            //关联分类
                                            CategoryRelated::saveRelated($id, array_column($category_arr, 'id'));
                                            //扩展字段
                                            if ($item['extend'] ?? []) {
                                                foreach (array_reverse($item['extend']) as $ke => $ve) {
                                                    if ($ve == '--') {
                                                        $ve = '';
                                                    }
                                                    $extend = $extend_model->read(['title' => $ke]);

                                                    if ($extend) {
                                                        if ($extend['type'] == 3) {
                                                            //图片
                                                            $gallery = [];
                                                            if (is_array($ve)) {
                                                                foreach ($ve as $ve_img) {
                                                                    $gallery[] = ['title' => '', 'description' => '', 'url' => $this->source_download($ve_img, $project_id, $domain_arr['host'], $web_url_domain, $home_url, 1)];
                                                                }
                                                            } else {
                                                                $gallery[] = ['title' => '', 'description' => '', 'url' => $this->source_download($ve, $project_id, $domain_arr['host'], $web_url_domain, $home_url, 1)];
                                                            }
                                                            $value = Arr::a2s($gallery);
                                                        } elseif ($extend['type'] == 4) {
                                                            //文件
                                                            $file = [];
                                                            if (is_array($ve)) {
                                                                $file_title = $ve['title'] ?? '';
                                                                $file_url = $ve['url'] ?? '';
                                                                if ($file_url) {
                                                                    $file[] = [
                                                                        'name' => $file_title,
                                                                        'url' => $this->source_download($file_url, $project_id, $domain_arr['host'], $web_url_domain, $home_url, 1)
                                                                    ];
                                                                }
                                                            } elseif ($ve) {
                                                                $ve_array = explode('/', $ve);
                                                                $file[] = [
                                                                    'name' => end($ve_array),
                                                                    'url' => $this->source_download($ve, $project_id, $domain_arr['host'], $web_url_domain, $home_url, 1)
                                                                ];
                                                            }
                                                            $value = Arr::a2s($file);
                                                        } else {
                                                            //文本
                                                            $value = $ve;
                                                        }

                                                        $extend_info = $extend_info_model->read(['key' => $extend['key'], 'product_id' => $id]);
                                                        if (!$extend_info) {
                                                            $extend_info_model->add([
                                                                'key' => $extend['key'],
                                                                'type' => $extend['type'],
                                                                'project_id' => $project_id,
                                                                'product_id' => $id,
                                                                'values' => $value,
                                                            ]);
                                                        } else {
                                                            $extend_info_model->edit(['type' => $extend['type'], 'values' => $value], ['key' => $extend['key'], 'product_id' => $id]);
                                                        }
                                                    }
                                                }
                                            }
                                        }
                                    } catch (\Exception $e) {
                                        echo 'date:' . date('Y-m-d H:i:s') . ', task_id: ' . $task->id . ', error: ' . $e->getMessage() . PHP_EOL;
                                        continue;
                                    }
                                }
                            }
                        }
                    }
                } else {
                    return true;
                }
            } elseif ($api_type == 'news' || $api_type == 'blog') {
                //新闻或博客
                $url = $api_url . '?' . http_build_query(['w' => $api_type, 'page' => 1, 'pagesize' => 0]);
                $data = curl_c($url);
                if (isset($data['code']) && $data['code'] == 200) {
                    $count = $data['data']['count'] ?? 0;

                    $total_page = ceil($count / $page_size);
                    for ($page = 1; $page <= $total_page; $page++) {
                        $url_page = $api_url . '?' . http_build_query(['w' => $api_type, 'page' => $page, 'pagesize' => $page_size]);
                        $data_page = curl_c($url_page);
                        if (isset($data_page['code']) && $data_page['code'] == 200) {
                            $items = $data_page['data']['data'] ?? [];

                            if ($api_type == 'news') {
                                $model = new News();
                                $category_model = new NewsCategory();
                                $logic = new NewsLogic();
                            } else {
                                $model = new Blog();
                            }

                            foreach ($items as $item) {
                                $route = $this->get_url_route($item['url'] ?? '');
                                if ($route) {
                                    //分类
                                    $category_id = '';
                                    if ($api_type == 'news' && ($item['category'] ?? [])) {
                                        $category_arr = $category_model->list(['original_id' => ['in', array_column($item['category'], 'id')]]);
                                        $category_id = $logic->getCategory(array_column($category_arr, 'id'));
                                    }
                                    //图片
                                    if (is_array($item['images'] ?? '')) {
                                        $image = $item['images'][0] ?? '';
                                    } else {
                                        $image = $item['images'] ?? '';
                                    }
                                    $new_img = $this->source_download($image, $project_id, $domain_arr['host'], $web_url_domain, $home_url);
                                    //名称去掉特殊符号
                                    $item['ttile'] = $this->special2str($item['ttile'] ?? '');

                                    try {
                                        $news = $model->read(['url' => $route], ['id', 'six_read']);
                                        if (!$news) {
                                            $id = $model->insertGetId([
                                                'project_id' => $project_id,
                                                'name' => $item['ttile'],
                                                'category_id' => $category_id,
                                                'seo_title' => $item['ttile'],
                                                'seo_keywords' => $item['keywords'] ?? '',
                                                'seo_description' => $item['description'] ?? '',
                                                'text' => $item['content'] ?? '',
                                                'image' => $new_img,
                                                'status' => $api_type == 'news' ? News::STATUS_ONE : Blog::STATUS_ONE,
                                                'created_at' => $item['post_date'] ?? date('Y-m-d H:i:s'),
                                                'updated_at' => $item['post_date'] ?? date('Y-m-d H:i:s'),
                                                'release_at' => $item['post_date'] ?? date('Y-m-d H:i:s'),
                                                'sort' => $item['sort'] ?? 0,
                                                'is_upgrade' => 1,
                                                'six_read' => 1,
                                                'url' => $route
                                            ]);
                                            $this->set_map($route, $api_type == 'news' ? RouteMap::SOURCE_NEWS : RouteMap::SOURCE_BLOG, $id, $project_id);

                                            CollectTask::_insert($item['url'], $project_id, $api_type == 'news' ? RouteMap::SOURCE_NEWS : RouteMap::SOURCE_BLOG, $id, $domain_arr['host'], $link_type, $language_list, $page_list);
                                        } else {
                                            if ($news['six_read']) {
                                                //按5.0展示才需要更新数据
                                                $model->edit([
                                                    'name' => $item['ttile'],
                                                    'category_id' => $category_id,
                                                    'seo_title' => $item['ttile'],
                                                    'seo_keywords' => $item['keywords'] ?? '',
                                                    'seo_description' => $item['description'] ?? '',
                                                    'text' => $item['content'] ?? '',
                                                    'image' => $new_img,
                                                    'release_at' => $item['post_date'] ?? date('Y-m-d H:i:s'),
                                                    'sort' => $item['sort'] ?? 0,
                                                ], ['id' => $news['id']]);
                                            }
                                        }
                                    } catch (\Exception $e) {
                                        echo 'date:' . date('Y-m-d H:i:s') . ', task_id: ' . $task->id . ', error: ' . $e->getMessage() . PHP_EOL;
                                        continue;
                                    }
                                }
                            }
                        }
                    }
                } else {
                    return true;
                }
            } elseif ($api_type == 'page') {
                //单页
                $url = $api_url . '?' . http_build_query(['w' => 'page', 'page' => 1, 'pagesize' => 0]);
                $data = curl_c($url);
                if (isset($data['code']) && $data['code'] == 200) {
                    $count = $data['data']['count'] ?? 0;

                    $total_page = ceil($count / $page_size);
                    for ($page = 1; $page <= $total_page; $page++) {
                        $url_page = $api_url . '?' . http_build_query(['w' => 'page', 'page' => $page, 'pagesize' => $page_size]);
                        $data_page = curl_c($url_page);
                        if (isset($data_page['code']) && $data_page['code'] == 200) {
                            $items = $data_page['data']['data'] ?? [];

                            $model = new BCustomTemplate();

                            foreach ($items as $item) {
                                $route = $this->get_url_route($item['url'] ?? '');
                                if ($route) {
                                    //名称去掉特殊符号
                                    $item['ttile'] = $this->special2str($item['ttile'] ?? '');
                                    try {
                                        $custom = $model->read(['url' => $route], ['id', 'six_read']);
                                        if (!$custom) {
                                            $id = $model->insertGetId([
                                                'project_id' => $project_id,
                                                'name' => $item['ttile'],
                                                'title' => $item['ttile'],
                                                'keywords' => $item['keywords'] ?? '',
                                                'description' => $item['description'] ?? '',
                                                'html' => $item['content'] ?? '',
                                                'status' => 1,
                                                'created_at' => $item['post_date'] ?? date('Y-m-d H:i:s'),
                                                'updated_at' => $item['post_date'] ?? date('Y-m-d H:i:s'),
                                                'is_upgrade' => 1,
                                                'six_read' => 1,
                                                'url' => $route
                                            ]);
                                            $this->set_map($route, RouteMap::SOURCE_PAGE, $id, $project_id);

                                            CollectTask::_insert($item['url'], $project_id, RouteMap::SOURCE_PAGE, $id, $domain_arr['host'], $link_type, $language_list, $page_list);
                                        } else {
                                            $id = $custom['id'];
                                            $six_read = $custom['six_read'];
                                            if ($six_read) {
                                                //按5.0展示才需要更新数据
                                                $model->edit([
                                                    'name' => $item['ttile'],
                                                    'title' => $item['ttile'],
                                                    'keywords' => $item['keywords'] ?? '',
                                                    'description' => $item['description'] ?? '',
                                                    'html' => $item['content'] ?? '',
                                                ], ['id' => $id]);
                                            }
                                        }
                                    } catch (\Exception $e) {
                                        echo 'date:' . date('Y-m-d H:i:s') . ', task_id: ' . $task->id . ', error: ' . $e->getMessage() . PHP_EOL;
                                        continue;
                                    }
                                }
                            }
                        }
                    }
                } else {
                    return true;
                }
            } else {
                //自定义模块
                $custom_model = new CustomModule();
                $custom_info = $custom_model->read(['route' => $api_type]);
                if ($custom_info) {
                    $url = $api_url . '?' . http_build_query(['w' => $api_type, 'page' => 1, 'pagesize' => 0]);
                    $data = curl_c($url);
                    if (isset($data['code']) && $data['code'] == 200) {
                        $category = $data['data']['category'];
                        //判断分类里面是否存在自定义模块路由
                        $is_exist = 0;
                        if ($data['data']['category'] ?? []) {
                            foreach ($data['data']['category'] as $vc) {
                                $c_route = $this->get_url_route($vc['url'] ?? '');
                                if ($c_route == $custom_info['route']) {
                                    $is_exist = 1;
                                }
                            }
                        }
                        if (!$is_exist) {
                            array_push($category, [
                                'id' => 0,
                                'name' => $custom_info['route'],
                                'url' => '/' . $custom_info['route'],
                                'parent' => 0
                            ]);
                        }
                        $this->category_custom_insert($project_id, $custom_info['id'], $category);

                        $count = $data['data']['count'] ?? 0;

                        $total_page = ceil($count / $page_size);
                        for ($page = 1; $page <= $total_page; $page++) {
                            $url_page = $api_url . '?' . http_build_query(['w' => $api_type, 'page' => $page, 'pagesize' => $page_size]);
                            $data_page = curl_c($url_page);
                            if (isset($data_page['code']) && $data_page['code'] == 200) {
                                $items = $data_page['data']['data'] ?? [];

                                $model = new CustomModuleContent();
                                $category_model = new CustomModuleCategory();
                                $extend_model = new CustomModuleExtend();
                                $extend_info_model = new CustomModuleExtentContent();

                                foreach ($items as $item) {
                                    $route = $this->get_url_route($item['url'] ?? '');
                                    if ($route) {
                                        //分类
                                        $category_id = '';
                                        if ($item['category'] ?? []) {
                                            $category_arr = $category_model->list(['original_id' => ['in', array_column($item['category'], 'id')]]);
                                            $category_id = implode(',', array_column($category_arr, 'id'));
                                        } else {
                                            $category_custom = $category_model->read(['route' => $custom_info['route']], 'id');
                                            if ($category_custom) {
                                                $category_id = $category_custom['id'];
                                            }
                                        }
                                        if ($category_id) {
                                            $category_id = ',' . $category_id . ',';
                                        }
                                        //图片
                                        if (is_array($item['images'] ?? '')) {
                                            $image = $item['images'][0] ?? '';
                                        } else {
                                            $image = $item['images'] ?? '';
                                        }
                                        $new_img = $this->source_download($image, $project_id, $domain_arr['host'], $web_url_domain, $home_url);
                                        //名称去掉特殊符号
                                        $item['title'] = $this->special2str($item['title'] ?? '');
                                        //排序
                                        $sort = 0;
                                        if (isset($item['listorder'])) {
                                            $sort = $item['listorder'];
                                        }
                                        if (isset($item['sort'])) {
                                            $sort = $item['sort'];
                                        }

                                        try {
                                            $custom_content = $model->read(['route' => $route], ['id', 'six_read']);
                                            if (!$custom_content) {
                                                $id = $model->insertGetId([
                                                    'project_id' => $project_id,
                                                    'name' => $item['title'],
                                                    'module_id' => $custom_info['id'],
                                                    'category_id' => $category_id,
                                                    'content' => $item['content'] ?? '',
                                                    'sort' => $sort,
                                                    'image' => $new_img,
                                                    'created_at' => $item['post_time'] ?? date('Y-m-d H:i:s'),
                                                    'updated_at' => $item['post_time'] ?? date('Y-m-d H:i:s'),
                                                    'release_at' => $item['post_time'] ?? date('Y-m-d H:i:s'),
                                                    'is_upgrade' => 1,
                                                    'six_read' => 1,
                                                    'route' => $route
                                                ]);
                                                $this->set_map($route, RouteMap::SOURCE_MODULE, $id, $project_id);

                                                CollectTask::_insert($item['url'], $project_id, RouteMap::SOURCE_MODULE, $id, $domain_arr['host'], $link_type, $language_list, $page_list);
                                                $six_read = 1;
                                            } else {
                                                $id = $custom_content['id'];
                                                $six_read = $custom_content['six_read'];
                                                if ($six_read) {
                                                    $model->edit([
                                                        'name' => $item['title'],
                                                        'category_id' => $category_id,
                                                        'content' => $item['content'] ?? '',
                                                        'sort' => $sort,
                                                        'release_at' => $item['post_time'] ?? date('Y-m-d H:i:s'),
                                                        'image' => $new_img
                                                    ], ['id' => $id]);
                                                }
                                            }

                                            //扩展字段
                                            if ($six_read) {
                                                if ($item['extend'] ?? []) {
                                                    foreach ($item['extend'] as $ke => $ve) {
                                                        if ($ve == '--') {
                                                            $ve = '';
                                                        }
                                                        $extend = $extend_model->read(['title' => $ke]);
                                                        if ($extend) {
                                                            if ($extend['type'] == 3) {
                                                                //图片
                                                                $gallery = [];
                                                                if (is_array($ve)) {
                                                                    foreach ($ve as $ve_img) {
                                                                        $gallery[] = ['title' => '', 'description' => '', 'url' => $this->source_download($ve_img, $project_id, $domain_arr['host'], $web_url_domain, $home_url, 1)];
                                                                    }
                                                                } else {
                                                                    $gallery[] = ['title' => '', 'description' => '', 'url' => $this->source_download($ve, $project_id, $domain_arr['host'], $web_url_domain, $home_url, 1)];
                                                                }
                                                                $value = Arr::a2s($gallery);
                                                            } elseif ($extend['type'] == 4) {
                                                                //文件
                                                                $file = [];
                                                                if (is_array($ve)) {
                                                                    $file_title = $ve['title'] ?? '';
                                                                    $file_url = $ve['url'] ?? '';
                                                                    if ($file_url) {
                                                                        $file[] = [
                                                                            'name' => $file_title,
                                                                            'url' => $this->source_download($file_url, $project_id, $domain_arr['host'], $web_url_domain, $home_url, 1)
                                                                        ];
                                                                    }
                                                                } elseif ($ve) {
                                                                    $ve_array = explode('/', $ve);
                                                                    $file[] = [
                                                                        'name' => end($ve_array),
                                                                        'url' => $this->source_download($ve, $project_id, $domain_arr['host'], $web_url_domain, $home_url, 1)
                                                                    ];
                                                                }
                                                                $value = Arr::a2s($file);
                                                            } else {
                                                                //文本
                                                                $value = $ve;
                                                            }

                                                            $extend_info = $extend_info_model->read(['key' => $extend['key'], 'content_id' => $id]);
                                                            if (!$extend_info) {
                                                                $extend_info_model->add([
                                                                    'key' => $extend['key'],
                                                                    'type' => $extend['type'],
                                                                    'project_id' => $project_id,
                                                                    'content_id' => $id,
                                                                    'module_id' => $custom_info['id'],
                                                                    'values' => $value,
                                                                ]);
                                                            } else {
                                                                $extend_info_model->edit(['type' => $extend['type'], 'values' => $value], ['key' => $extend['key'], 'content_id' => $id]);
                                                            }
                                                        }
                                                    }
                                                }
                                            }
                                        } catch (\Exception $e) {
                                            echo 'date:' . date('Y-m-d H:i:s') . ', task_id: ' . $task->id . ', error: ' . $e->getMessage() . PHP_EOL;
                                            continue;
                                        }
                                    }
                                }
                            }
                        }
                    } else {
                        return true;
                    }
                }
            }
        }
        //关闭数据库
        DB::disconnect('custom_mysql');

        $task->status = UpdateLog::STATUS_COM;//同步完成
        $task->save();

        echo 'date:' . date('Y-m-d H:i:s') . ', task_id: ' . $task->id . ', task_type: ' . $api_type . ', update end ' . PHP_EOL;

        sleep(2);
    }

    //获取任务
    protected function get_task()
    {
        $key = 'console_update_task';
        $task_id = Redis::rpop($key);
        if ($task_id) {
            return $task_id;
        }

        if (!Cache::add('console_update_task_select_mysql', true, 5)) {
            //如果存在数据锁，表示有其他进程已往下执行，当前进程直接返回
            return 0;
        }

        $task_list = UpdateLog::where('status', UpdateLog::STATUS_UN)->orderBy('sort', 'asc')->orderBy('updated_at', 'asc')->limit(20)->get();
        if ($task_list->count() == 0) {
            return false;
        }

        foreach ($task_list as $value) {
            Redis::lpush($key, $value->id);
        }

        $task_id = Redis::rpop($key);
        return $task_id;
    }

    //获取地址路由
    protected function get_url_route($url)
    {
        if (strpos($url, '%') !== false) {
            $url = urldecode($url);
        }
        $arr = parse_url($url);
        if (empty($arr['path'])) {
            return '';
        }
        $path = $arr['path'];

        $path_arr = explode('/', $path);

        $route = end($path_arr) ? end($path_arr) : $path_arr[count($path_arr) - 2];

        if (strpos($route, ',') !== false || strpos($route, '，') !== false) {
            return '';
        }

        return $route;
    }

    //产品多级分类入库
    protected function category_insert($project_id, $items, $pid, $domain, $web_url_domain, $home_url)
    {
        $model = new Category();
        foreach ($items as $item) {
            $route = $this->get_url_route($item['url'] ?? '');
            if ($route) {
                $item['name'] = $this->special2str($item['name'] ?? '');
                //图片
                if (is_array($item['images'] ?? '')) {
                    $image = $item['images'][0] ?? '';
                } else {
                    $image = $item['images'] ?? '';
                }
                $new_img = $this->source_download($image, $project_id, $domain, $web_url_domain, $home_url);
                //描述
                $describe = '';
                if (isset($item['description']) && $item['description']) {
                    $describe = $item['description'];
                    //匹配描述资源
                    $source_list = $this->html_preg($describe, $project_id, $domain, $web_url_domain, $home_url);
                    if ($source_list) {
                        foreach ($source_list as $vs) {
                            if ($vs['download']) {
                                //需要下载资源
                                $down_url = $this->source_download($vs['url_complete'], $project_id, $domain, $web_url_domain, $home_url);
                            } else {
                                //已经下载过资源
                                $down_url = getImageUrl($vs['url_complete']);
                            }
                            $describe = str_replace($vs['url'], $down_url, $describe);
                        }
                    }
                }
                try {
                    $parent = $model->read(['route' => $route], 'id');
                    if (!$parent) {
                        $parent_id = $model->addReturnId([
                            'project_id' => $project_id,
                            'title' => $item['name'],
                            'image' => $new_img,
                            'pid' => $pid,
                            'keywords' => $item['keywords'] ?? '',
                            'describe' => $describe,
                            'original_id' => $item['id'],
                            'route' => $route
                        ]);
                        $this->set_map($route, RouteMap::SOURCE_PRODUCT_CATE, $parent_id, $project_id);
                    } else {
                        $parent_id = $parent['id'];
                        $model->edit([
                            'title' => $item['name'],
                            'image' => $new_img,
                            'pid' => $pid,
                            'keywords' => $item['keywords'] ?? '',
                            'describe' => $describe,
                            'original_id' => $item['id']
                        ], ['id' => $parent_id]);
                    }
                } catch (\Exception $e) {
                    echo 'date:' . date('Y-m-d H:i:s') . ', category_insert error: ' . $e->getMessage() . PHP_EOL;
                    continue;
                }

                if (!empty($item['children'] ?? [])) {
                    $this->category_insert($project_id, $item['children'], $parent_id, $domain, $web_url_domain, $home_url);
                }
            }
        }
    }

    //新闻多级分类入库
    protected function category_news_insert($project_id, $items, $pid = 0)
    {
        $model = new NewsCategory();
        foreach ($items as $item) {
            $route = $this->get_url_route($item['url'] ?? '');
            if ($route) {
                $item['name'] = $this->special2str($item['name'] ?? '');
                try {
                    $parent = $model->read(['alias' => $route], 'id');
                    if (!$parent) {
                        $parent_id = $model->addReturnId([
                            'project_id' => $project_id,
                            'name' => $item['name'],
                            'pid' => $pid,
                            'original_id' => $item['id'],
                            'alias' => $route
                        ]);
                        $this->set_map($route, RouteMap::SOURCE_NEWS_CATE, $parent_id, $project_id);

                    } else {
                        $parent_id = $parent['id'];
                        $model->edit([
                            'name' => $item['name'],
                            'pid' => $pid,
                            'original_id' => $item['id'],
                        ], ['id' => $parent_id]);
                    }
                } catch (\Exception $e) {
                    echo 'date:' . date('Y-m-d H:i:s') . ', category_news_insert error: ' . $e->getMessage() . PHP_EOL;
                    continue;
                }

                if (!empty($item['children'] ?? [])) {
                    $this->category_news_insert($project_id, $item['children'], $parent_id);
                }
            }
        }
    }

    //扩展模块多级分类入库
    protected function category_custom_insert($project_id, $module_id, $items)
    {
        $model = new CustomModuleCategory();
        foreach ($items as $item) {
            $route = $this->get_url_route($item['url'] ?? '');
            if ($route) {
                $item['name'] = $this->special2str($item['name'] ?? '');
                try {
                    $cate = $model->read(['route' => $route], 'id');
                    if (!$cate) {
                        $cate_id = $model->addReturnId([
                            'project_id' => $project_id,
                            'module_id' => $module_id,
                            'name' => $item['name'],
                            'original_id' => $item['id'],
                            'route' => $route
                        ]);
                        $this->set_map($route, RouteMap::SOURCE_MODULE_CATE, $cate_id, $project_id);
                    } else {
                        $model->edit([
                            'name' => $item['name'],
                            'original_id' => $item['id'],
                        ], ['id' => $cate['id']]);
                    }
                } catch (\Exception $e) {
                    echo 'date:' . date('Y-m-d H:i:s') . ', category_custom_insert error: ' . $e->getMessage() . PHP_EOL;
                    continue;
                }
            }
        }
    }

    //特殊字符转换
    protected function special2str($str)
    {
        if (strpos($str, ';') === false) {
            return $str;
        }

        $list = [
            '&lt;' => '<',
            '&gt;' => '>',
            '&amp;' => '&',
            '&acute;' => '´',
            '&quot;' => '“',
            '&nbsp;' => ' '
        ];

        foreach ($list as $k => $v) {
            $str = str_replace($k, $v, $str);
        }

        return $str;
    }

    //路由入库
    protected function set_map($route, $source, $source_id, $project_id)
    {
        if ($route) {
            $route_map = RouteMap::where('project_id', $project_id)->where('source', $source)->where('source_id', $source_id)->first();
            if (!$route_map) {
                $route_map = new RouteMap();
                $route_map->project_id = $project_id;
                $route_map->source = $source;
                $route_map->source_id = $source_id;
                $route_map->route = $route;

                if ($source == RouteMap::SOURCE_NEWS) {
                    $route_map->path = RouteMap::SOURCE_NEWS;
                } elseif ($source == RouteMap::SOURCE_BLOG) {
                    $route_map->path = RouteMap::SOURCE_BLOG . 's';
                }

                $route_map->save();
            }
        }
    }

    //资源下载
    protected function source_download($url, $project_id, $domain, $web_url_domain, $home_url, $same_name = 0)
    {
        if (!$url) {
            return '';
        }

        $arr = parse_url($url);
        $scheme = $arr['scheme'] ?? '';
        $host = $arr['host'] ?? '';
        $path = $arr['path'] ?? '';

        if (
            (empty($scheme) || $scheme == 'https' || $scheme == 'http')
            && (empty($host) || (strpos($web_url_domain, $host) !== false) || (strpos($home_url, $host) !== false))
            && $path
        ) {
            $source = CollectSource::where('project_id', $project_id)->where('origin', $url)->first();
            if (!$source) {
                if (empty($scheme) && empty($host) && substr($path, 0, 1) != '/') {
                    $path = '/' . $path;
                }
                $url_complete = ($scheme ?: 'https') . '://' . $domain . $path;
                $new_url = CosService::uploadRemote($project_id, 'image_product', $url_complete, '', '', $same_name);

                if ($new_url) {
                    CollectSource::insert([
                        'project_id' => $project_id,
                        'origin' => $url,
                        'target' => $new_url,
                        'created_at' => date('Y-m-d H:i:s'),
                        'updated_at' => date('Y-m-d H:i:s'),
                    ]);

                    //同步到177
                    $error_file = ErrorFile::where('path', $new_url)->first();
                    if (!$error_file) {
                        $error_file = new ErrorFile();
                        $error_file->path = $new_url;
                        $error_file->status = 0;
                        $error_file->save();
                    }

                    return getImageUrl($new_url);
                } else {
                    return $url_complete;
                }
            } else {
                return getImageUrl($source['target']);
            }
        } else {
            return ($scheme ?: 'https') . '://' . ($host ?: $domain) . $path;
        }
    }

    //获取自定义字段key
    protected function get_extend_key($model, $i = 1)
    {
        $key = 'pd_extended_field_';
        $info = $model->read(['key' => $key . $i]);
        if ($info !== false) {
            return $this->get_extend_key($model, $i + 1);
        } else {
            return $key . $i;
        }
    }

    //正则匹配html资源
    protected function html_preg($html, $project_id, $domain, $web_url_domain, $home_url)
    {
        $source = [];

        if (!$html) {
            return $source;
        }

        //image
        preg_match_all('/<img\s+[^>]*?src\s*=\s*(\'|\")(.*?)\\1[^>]*?\/?\s*>/i', $html, $result_img);
        $img = $result_img[2] ?? [];
        foreach ($img as $vi) {
            $check_vi = $this->url_check($vi, $project_id, $domain, $web_url_domain, $home_url);
            if ($check_vi && (!in_array($check_vi, $source))) {
                $source[] = $check_vi;
            }
        }

        //video
        preg_match_all('/<source\s+[^>]*?src\s*=\s*(\'|\")(.*?)\\1[^>]*?\/?\s*>/i', $html, $result_video);
        $video = $result_video[2] ?? [];
        foreach ($video as $vv) {
            $check_vv = $this->url_check($vv, $project_id, $domain, $web_url_domain, $home_url);
            if ($check_vv && (!in_array($check_vv, $source))) {
                $source[] = $check_vv;
            }
        }
        preg_match_all('/<video\s+[^>]*?src\s*=\s*(\'|\")(.*?)\\1[^>]*?\/?\s*>/i', $html, $result_video_2);
        $video_2 = $result_video_2[2] ?? [];
        foreach ($video_2 as $vv2) {
            $check_vv2 = $this->url_check($vv2, $project_id, $domain, $web_url_domain, $home_url);
            if ($check_vv2 && (!in_array($check_vv2, $source))) {
                $source[] = $check_vv2;
            }
        }

        //a标签下载资源
        preg_match_all('/<a\s+[^>]*?href\s*=\s*(\'|\")(.*?)\\1[^>]*?\/?\s*>/i', $html, $result_a);
        $down = $result_a[2] ?? [];
        foreach ($down as $vd) {
            $check_vd = $this->url_check($vd, $project_id, $domain, $web_url_domain, $home_url);
            if ($check_vd && (!in_array($check_vd, $source))) {
                $source[] = $check_vd;
            }
        }

        return $source;
    }

    //判断资源是否需要下载
    protected function url_check($url, $project_id, $domain, $web_url_domain, $home_url)
    {
        $url = trim($url);
        if ($url) {
            $url = str_replace('&quot;', '', $url);
            $arr = parse_url($url);
            $scheme = $arr['scheme'] ?? '';
            $host = $arr['host'] ?? '';
            $path = $arr['path'] ?? '';
            $query = $arr['query'] ?? '';

            $path_arr = explode('.', $path);
            $path_end = end($path_arr);
            if (
                (empty($scheme) || $scheme == 'https' || $scheme == 'http')
                && (empty($host) || (strpos($web_url_domain, $host) !== false) || (strpos($home_url, $host) !== false))
                && $path
                && (substr($path, 0, 1) == '/')
                && (strpos($path, '.') !== false)
                && (strpos($path_end, 'html') === false)
                && (strpos($path_end, 'php') === false)
                && (strpos($path_end, 'com') === false)
                && (strpos($path_end, 'xml') === false)
            ) {
                $source = CollectSource::where('project_id', $project_id)->where('origin', $url)->first();
                if (!$source) {
                    return [
                        'download' => true,
                        'url' => $url,
                        'url_complete' => ($scheme ?: 'https') . '://' . $domain . $path . ($query ? '?' . $query : '')
                    ];
                } else {
                    return [
                        'download' => false,
                        'url' => $url,
                        'url_complete' => $source['target']
                    ];
                }
            } else {
                return false;
            }
        } else {
            return false;
        }
    }
}