作者 lyh
... ... @@ -64,7 +64,7 @@ class SyncProject extends Command
while (true){
$list = NoticeLog::where('type', NoticeLog::TYPE_PROJECT)->where('status', NoticeLog::STATUS_PENDING)->get();
foreach ($list as $item){
echo 'start:' . $item['id'] . PHP_EOL;
echo date('Y-m-d') . ' start:' . $item['id'] . PHP_EOL;
try {
$is_update = $item['data']['is_update']??0;//是否是4.0或5.0更新到6.0
$order_id = $item['data']['order_id'];
... ... @@ -97,7 +97,7 @@ class SyncProject extends Command
$item->save();
echo 'success:' . $item['id'] . PHP_EOL;
}catch (\Exception $e){
echo 'error:' . $item['id'] . $e->getMessage() . PHP_EOL;
echo 'error:' . $item['id'] . $e->getMessage() .' line ' . $e->getLine() . PHP_EOL;
errorLog('项目同步失败', $item, $e);
$this->retry($item, $e->getMessage());
}
... ... @@ -217,7 +217,7 @@ class SyncProject extends Command
DB::beginTransaction();
try {
$id = $this->saveProject($data['project']);
$this->setPostId($data['deploy_build']['plan'],$id);;
$this->setPostId($data['deploy_build']['plan'],$id);
$this->savePayment($data['payment'],$id);
$this->saveDeployBuild($data['deploy_build'],$id);
$this->saveDeployOptimize($data['deploy_optimize'],$id);
... ... @@ -244,7 +244,6 @@ class SyncProject extends Command
foreach ($task_list as $task){
UpdateLog::createLog($id,$task,$param['get_data_url']);
}
}
DB::commit();
... ... @@ -279,9 +278,9 @@ class SyncProject extends Command
* @time :2023/8/30 15:53
*/
public function saveProject($param){
if(isset($param['channel']) && !empty($param['channel'])){
// if(isset($param['channel']) && !empty($param['channel'])){
$param['channel'] = Arr::a2s($param['channel']);
}
// }
$projectModel = new Project();
$info = $projectModel->read(['from_order_id'=>$param['from_order_id'],'delete_status'=>0]);
if($info !== false){
... ...
... ... @@ -85,48 +85,51 @@ class ProjectUpdate extends Command
$data = http_get($url, ['charset' => 'UTF-8']);
if (isset($data['code']) && $data['code'] == 200) {
$items = $data['data'] ?? [];
$model = new Category();
foreach ($items as $item) {
$parent = $model->read(['pid' => 0, 'title' => $item['name']], 'id');
if (!$parent) {
try {
$parent_id = $model->addReturnId([
'project_id' => $project_id,
'title' => $item['name'],
'pid' => 0,
'keywords' => $item['keywords'],
'describe' => $item['description']
]);
$route = RouteMap::setRoute($item['url'] ? $this->get_url_route($item['url']) : $item['name'], RouteMap::SOURCE_PRODUCT_CATE, $parent_id, $project_id);
$model->edit(['route' => $route], ['id' => $parent_id]);
} catch (\Exception $e) {
echo 'date:' . date('Y-m-d H:i:s') . ', task_id: ' . $task->id . ', error: ' . $e->getMessage() . PHP_EOL;
continue;
}
}else{
$parent_id = $parent['id'];
}
foreach ($item['children'] as $child) {
$child_info = $model->read(['pid' => $parent_id, 'title' => $child['name']]);
if (!$child_info) {
try {
$child_id = $model->addReturnId([
'project_id' => $project_id,
'title' => $child['name'],
'pid' => $parent_id,
'keywords' => $child['keywords'],
'describe' => $child['description']
]);
$route = RouteMap::setRoute($child['url'] ? $this->get_url_route($child['url']) : $child['name'], RouteMap::SOURCE_PRODUCT_CATE, $child_id, $project_id);
$model->edit(['route' => $route], ['id' => $child_id]);
} catch (\Exception $e) {
echo 'date:' . date('Y-m-d H:i:s') . ', task_id: ' . $task->id . ', error: ' . $e->getMessage() . PHP_EOL;
continue;
}
}
}
}
$this->category_insert($project_id, $items, 0);
// $model = new Category();
// foreach ($items as $item) {
// $parent = $model->read(['pid' => 0, 'title' => $item['name']], 'id');
// if (!$parent) {
// try {
// $parent_id = $model->addReturnId([
// 'project_id' => $project_id,
// 'title' => $item['name'],
// 'pid' => 0,
// 'keywords' => $item['keywords'],
// 'describe' => $item['description']
// ]);
// $route = RouteMap::setRoute($item['url'] ? $this->get_url_route($item['url']) : $item['name'], RouteMap::SOURCE_PRODUCT_CATE, $parent_id, $project_id);
// $model->edit(['route' => $route], ['id' => $parent_id]);
// } catch (\Exception $e) {
// echo 'date:' . date('Y-m-d H:i:s') . ', task_id: ' . $task->id . ', error: ' . $e->getMessage() . PHP_EOL;
// continue;
// }
// } else {
// $parent_id = $parent['id'];
// }
//
// foreach ($item['children'] as $child) {
// $child_info = $model->read(['pid' => $parent_id, 'title' => $child['name']]);
// if (!$child_info) {
// try {
// $child_id = $model->addReturnId([
// 'project_id' => $project_id,
// 'title' => $child['name'],
// 'pid' => $parent_id,
// 'keywords' => $child['keywords'],
// 'describe' => $child['description']
// ]);
// $route = RouteMap::setRoute($child['url'] ? $this->get_url_route($child['url']) : $child['name'], RouteMap::SOURCE_PRODUCT_CATE, $child_id, $project_id);
// $model->edit(['route' => $route], ['id' => $child_id]);
// } catch (\Exception $e) {
// echo 'date:' . date('Y-m-d H:i:s') . ', task_id: ' . $task->id . ', error: ' . $e->getMessage() . PHP_EOL;
// continue;
// }
// }
// }
// }
} else {
return true;
}
} elseif ($api_type == 'tag') {
//关键词
... ... @@ -144,17 +147,18 @@ class ProjectUpdate extends Command
$model = new Keyword();
foreach ($items as $item) {
$keyword = $model->read(['title' => $item['text_title']], 'id');
if ($item['name'] ?? '') {
$keyword = $model->read(['title' => $item['name']], 'id');
if (!$keyword) {
try {
$id = $model->addReturnId([
'project_id' => $project_id,
'title' => $item['text_title'],
'seo_title' => $item['seo_title'],
'seo_keywords' => $item['seo_keywords'],
'seo_description' => $item['seo_description'],
'title' => $item['name'],
'seo_title' => $item['seo_title'] ?? '',
'seo_keywords' => $item['seo_keywords'] ?? '',
'seo_description' => $item['seo_description'] ?? '',
]);
$route = RouteMap::setRoute($item['url'] ? $this->get_url_route($item['url']) : $item['text_title'], RouteMap::SOURCE_PRODUCT_KEYWORD, $id, $project_id);
$route = RouteMap::setRoute($item['url'] ? $this->get_url_route($item['url']) : $item['name'], RouteMap::SOURCE_PRODUCT_KEYWORD, $id, $project_id);
$model->edit(['route' => $route], ['id' => $id]);
} catch (\Exception $e) {
echo 'date:' . date('Y-m-d H:i:s') . ', task_id: ' . $task->id . ', error: ' . $e->getMessage() . PHP_EOL;
... ... @@ -165,6 +169,9 @@ class ProjectUpdate extends Command
}
}
}
} else {
return true;
}
} elseif ($api_type == 'website_info') {
//网站信息
$url = $api_url . '?' . http_build_query(['w' => 'website_info']);
... ... @@ -175,26 +182,38 @@ class ProjectUpdate extends Command
$model = new WebSettingReceiving();
if ($phones) {
$receiving_phones = $model->read(['type' => 1, 'values' => $phones]);
$phone_arr = explode(',', $phones);
foreach ($phone_arr as $v_phone) {
if ($v_phone) {
$receiving_phones = $model->read(['type' => 2, 'values' => $v_phone]);
if (!$receiving_phones) {
$model->add([
'type' => 1,
'values' => $phones,
'values' => $v_phone,
'project_id' => $project_id
]);
}
}
}
}
if ($emails) {
$receiving_emails = $model->read(['type' => 2, 'values' => $emails]);
$email_arr = explode(',', $emails);
foreach ($email_arr as $v_email) {
if ($v_email) {
$receiving_emails = $model->read(['type' => 1, 'values' => $v_email]);
if (!$receiving_emails) {
$model->add([
'type' => 2,
'values' => $emails,
'values' => $v_email,
'project_id' => $project_id
]);
}
}
}
}
} else {
return true;
}
} elseif ($api_type == 'post') {
//产品
$url = $api_url . '?' . http_build_query(['w' => 'post', 'page' => 1, 'pagesize' => 0]);
... ... @@ -212,17 +231,19 @@ class ProjectUpdate extends Command
$model = new Product();
foreach ($items as $item) {
if ($item['ttile'] ?? '') {
$product = $model->read(['title' => $item['ttile']], 'id');
if (!$product) {
//图片
$gallery = [];
if ($item['images'] ?? []) {
foreach ($item['images'] as $k_img => $img) {
$gallery[] = ['alt' => '这是一张产品图', 'url' => $img];
}
}
//分类
$category_id = '';
if ($item['category']) {
if ($item['category'] ?? []) {
$category_arr = [];
foreach ($item['category'] as $cate) {
if ($cate['parent'] == 0) {
... ... @@ -240,15 +261,15 @@ class ProjectUpdate extends Command
$id = $model->addReturnId([
'project_id' => $project_id,
'title' => $item['ttile'],
'intro' => $item['description'],
'content' => $item['content'],
'intro' => $item['description'] ?? '',
'content' => $item['content'] ?? '',
'category_id' => $category_id,
'thumb' => isset($gallery[0]) ? Arr::a2s($gallery[0]) : '',
'gallery' => Arr::a2s($gallery),
'seo_mate' => Arr::a2s([
'title' => $item['ttile'],
'keyword' => $item['keywords'],
'description' => $item['description']
'keyword' => $item['keywords'] ?? '',
'description' => $item['description'] ?? ''
]),
'status' => Product::STATUS_ON
]);
... ... @@ -263,6 +284,9 @@ class ProjectUpdate extends Command
}
}
}
} else {
return true;
}
} elseif ($api_type == 'news' || $api_type == 'blog') {
//新闻或博客
$url = $api_url . '?' . http_build_query(['w' => $api_type, 'page' => 1, 'pagesize' => 0]);
... ... @@ -285,6 +309,7 @@ class ProjectUpdate extends Command
foreach ($items as $item) {
if ($item['ttile'] ?? '') {
$news = $model->read(['name' => $item['ttile']], 'id');
if (!$news) {
try {
... ... @@ -292,9 +317,9 @@ class ProjectUpdate extends Command
'project_id' => $project_id,
'name' => $item['ttile'],
'seo_title' => $item['ttile'],
'seo_keywords' => $item['keywords'],
'seo_description' => $item['description'],
'text' => $item['content'],
'seo_keywords' => $item['keywords'] ?? '',
'seo_description' => $item['description'] ?? '',
'text' => $item['content'] ?? '',
'image' => $item['images'][0] ?? '',
'status' => $api_type == 'news' ? News::STATUS_ONE : Blog::STATUS_ONE
]);
... ... @@ -310,6 +335,9 @@ class ProjectUpdate extends Command
}
}
} else {
return true;
}
} else {
//单页
$url = $api_url . '?' . http_build_query(['w' => 'page', 'page' => 1, 'pagesize' => 0]);
$data = http_get($url, ['charset' => 'UTF-8']);
... ... @@ -326,7 +354,7 @@ class ProjectUpdate extends Command
$model = new BCustomTemplate();
foreach ($items as $item) {
if ($item['ttile'] ?? '') {
$custom = $model->read(['name' => $item['ttile']], 'id');
if (!$custom) {
try {
... ... @@ -334,9 +362,9 @@ class ProjectUpdate extends Command
'project_id' => $project_id,
'name' => $item['ttile'],
'title' => $item['ttile'],
'keywords' => $item['keywords'],
'description' => $item['description'],
'html' => $item['content'],
'keywords' => $item['keywords'] ?? '',
'description' => $item['description'] ?? '',
'html' => $item['content'] ?? '',
'status' => 1
]);
$route = RouteMap::setRoute($item['url'] ? $this->get_url_route($item['url']) : $item['ttile'], RouteMap::SOURCE_PAGE, $id, $project_id);
... ... @@ -350,6 +378,9 @@ class ProjectUpdate extends Command
}
}
}
} else {
return true;
}
}
}
//关闭数据库
... ... @@ -391,4 +422,31 @@ class ProjectUpdate extends Command
$arr = explode('/', $url);
return $arr[count($arr) - 2];
}
protected function category_insert($project_id, $items, $pid = 0)
{
$model = new Category();
foreach ($items as $item) {
if ($item['name'] ?? '') {
$parent = $model->read(['pid' => $pid, 'title' => $item['name']], 'id');
if (!$parent) {
$parent_id = $model->addReturnId([
'project_id' => $project_id,
'title' => $item['name'],
'pid' => $pid,
'keywords' => $item['keywords'] ?? '',
'describe' => $item['description'] ?? ''
]);
$route = RouteMap::setRoute($item['url'] ? $this->get_url_route($item['url']) : $item['name'], RouteMap::SOURCE_PRODUCT_CATE, $parent_id, $project_id);
$model->edit(['route' => $route], ['id' => $parent_id]);
} else {
$parent_id = $parent['id'];
}
if (!empty($item['children'])) {
$this->category_insert($project_id, $item['children'], $parent_id);
}
}
}
}
}
... ...
... ... @@ -9,11 +9,12 @@ use App\Helper\Translate;
use App\Models\Ai\AiCommand;
use App\Models\Mail\Mail;
use App\Models\Project\DeployOptimize;
use App\Models\Project\ProjectUpdateTdk;
use App\Models\User\User;
use App\Services\ProjectServer;
use Illuminate\Console\Command;
use Illuminate\Support\Facades\Cache;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Redis;
/**
* 初始化项目
... ... @@ -132,47 +133,54 @@ class UpdateSeoTdk extends Command
public function handle()
{
while (true) {
$project_id = Redis::rpop('updateSeoTdk');
if (!$project_id) {
sleep(2);
$task = ProjectUpdateTdk::getPendingTask();
if (!$task) {
sleep(10);
continue;
}
$project_id = $task->project_id;
echo date('Y-m-d H:i:s') . ' start project_id: ' . $project_id . PHP_EOL;
try {
ProjectServer::useProject($project_id);
$this->seo_tdk($project_id);
$this->seo_tdk($project_id, $task->id);
DB::disconnect('custom_mysql');
}catch (\Exception $e){
echo date('Y-m-d H:i:s') . ' error: ' . $project_id . '->' . $e->getMessage() . PHP_EOL;
echo date('Y-m-d H:i:s') . 'line: '. $e->getLine() .' error: ' . $project_id . '->' . $e->getMessage() . PHP_EOL;
ProjectUpdateTdk::retry($task->id, $e->getMessage());
}
echo date('Y-m-d H:i:s') . ' end: ' . $project_id . PHP_EOL;
echo date('Y-m-d H:i:s') . ' end project_id: ' . $project_id . PHP_EOL;
}
}
public function seo_tdk($project_id)
public function seo_tdk($project_id, $task_id)
{
$data = [];
$update = [
'created_at'=>date('Y-m-d H:i:s'),
'updated_at'=>date('Y-m-d H:i:s'),
'project_id'=>$project_id,
];//更新统计
//更新统计
$update = [];
$ai_commands = AiCommand::where('is_batch', 1)->select('key', 'scene', 'ai')->get()->toArray();
$ai_commands = Arr::setValueToKey($ai_commands, 'key');
foreach ($this->maps as $table => $map) {
$update[$table] = ['total_page'=>0, 'title'=>0, 'keyword'=>0, 'des'=>0];
echo date('Y-m-d H:i:s') . '更新--' . $table . ': 项目id' . $project_id . PHP_EOL . json_encode($update[$table]);
echo date('Y-m-d H:i:s') . '更新--' . $table . ': 项目id' . $project_id . PHP_EOL;
$list = DB::connection('custom_mysql')->table($table)->get();
if (!empty($list)) {
$list = $list->toArray();
foreach ($list as $v) {
echo '打印数据:'.date('Y-m-d H:i:s') . $update[$table]['total_page'].PHP_EOL;
$update[$table]['total_page']++;
$v = (array)$v;
echo date('Y-m-d H:i:s') . '更新--' . $table . ':id' . $v['id'] . PHP_EOL;
//缓存 在处理的 项目 数据 id
$cache_key = "seo_tdk_{$project_id}_{$table}_{$v['id']}";
if(Cache::get($cache_key)){
continue;
}
Cache::put($cache_key, 1, 120);
echo date('Y-m-d H:i:s') . '更新--' . $table . ': 项目id' . $project_id . ':id' . $v['id'] . PHP_EOL;
$data = [];
$json_field = '';
echo date('Y-m-d H:i:s') . json_encode($map) . PHP_EOL;
foreach ($map as $ai_key => $field) {
$field_arr = explode('.', $field);
if (count($field_arr) > 1) {
... ... @@ -186,6 +194,7 @@ class UpdateSeoTdk extends Command
echo $field.'已有值 跳过' . PHP_EOL;
continue;
}
//AI生成
if (!empty($ai_commands[$ai_key]['ai'])) {
$prompt = $this->getPrompt($project_id, $ai_commands[$ai_key]['ai'], $table, $v);
... ... @@ -255,17 +264,16 @@ class UpdateSeoTdk extends Command
}
if($json_field){
$old_data = json_decode($v[$field_arr[0]], true);
foreach ($old_data as $kk=>$vv){
foreach ($old_data ?: [] as $kk=>$vv){
empty($data[$json_field][$kk]) && $data[$json_field][$kk] = $vv;
}
$data[$json_field] = json_encode($data[$json_field]);
}
DB::connection('custom_mysql')->table($table)->where(['id' => $v['id']])->update($data);
}
$update[$table] = json_encode($update[$table]);
}
}
DB::table('gl_project_update_tdk')->insert($update);
ProjectUpdateTdk::finish($task_id, $update);
}
public function getPrompt($project_id, $prompt, $table, $data){
... ...
... ... @@ -151,6 +151,9 @@ class Arr extends \Illuminate\Support\Arr
*/
public static function s2a($str)
{
if(!$str){
return [];
}
if (is_array($str)) {
return $str;
}
... ...
... ... @@ -74,7 +74,7 @@ class Common
if (isset($result['texts']['sl']) && isset(Translate::$tls_list[$result['texts']['sl']])) {
$lang = Translate::$tls_list[$result['texts']['sl']]['text'];
} else {
$lang = 'Chinese';
$lang = 'English';
}
}
$str = 'Please answer in '.$lang;
... ...
... ... @@ -14,9 +14,20 @@ use Illuminate\Support\Facades\Log;
*/
class Gpt
{
public $api = 'http://openai.waimaoq.com/';
/**
* 头
* @var string[]
*/
public $header = [
'apikey' => 'UkzZljFv83Z2qBi5YR1o3f2otAVWtug6',
'X-CmerApi-Host' => 'bizopenai.p.cmer.com',
];
public $header = [];
/**
* 请求域名
* @var string
*/
public $api = 'https://api.cmer.com';
private static $instance;
... ... @@ -35,7 +46,7 @@ class Gpt
*/
public function openai_chat_qqs($content, $system_content = '')
{
$url = $this->api . 'v2/openai_chat_qqs';
$url = $this->api . '/v1/openai_chat';
$data = [
'messages' => [],
... ... @@ -57,9 +68,10 @@ class Gpt
->withBody(json_encode($data, JSON_UNESCAPED_UNICODE), 'application/json')
->post($url);
$json = $result->json();
if (!isset($json['text'])) {
if (!isset($json['text']) || $json['code'] !==200) {
Log::error('openai_chat_qqs data:', $data);
Log::error('openai_chat_qqs result:' . (time() - $time), $json === null ? ['null'] : $json);
$json = [];
}
} catch (\Throwable $e) {
Log::error('openai_chat_qqs time ' . (time() - $time) . ' error:' . $e->getMessage());
... ...
... ... @@ -11,6 +11,7 @@ namespace App\Http\Controllers\Aside\Com;
use App\Helper\Common;
use App\Http\Controllers\Bside\BaseController;
use App\Models\Project\ProjectUpdateTdk;
use App\Services\ProjectServer;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Redis;
... ... @@ -37,7 +38,7 @@ class UpdateController extends BaseController
],[
'project_id.required' => 'project_id不能为空',
]);
Redis::lpush('updateSeoTdk', $this->param['project_id']);
ProjectUpdateTdk::add_task($this->param['project_id']);
$this->response('任务添加成功');
}
}
... ...
... ... @@ -9,10 +9,104 @@
namespace App\Models\Project;
use App\Helper\Arr;
use App\Models\Base;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Log;
class ProjectUpdateTdk extends Base
{
//设置关联表名
protected $table = 'gl_project_update_tdk';
const STATUS_PENDING = 0;
const STATUS_SUCCESS = 1;
const STATUS_FAIL = 2;
/**
* 新建任务
* @param $project_id
* @author zbj
* @date 2023/11/9
*/
public static function add_task($project_id){
$model = new self();
$model->project_id = $project_id;
$model->save();
}
/**
* 获取待处理任务
* @return mixed
* @author zbj
* @date 2023/11/9
*/
public static function getPendingTask(){
return self::where('status', self::STATUS_PENDING)->orderBy('id', 'asc')->first();
}
/**
* 重试任务
* @param $id
* @param $remark
* @author zbj
* @date 2023/11/9
*/
public static function retry($id, $remark)
{
DB::beginTransaction();
try {
//行锁 避免脏读写
$data = self::where('id', $id)->lockForUpdate()->first();
$data->retry = $data->retry + 1;
if ($data->retry > 3) {
$data->status = self::STATUS_FAIL;
}else{
$data->status = self::STATUS_PENDING;
}
$data->remark = mb_substr($remark, 0, 250);
$data->save();
DB::commit();
} catch (\Exception $e) {
DB::rollback();
Log::error('project_update_tdk retry error:' . $e->getMessage());
}
}
/**
* 完成
* @param $id
* @param $update_data
* @author zbj
* @date 2023/11/9
*/
public static function finish($id, $update_data){
DB::beginTransaction();
try {
//行锁 避免脏读写
$data = self::where('id', $id)->lockForUpdate()->first();
$data->status = self::STATUS_SUCCESS;
foreach($update_data as $field => $item){
$old_date = Arr::s2a($data->$field);
$new_data = [];
foreach($item as $k=>$v){
if($k == 'total_page'){
$new_data[$k] = $v;
}else{
$new_data[$k] = $v + ($old_date[$k] ?? 0);
}
}
$data->$field = $new_data;
}
$data->save();
DB::commit();
} catch (\Exception $e) {
DB::rollback();
Log::error('project_update_tdk finish error:' . $e->getMessage());
}
}
}
... ...