作者 邓超

es 批量添加

... ... @@ -108,8 +108,9 @@ class SyncToEsCmd {
$data = $this->getDataByEs($id,$is_check_body);
if($data){
list($doc_id,$data) = $data;
$bulkData = new \Lib\Es\BulkData();
// 主库
$this->toDataEs('email_lists_copy',$id,$doc_id,$data);
$bulkData->add('email_lists_copy',$doc_id,$data);
// 个人邮箱的情况
if(!$data['postid']){
$postids = $this->getPostids($data['email_id']);
... ... @@ -118,17 +119,19 @@ class SyncToEsCmd {
$data['postid'] = $postid;
$data['source'] = 1;
// 分库 个人邮箱
$this->toDataEs('email_lists_branch_'.$data['postid'],$id,$doc_id,$data);
$bulkData ->add('email_lists_branch_'.$postid,$doc_id,$data);
}
}else{
// 分库 其他 非fob数据源
$this->toDataEs('email_lists_branch_0',$id,$doc_id,$data);
$bulkData->add('email_lists_branch_0',$doc_id,$data);
}
}else{
// 分库
$this->toDataEs('email_lists_branch_'.$data['postid'],$id,$doc_id,$data);
$bulkData->add('email_lists_branch_'.$data['postid'],$doc_id,$data);
}
$this->toDataEs($id,$bulkData);
}
}else{
// 没有数据时暂停1秒
... ... @@ -287,39 +290,36 @@ class SyncToEsCmd {
/**
* 同步数据到es
* @param string $index 索引名称
* @param int $data_id list表的id
* @param string $doc_id es文档id
* @param array $data es数据
* @param int $data_id
* @param \Lib\Es\BulkData $bulkData
* @throws \Elastic\Elasticsearch\Exception\ClientResponseException
* @throws \Elastic\Elasticsearch\Exception\ServerResponseException
* @author:dc
* @time 2025/8/5 10:09
* @time 2025/8/6 20:55
*/
public function toDataEs($index,$data_id,$doc_id,$data){
// echo $index;
// echo ' ==== ';
public function toDataEs(int $data_id, \Lib\Es\BulkData $bulkData){
foreach ($bulkData->getIndexs() as $index){
$this->es->setIndex($index);
// 检查数据库是否存在
if(empty($this->checkEsIndex[$index]) && $index != 'email_lists_copy'){
if(!redis()->has('esmapcheck:'.$index)){
$m = $this->setEsMap($index);
if($m !== 9) _echo("{$index} 创建索引 ".$m);
if(!$m){
$this->log($data_id,$index);
return;
}
redis()->set('esmapcheck:'.$index,1,86400);
}
}
// 下次不在检查
$this->checkEsIndex[$index] = 1;
$code = $this->es->save($doc_id,$data);
if($code!==200){
$this->log($data_id,$index);
@file_put_contents(LOG_PATH.'/sync_es_fail.log',$index.":".$data_id."\n",FILE_APPEND);
_echo($index.': '.$data_id.'===>'.$code);
}
// 批量提交数据的
$ret = $this->es->bulk($bulkData);
if(!empty($ret['errors'])){
$this->log($data_id);
@file_put_contents(LOG_PATH.'/sync_es_fail.log',$data_id."\n",FILE_APPEND);
_echo($data_id.' ===> 0');
}
}
/**
... ...
... ... @@ -38,6 +38,16 @@ class BulkData {
/**
* 获取所有索引名称
* @return array
* @author:dc
* @time 2025/8/6 16:45
*/
public function getIndexs(): array {
return array_keys($this->data);
}
/**
* 转换成es可识别的数据
* @return array
* @author:dc
... ...