正在显示
1 个修改的文件
包含
29 行增加
和
17 行删除
| @@ -60,7 +60,7 @@ class ProjectVisit extends Command | @@ -60,7 +60,7 @@ class ProjectVisit extends Command | ||
| 60 | $api_type = $task->api_type; | 60 | $api_type = $task->api_type; |
| 61 | $api_url = $task->api_url; | 61 | $api_url = $task->api_url; |
| 62 | 62 | ||
| 63 | - $page_size = 200; | 63 | + $page_size = 1000; |
| 64 | 64 | ||
| 65 | echo 'date:' . date('Y-m-d H:i:s') . ', task_id: ' . $task->id . ', task_type: ' . $api_type . ', update start' . PHP_EOL; | 65 | echo 'date:' . date('Y-m-d H:i:s') . ', task_id: ' . $task->id . ', task_type: ' . $api_type . ', update start' . PHP_EOL; |
| 66 | 66 | ||
| @@ -76,6 +76,7 @@ class ProjectVisit extends Command | @@ -76,6 +76,7 @@ class ProjectVisit extends Command | ||
| 76 | $data = curl_c($url); | 76 | $data = curl_c($url); |
| 77 | if (isset($data['count']) && $data['count'] > 0) { | 77 | if (isset($data['count']) && $data['count'] > 0) { |
| 78 | $count = $data['count']; | 78 | $count = $data['count']; |
| 79 | + $max_id = $data['data'][0]['id'] ?? 0; | ||
| 79 | 80 | ||
| 80 | $total_page = ceil($count / $page_size); | 81 | $total_page = ceil($count / $page_size); |
| 81 | for ($page = 1; $page <= $total_page; $page++) { | 82 | for ($page = 1; $page <= $total_page; $page++) { |
| @@ -84,13 +85,12 @@ class ProjectVisit extends Command | @@ -84,13 +85,12 @@ class ProjectVisit extends Command | ||
| 84 | if (isset($data_page['data']) && $data_page['data']) { | 85 | if (isset($data_page['data']) && $data_page['data']) { |
| 85 | $items = $data_page['data']; | 86 | $items = $data_page['data']; |
| 86 | $model = new Visit(); | 87 | $model = new Visit(); |
| 88 | + $insert = []; | ||
| 87 | foreach ($items as $item) { | 89 | foreach ($items as $item) { |
| 88 | if (isset($item['id']) && $item['id']) { | 90 | if (isset($item['id']) && $item['id']) { |
| 89 | - $visit = $model->read(['original_id' => $item['id']], 'id'); | ||
| 90 | - if (!$visit) { | ||
| 91 | - try { | 91 | + if ($item['id'] > $task->max_id) { |
| 92 | $url_arr = parse_url($item['request'] ?? ''); | 92 | $url_arr = parse_url($item['request'] ?? ''); |
| 93 | - $model->insert([ | 93 | + $insert[] = [ |
| 94 | 'url' => $item['request'] ?? '', | 94 | 'url' => $item['request'] ?? '', |
| 95 | 'referrer_url' => $item['referrer'] ?? '', | 95 | 'referrer_url' => $item['referrer'] ?? '', |
| 96 | 'device_port' => isset($item['is_moblie']) && $item['is_moblie'] == 1 ? 2 : 1, | 96 | 'device_port' => isset($item['is_moblie']) && $item['is_moblie'] == 1 ? 2 : 1, |
| @@ -103,15 +103,21 @@ class ProjectVisit extends Command | @@ -103,15 +103,21 @@ class ProjectVisit extends Command | ||
| 103 | 'updated_at' => date('Y-m-d H:i:s', isset($item['update']) && $item['update'] ? $item['update'] : time()), | 103 | 'updated_at' => date('Y-m-d H:i:s', isset($item['update']) && $item['update'] ? $item['update'] : time()), |
| 104 | 'updated_date' => date('Y-m-d', isset($item['c_time']) && $item['c_time'] ? strtotime($item['c_time']) : time()), | 104 | 'updated_date' => date('Y-m-d', isset($item['c_time']) && $item['c_time'] ? strtotime($item['c_time']) : time()), |
| 105 | 'original_id' => $item['id'], | 105 | 'original_id' => $item['id'], |
| 106 | - ]); | ||
| 107 | - } catch (\Exception $e) { | ||
| 108 | - echo 'date:' . date('Y-m-d H:i:s') . ', task_id: ' . $task->id . ', error: ' . $e->getMessage() . PHP_EOL; | ||
| 109 | - continue; | 106 | + ]; |
| 107 | + } | ||
| 110 | } | 108 | } |
| 111 | } | 109 | } |
| 110 | + try { | ||
| 111 | + $model->insert($insert); | ||
| 112 | + } catch (\Exception $e) { | ||
| 113 | + echo 'date:' . date('Y-m-d H:i:s') . ', task_id: ' . $task->id . ',page: ' . $page . ' error: ' . $e->getMessage() . PHP_EOL; | ||
| 114 | + continue; | ||
| 112 | } | 115 | } |
| 113 | } | 116 | } |
| 114 | } | 117 | } |
| 118 | + if ($max_id) { | ||
| 119 | + $task->max_id = $max_id; | ||
| 120 | + $task->save(); | ||
| 115 | } | 121 | } |
| 116 | } else { | 122 | } else { |
| 117 | return true; | 123 | return true; |
| @@ -122,6 +128,7 @@ class ProjectVisit extends Command | @@ -122,6 +128,7 @@ class ProjectVisit extends Command | ||
| 122 | $data = curl_c($url); | 128 | $data = curl_c($url); |
| 123 | if (isset($data['count']) && $data['count'] > 0) { | 129 | if (isset($data['count']) && $data['count'] > 0) { |
| 124 | $count = $data['count']; | 130 | $count = $data['count']; |
| 131 | + $max_id = $data['data'][0]['id'] ?? 0; | ||
| 125 | 132 | ||
| 126 | $total_page = ceil($count / $page_size); | 133 | $total_page = ceil($count / $page_size); |
| 127 | for ($page = 1; $page <= $total_page; $page++) { | 134 | for ($page = 1; $page <= $total_page; $page++) { |
| @@ -131,14 +138,13 @@ class ProjectVisit extends Command | @@ -131,14 +138,13 @@ class ProjectVisit extends Command | ||
| 131 | $items = $data_page['data']; | 138 | $items = $data_page['data']; |
| 132 | $model = new VisitItem(); | 139 | $model = new VisitItem(); |
| 133 | $p_model = new Visit(); | 140 | $p_model = new Visit(); |
| 141 | + $insert = []; | ||
| 134 | foreach ($items as $item) { | 142 | foreach ($items as $item) { |
| 135 | if (isset($item['id']) && $item['id']) { | 143 | if (isset($item['id']) && $item['id']) { |
| 136 | - $visit = $model->read(['original_id' => $item['id']], 'id'); | ||
| 137 | - if (!$visit) { | ||
| 138 | - try { | 144 | + if ($item['id'] > $task->max_id) { |
| 139 | $p_info = $p_model->read(['ip' => $item['ip'] ?? '', 'updated_date' => $item['day_at'] ?? '']); | 145 | $p_info = $p_model->read(['ip' => $item['ip'] ?? '', 'updated_date' => $item['day_at'] ?? '']); |
| 140 | if ($p_info) { | 146 | if ($p_info) { |
| 141 | - $model->insert([ | 147 | + $insert[] = [ |
| 142 | 'customer_visit_id' => $p_info['id'], | 148 | 'customer_visit_id' => $p_info['id'], |
| 143 | 'url' => $p_info['url'], | 149 | 'url' => $p_info['url'], |
| 144 | 'referrer_url' => $p_info['referrer_url'], | 150 | 'referrer_url' => $p_info['referrer_url'], |
| @@ -150,16 +156,22 @@ class ProjectVisit extends Command | @@ -150,16 +156,22 @@ class ProjectVisit extends Command | ||
| 150 | 'updated_at' => $item['time_str'] ?? $p_info['updated_at'], | 156 | 'updated_at' => $item['time_str'] ?? $p_info['updated_at'], |
| 151 | 'updated_date' => $item['day_at'] ?? $p_info['updated_date'], | 157 | 'updated_date' => $item['day_at'] ?? $p_info['updated_date'], |
| 152 | 'original_id' => $item['id'], | 158 | 'original_id' => $item['id'], |
| 153 | - ]); | 159 | + ]; |
| 160 | + } | ||
| 154 | } | 161 | } |
| 155 | - } catch (\Exception $e) { | ||
| 156 | - echo 'date:' . date('Y-m-d H:i:s') . ', task_id: ' . $task->id . ', error: ' . $e->getMessage() . PHP_EOL; | ||
| 157 | - continue; | ||
| 158 | } | 162 | } |
| 159 | } | 163 | } |
| 164 | + try { | ||
| 165 | + $model->insert($insert); | ||
| 166 | + } catch (\Exception $e) { | ||
| 167 | + echo 'date:' . date('Y-m-d H:i:s') . ', task_id: ' . $task->id . ',page: ' . $page . ' error: ' . $e->getMessage() . PHP_EOL; | ||
| 168 | + continue; | ||
| 160 | } | 169 | } |
| 161 | } | 170 | } |
| 162 | } | 171 | } |
| 172 | + if ($max_id) { | ||
| 173 | + $task->max_id = $max_id; | ||
| 174 | + $task->save(); | ||
| 163 | } | 175 | } |
| 164 | } else { | 176 | } else { |
| 165 | return true; | 177 | return true; |
-
请 注册 或 登录 后发表评论