周海龙 / globalso · 提交

转到一个项目

作者 刘锟
提交 ebd0d2904256ab1b4d7acdb2074d4eba1db79240 1 个父辈 e37de464

update

内嵌 并排对比
正在显示 1 个修改的文件 包含 57 行增加 和 45 行删除
@@ -60,7 +60,7 @@ class ProjectVisit extends Command @@ -60,7 +60,7 @@ class ProjectVisit extends Command
60 $api_type = $task->api_type; 60 $api_type = $task->api_type;
61 $api_url = $task->api_url; 61 $api_url = $task->api_url;
62 62
63 - $page_size = 200; 63 + $page_size = 1000;
64 64
65 echo 'date:' . date('Y-m-d H:i:s') . ', task_id: ' . $task->id . ', task_type: ' . $api_type . ', update start' . PHP_EOL; 65 echo 'date:' . date('Y-m-d H:i:s') . ', task_id: ' . $task->id . ', task_type: ' . $api_type . ', update start' . PHP_EOL;
66 66
@@ -76,6 +76,7 @@ class ProjectVisit extends Command @@ -76,6 +76,7 @@ class ProjectVisit extends Command
76 $data = curl_c($url); 76 $data = curl_c($url);
77 if (isset($data['count']) && $data['count'] > 0) { 77 if (isset($data['count']) && $data['count'] > 0) {
78 $count = $data['count']; 78 $count = $data['count'];
  79 + $max_id = $data['data'][0]['id'] ?? 0;
79 80
80 $total_page = ceil($count / $page_size); 81 $total_page = ceil($count / $page_size);
81 for ($page = 1; $page <= $total_page; $page++) { 82 for ($page = 1; $page <= $total_page; $page++) {
@@ -84,35 +85,40 @@ class ProjectVisit extends Command @@ -84,35 +85,40 @@ class ProjectVisit extends Command
84 if (isset($data_page['data']) && $data_page['data']) { 85 if (isset($data_page['data']) && $data_page['data']) {
85 $items = $data_page['data']; 86 $items = $data_page['data'];
86 $model = new Visit(); 87 $model = new Visit();
  88 + $insert = [];
87 foreach ($items as $item) { 89 foreach ($items as $item) {
88 if (isset($item['id']) && $item['id']) { 90 if (isset($item['id']) && $item['id']) {
89 - $visit = $model->read(['original_id' => $item['id']], 'id');  
90 - if (!$visit) {  
91 - try {  
92 - $url_arr = parse_url($item['request'] ?? '');  
93 - $model->insert([  
94 - 'url' => $item['request'] ?? '',  
95 - 'referrer_url' => $item['referrer'] ?? '',  
96 - 'device_port' => isset($item['is_moblie']) && $item['is_moblie'] == 1 ? 2 : 1,  
97 - 'country' => $item['ip_area'] ?? '',  
98 - 'ip' => $item['ip'] ?? '',  
99 - 'depth' => $item['pv'],  
100 - 'domain' => $url_arr['host'] ?? '',  
101 - 'is_inquiry' => $item['is_cf'] ?? 0,  
102 - 'created_at' => date('Y-m-d H:i:s', isset($item['update']) && $item['update'] ? $item['update'] : time()),  
103 - 'updated_at' => date('Y-m-d H:i:s', isset($item['update']) && $item['update'] ? $item['update'] : time()),  
104 - 'updated_date' => date('Y-m-d', isset($item['c_time']) && $item['c_time'] ? strtotime($item['c_time']) : time()),  
105 - 'original_id' => $item['id'],  
106 - ]);  
107 - } catch (\Exception $e) {  
108 - echo 'date:' . date('Y-m-d H:i:s') . ', task_id: ' . $task->id . ', error: ' . $e->getMessage() . PHP_EOL;  
109 - continue;  
110 - } 91 + if ($item['id'] > $task->max_id) {
  92 + $url_arr = parse_url($item['request'] ?? '');
  93 + $insert[] = [
  94 + 'url' => $item['request'] ?? '',
  95 + 'referrer_url' => $item['referrer'] ?? '',
  96 + 'device_port' => isset($item['is_moblie']) && $item['is_moblie'] == 1 ? 2 : 1,
  97 + 'country' => $item['ip_area'] ?? '',
  98 + 'ip' => $item['ip'] ?? '',
  99 + 'depth' => $item['pv'],
  100 + 'domain' => $url_arr['host'] ?? '',
  101 + 'is_inquiry' => $item['is_cf'] ?? 0,
  102 + 'created_at' => date('Y-m-d H:i:s', isset($item['update']) && $item['update'] ? $item['update'] : time()),
  103 + 'updated_at' => date('Y-m-d H:i:s', isset($item['update']) && $item['update'] ? $item['update'] : time()),
  104 + 'updated_date' => date('Y-m-d', isset($item['c_time']) && $item['c_time'] ? strtotime($item['c_time']) : time()),
  105 + 'original_id' => $item['id'],
  106 + ];
111 } 107 }
112 } 108 }
113 } 109 }
  110 + try {
  111 + $model->insert($insert);
  112 + } catch (\Exception $e) {
  113 + echo 'date:' . date('Y-m-d H:i:s') . ', task_id: ' . $task->id . ',page: ' . $page . ' error: ' . $e->getMessage() . PHP_EOL;
  114 + continue;
  115 + }
114 } 116 }
115 } 117 }
  118 + if ($max_id) {
  119 + $task->max_id = $max_id;
  120 + $task->save();
  121 + }
116 } else { 122 } else {
117 return true; 123 return true;
118 } 124 }
@@ -122,6 +128,7 @@ class ProjectVisit extends Command @@ -122,6 +128,7 @@ class ProjectVisit extends Command
122 $data = curl_c($url); 128 $data = curl_c($url);
123 if (isset($data['count']) && $data['count'] > 0) { 129 if (isset($data['count']) && $data['count'] > 0) {
124 $count = $data['count']; 130 $count = $data['count'];
  131 + $max_id = $data['data'][0]['id'] ?? 0;
125 132
126 $total_page = ceil($count / $page_size); 133 $total_page = ceil($count / $page_size);
127 for ($page = 1; $page <= $total_page; $page++) { 134 for ($page = 1; $page <= $total_page; $page++) {
@@ -131,36 +138,41 @@ class ProjectVisit extends Command @@ -131,36 +138,41 @@ class ProjectVisit extends Command
131 $items = $data_page['data']; 138 $items = $data_page['data'];
132 $model = new VisitItem(); 139 $model = new VisitItem();
133 $p_model = new Visit(); 140 $p_model = new Visit();
  141 + $insert = [];
134 foreach ($items as $item) { 142 foreach ($items as $item) {
135 if (isset($item['id']) && $item['id']) { 143 if (isset($item['id']) && $item['id']) {
136 - $visit = $model->read(['original_id' => $item['id']], 'id');  
137 - if (!$visit) {  
138 - try {  
139 - $p_info = $p_model->read(['ip' => $item['ip'] ?? '', 'updated_date' => $item['day_at'] ?? '']);  
140 - if ($p_info) {  
141 - $model->insert([  
142 - 'customer_visit_id' => $p_info['id'],  
143 - 'url' => $p_info['url'],  
144 - 'referrer_url' => $p_info['referrer_url'],  
145 - 'device_port' => $p_info['device_port'],  
146 - 'country' => $p_info['country'],  
147 - 'ip' => $p_info['ip'],  
148 - 'domain' => $p_info['domain'],  
149 - 'created_at' => $item['time_str'] ?? $p_info['created_at'],  
150 - 'updated_at' => $item['time_str'] ?? $p_info['updated_at'],  
151 - 'updated_date' => $item['day_at'] ?? $p_info['updated_date'],  
152 - 'original_id' => $item['id'],  
153 - ]);  
154 - }  
155 - } catch (\Exception $e) {  
156 - echo 'date:' . date('Y-m-d H:i:s') . ', task_id: ' . $task->id . ', error: ' . $e->getMessage() . PHP_EOL;  
157 - continue; 144 + if ($item['id'] > $task->max_id) {
  145 + $p_info = $p_model->read(['ip' => $item['ip'] ?? '', 'updated_date' => $item['day_at'] ?? '']);
  146 + if ($p_info) {
  147 + $insert[] = [
  148 + 'customer_visit_id' => $p_info['id'],
  149 + 'url' => $p_info['url'],
  150 + 'referrer_url' => $p_info['referrer_url'],
  151 + 'device_port' => $p_info['device_port'],
  152 + 'country' => $p_info['country'],
  153 + 'ip' => $p_info['ip'],
  154 + 'domain' => $p_info['domain'],
  155 + 'created_at' => $item['time_str'] ?? $p_info['created_at'],
  156 + 'updated_at' => $item['time_str'] ?? $p_info['updated_at'],
  157 + 'updated_date' => $item['day_at'] ?? $p_info['updated_date'],
  158 + 'original_id' => $item['id'],
  159 + ];
158 } 160 }
159 } 161 }
160 } 162 }
161 } 163 }
  164 + try {
  165 + $model->insert($insert);
  166 + } catch (\Exception $e) {
  167 + echo 'date:' . date('Y-m-d H:i:s') . ', task_id: ' . $task->id . ',page: ' . $page . ' error: ' . $e->getMessage() . PHP_EOL;
  168 + continue;
  169 + }
162 } 170 }
163 } 171 }
  172 + if ($max_id) {
  173 + $task->max_id = $max_id;
  174 + $task->save();
  175 + }
164 } else { 176 } else {
165 return true; 177 return true;
166 } 178 }