robots.txt
661 字节
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
User-agent: *
Allow: /
Disallow: /404/
User-agent: MSNBot
Disallow: /
User-agent: YoudaoBot
Disallow: /
User-agent: JikeSpider
Disallow: /
User-agent: EasouSpider
Disallow: /
User-agent: AhrefsBot
Disallow: /
User-agent: MJ12bot
Disallow: /
User-agent: YYSpider
Disallow: /
User-agent: Teoma
Disallow: /
User-agent: rogerbot
Disallow: /
User-agent: exabot
Disallow: /
User-agent: DotBot
Disallow: /
User-agent: gigabot
Disallow: /
User-agent: BLEXBot
Disallow: /
User-agent: DOCOMO Sprider
Disallow: /
User-agent: Applebot
Disallow: /
User-agent: SeznamBot
Disallow: /
User-agent: SemrushBot
Disallow: /
Sitemap: https://customsite.globalso.site/sitemap.xml