14 lines
545 B
Python
14 lines
545 B
Python
import urllib.request, sys, io
|
|
sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding="utf-8")
|
|
|
|
req = urllib.request.Request("https://gfil-lab.com/robots.txt", headers={"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"})
|
|
r = urllib.request.urlopen(req, timeout=10)
|
|
content = r.read().decode()
|
|
print("Status:", r.status)
|
|
print("Size:", len(content), "chars")
|
|
print("GPTBot:", "GPTBot" in content)
|
|
print("PerplexityBot:", "PerplexityBot" in content)
|
|
print("ClaudeBot:", "ClaudeBot" in content)
|
|
print()
|
|
print(content)
|