26 lines
1.2 KiB
Python
26 lines
1.2 KiB
Python
|
import urllib.request, sys, io
|
||
|
|
sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding="utf-8")
|
||
|
|
|
||
|
|
for domain in ["gfil-lab.com", "gfil-intel.xyz", "blog.quant-view.xyz"]:
|
||
|
|
try:
|
||
|
|
url = "https://" + domain + "/robots.txt?nocache=1"
|
||
|
|
req = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0"})
|
||
|
|
r = urllib.request.urlopen(req, timeout=10)
|
||
|
|
content = r.read().decode()
|
||
|
|
gb = "GPTBot" in content
|
||
|
|
cb = "ClaudeBot" in content
|
||
|
|
pb = "PerplexityBot" in content
|
||
|
|
sm = "Sitemap" in content
|
||
|
|
print(domain + ": " + str(r.status) + " | " + str(len(content)) + " chars | GPTBot=" + str(gb) + " ClaudeBot=" + str(cb) + " PerplexityBot=" + str(pb) + " Sitemap=" + str(sm))
|
||
|
|
except Exception as e:
|
||
|
|
print(domain + ": " + str(e))
|
||
|
|
|
||
|
|
# Also check homepage still works
|
||
|
|
for domain in ["gfil-lab.com", "gfil-intel.xyz"]:
|
||
|
|
try:
|
||
|
|
req = urllib.request.Request("https://" + domain + "/", headers={"User-Agent": "Mozilla/5.0"})
|
||
|
|
r = urllib.request.urlopen(req, timeout=10)
|
||
|
|
print(domain + " homepage: " + str(r.status) + " OK")
|
||
|
|
except Exception as e:
|
||
|
|
print(domain + " homepage: ERROR " + str(e))
|