import urllib.request, sys, io
sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding="utf-8")

for domain in ["gfil-lab.com", "gfil-intel.xyz", "blog.quant-view.xyz"]:
    try:
        url = "https://" + domain + "/robots.txt?nocache=1"
        req = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0"})
        r = urllib.request.urlopen(req, timeout=10)
        content = r.read().decode()
        gb = "GPTBot" in content
        cb = "ClaudeBot" in content
        pb = "PerplexityBot" in content
        sm = "Sitemap" in content
        print(domain + ": " + str(r.status) + " | " + str(len(content)) + " chars | GPTBot=" + str(gb) + " ClaudeBot=" + str(cb) + " PerplexityBot=" + str(pb) + " Sitemap=" + str(sm))
    except Exception as e:
        print(domain + ": " + str(e))

# Also check homepage still works
for domain in ["gfil-lab.com", "gfil-intel.xyz"]:
    try:
        req = urllib.request.Request("https://" + domain + "/", headers={"User-Agent": "Mozilla/5.0"})
        r = urllib.request.urlopen(req, timeout=10)
        print(domain + " homepage: " + str(r.status) + " OK")
    except Exception as e:
        print(domain + " homepage: ERROR " + str(e))