Update README
This commit is contained in:
13
check_https_robots.py
Normal file
13
check_https_robots.py
Normal file
@ -0,0 +1,13 @@
|
||||
import urllib.request, sys, io
|
||||
sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding="utf-8")
|
||||
|
||||
req = urllib.request.Request("https://gfil-lab.com/robots.txt", headers={"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"})
|
||||
r = urllib.request.urlopen(req, timeout=10)
|
||||
content = r.read().decode()
|
||||
print("Status:", r.status)
|
||||
print("Size:", len(content), "chars")
|
||||
print("GPTBot:", "GPTBot" in content)
|
||||
print("PerplexityBot:", "PerplexityBot" in content)
|
||||
print("ClaudeBot:", "ClaudeBot" in content)
|
||||
print()
|
||||
print(content)
|
||||
Reference in New Issue
Block a user