Update README

deploy_robots_v3.py

@@ -0,0 +1,138 @@
+"""Fix Nginx config on gfil-lab.com and deploy robots.txt to gfil-intel.xyz"""
+import paramiko
+import time
+
+JD_HOST = "111.228.37.165"
+JD_USER = "root"
+JD_PASS = "Liudecai110"
+
+LAB_HOST = "216.144.233.14"
+LAB_USER = "root"
+LAB_PASS = "Kt9V72Tx2c48ChikKU"
+
+RN_HOST = "107.174.186.162"
+RN_USER = "root"
+RN_PASS = "liudapao2026"
+
+jd = paramiko.SSHClient()
+jd.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+jd.connect(JD_HOST, port=22, username=JD_USER, password=JD_PASS,
+           timeout=20, banner_timeout=60, allow_agent=False, look_for_keys=False)
+
+# === Fix gfil-lab.com Nginx config ===
+print("[1/3] Fixing Nginx config on gfil-lab.com...")
+fix_cmd = """sshpass -p 'Kt9V72Tx2c48ChikKU' ssh -o StrictHostKeyChecking=no root@216.144.233.14 '
+# Remove the broken line with standalone "n"
+sed -i "/^n    location = \/robots\.txt/d" /etc/nginx/sites-enabled/gfil
+
+# Check current state
+echo "=== Current config (robots.txt area) ==="
+grep -n "robots" /etc/nginx/sites-enabled/gfil
+
+# If robots.txt location block already exists and is clean, just test
+# If not, add it properly
+if grep -q "robots.txt" /etc/nginx/sites-enabled/gfil; then
+    echo "robots.txt block exists, checking syntax..."
+else
+    echo "Adding robots.txt location block..."
+    sed -i "/location \/ {/i\\    location = /robots.txt {\\n        alias /var/www/gfil-lab/robots.txt;\\n        default_type text/plain;\\n    }" /etc/nginx/sites-enabled/gfil
+fi
+
+# Show the config around the robots block
+grep -n -B2 -A5 "robots" /etc/nginx/sites-enabled/gfil
+
+# Test nginx
+nginx -t 2>&1
+if [ $? -eq 0 ]; then
+    systemctl reload nginx
+    echo "Nginx reloaded OK"
+else
+    echo "Nginx test still failing, showing full config..."
+    cat -n /etc/nginx/sites-enabled/gfil
+fi
+'"""
+stdin, stdout, stderr = jd.exec_command(fix_cmd, timeout=30)
+print(stdout.read().decode()[:3000])
+
+# === Deploy robots.txt to gfil-intel.xyz on RackNerd ===
+print("\n[2/3] Deploying robots.txt to gfil-intel.xyz (RackNerd)...")
+intel_cmd = """sshpass -p 'liudapao2026' ssh -o StrictHostKeyChecking=no root@107.174.186.162 '
+mkdir -p /var/www/gfil-intel
+
+# Write robots.txt
+cat > /var/www/gfil-intel/robots.txt << '"'"'EOF'"'"'
+User-agent: *
+Allow: /
+
+User-agent: OAI-SearchBot
+Allow: /
+User-agent: ChatGPT-User
+Allow: /
+User-agent: GPTBot
+Allow: /
+User-agent: ClaudeBot
+Allow: /
+User-agent: anthropic-ai
+Allow: /
+User-agent: PerplexityBot
+Allow: /
+User-agent: Google-Extended
+Allow: /
+User-agent: GoogleOther
+Allow: /
+User-agent: Bytespider
+Allow: /
+User-agent: DeepSeekBot
+Allow: /
+User-agent: KimiBot
+Allow: /
+User-agent: Baiduspider
+Allow: /
+User-agent: YandexBot
+Allow: /
+
+Sitemap: https://gfil-intel.xyz/sitemap.xml
+EOF
+
+# Verify
+echo "=== robots.txt ==="
+cat /var/www/gfil-intel/robots.txt | head -5
+
+# Add Nginx location block to gfil-mask if not already there
+if grep -q "robots.txt" /etc/nginx/sites-available/gfil-mask; then
+    echo "robots.txt block already in gfil-mask"
+else
+    echo "Adding robots.txt to gfil-mask..."
+    sed -i "/location \\/ {/i\\    location = /robots.txt {\\n        alias /var/www/gfil-intel/robots.txt;\\n        default_type text/plain;\\n    }" /etc/nginx/sites-available/gfil-mask
+fi
+
+# Test and reload
+nginx -t 2>&1
+if [ $? -eq 0 ]; then
+    systemctl reload nginx
+    echo "Nginx reloaded on RackNerd"
+else
+    echo "Nginx test FAILED on RackNerd"
+    cat -n /etc/nginx/sites-available/gfil-mask | head -40
+fi
+'"""
+stdin, stdout, stderr = jd.exec_command(intel_cmd, timeout=30)
+print(stdout.read().decode()[:2000])
+
+# === Verify both from external ===
+print("\n[3/3] External verification...")
+jd.close()
+
+import urllib.request
+for domain in ['gfil-lab.com', 'gfil-intel.xyz', 'blog.quant-view.xyz']:
+    try:
+        req = urllib.request.Request(f'https://{domain}/robots.txt',
+                                     headers={'User-Agent': 'Mozilla/5.0'})
+        r = urllib.request.urlopen(req, timeout=10)
+        content = r.read().decode()
+        has_gptbot = 'GPTBot' in content
+        has_perplexity = 'PerplexityBot' in content
+        has_sitemap = 'Sitemap:' in content
+        print(f"  {domain}: {r.status} OK | GPTBot={has_gptbot} | PerplexityBot={has_perplexity} | Sitemap={has_sitemap} | {len(content)} chars")
+    except Exception as e:
+        print(f"  {domain}: {e}")