140 lines
5.5 KiB
Python
140 lines
5.5 KiB
Python
|
#!/usr/bin/env python
|
|||
|
|
# -*- coding: utf-8 -*-
|
|||
|
|
"""
|
|||
|
|
Gemini 地下室方案 #1 替代版: 用 GitHub Models (免费) 替代 Google Cloud NLP 做实体显著性检测
|
|||
|
|
GitHub Models 免费模型: gpt-4o-mini, Phi-4, Llama-3.3-70B 等
|
|||
|
|
用法: python nlp_content_audit.py article.html --targets "Gold" "XAUUSD" "position sizing"
|
|||
|
|
"""
|
|||
|
|
import sys, io, os, re, json, urllib.request
|
|||
|
|
sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8')
|
|||
|
|
|
|||
|
|
# GitHub Models free endpoint (via GitHub Marketplace)
|
|||
|
|
# 优先读环境变量，再读项目 .env config
|
|||
|
|
sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), '..'))
|
|||
|
|
from config import GITHUB_TOKEN as _CFG_TOKEN
|
|||
|
|
GITHUB_TOKEN = os.environ.get('GITHUB_TOKEN') or _CFG_TOKEN or ''
|
|||
|
|
if not GITHUB_TOKEN:
|
|||
|
|
print("❌ 未找到 GITHUB_TOKEN。请在 .env 中设置或设置环境变量")
|
|||
|
|
sys.exit(1)
|
|||
|
|
|
|||
|
|
MODEL = 'gpt-4o-mini' # Free tier on GitHub Models
|
|||
|
|
ENDPOINT = 'https://models.inference.ai.azure.com/chat/completions'
|
|||
|
|
|
|||
|
|
def analyze_entities_with_llm(text, target_entities=None):
|
|||
|
|
"""用 LLM 做实体显著性分析 — 替代 Google Cloud NLP"""
|
|||
|
|
|
|||
|
|
prompt = f"""You are a search engine content quality auditor. Analyze this trading/finance article and:
|
|||
|
|
|
|||
|
|
1. Extract the TOP 10 entities (people, concepts, instruments, strategies) mentioned.
|
|||
|
|
2. Rate each entity's SALIENCE (prominence) from 0.0 to 1.0 based on:
|
|||
|
|
- 1.0 = THE main topic, appears in title and first sentence, discussed throughout
|
|||
|
|
- 0.5 = Mentioned several times but not central
|
|||
|
|
- 0.1 = Passing mention only
|
|||
|
|
3. If the first paragraph doesn't directly state the main topic within 15 words, flag it.
|
|||
|
|
4. Score overall content focus: A (laser-focused), B (mostly focused), C (scattered), D (all over the place).
|
|||
|
|
|
|||
|
|
Text to analyze:
|
|||
|
|
---
|
|||
|
|
{text[:3000]}
|
|||
|
|
---
|
|||
|
|
|
|||
|
|
Respond in JSON format:
|
|||
|
|
{{"entities": [{{"name": "...", "salience": 0.X, "type": "..."}}], "focus_grade": "A/B/C/D", "first_paragraph_issue": true/false, "issues": ["..."]}}"""
|
|||
|
|
|
|||
|
|
payload = json.dumps({
|
|||
|
|
"model": MODEL,
|
|||
|
|
"messages": [{"role": "user", "content": prompt}],
|
|||
|
|
"temperature": 0.1,
|
|||
|
|
"max_tokens": 1000
|
|||
|
|
})
|
|||
|
|
|
|||
|
|
req = urllib.request.Request(ENDPOINT, data=payload.encode(),
|
|||
|
|
headers={
|
|||
|
|
'Content-Type': 'application/json',
|
|||
|
|
'Authorization': f'Bearer {GITHUB_TOKEN}',
|
|||
|
|
'User-Agent': 'GFIL-NLP-Audit/1.0'
|
|||
|
|
})
|
|||
|
|
|
|||
|
|
try:
|
|||
|
|
with urllib.request.urlopen(req, timeout=30) as resp:
|
|||
|
|
result = json.loads(resp.read().decode())
|
|||
|
|
content = result['choices'][0]['message']['content']
|
|||
|
|
# Extract JSON from response
|
|||
|
|
json_match = re.search(r'\{[\s\S]*\}', content)
|
|||
|
|
if json_match:
|
|||
|
|
return json.loads(json_match.group())
|
|||
|
|
return {"error": "Could not parse LLM response", "raw": content[:500]}
|
|||
|
|
except Exception as e:
|
|||
|
|
return {"error": str(e)}
|
|||
|
|
|
|||
|
|
def audit_html_file(filepath, target_entities=None):
|
|||
|
|
"""分析 HTML 文件文本"""
|
|||
|
|
with open(filepath, 'r', encoding='utf-8') as f:
|
|||
|
|
html = f.read()
|
|||
|
|
|
|||
|
|
title = re.search(r'<title>(.*?)</title>', html)
|
|||
|
|
h1 = re.search(r'<h1>(.*?)</h1>', html)
|
|||
|
|
body = re.sub(r'<[^>]+>', ' ', html)
|
|||
|
|
body = re.sub(r'\s+', ' ', body).strip()
|
|||
|
|
|
|||
|
|
title_text = title.group(1) if title else ''
|
|||
|
|
h1_text = h1.group(1) if h1 else ''
|
|||
|
|
|
|||
|
|
print(f"文件: {os.path.basename(filepath)}")
|
|||
|
|
print(f"标题: {title_text[:100]}")
|
|||
|
|
print(f"H1: {h1_text[:100]}")
|
|||
|
|
print(f"分析中...")
|
|||
|
|
|
|||
|
|
text = f"{title_text}\n{h1_text}\n{body[:2500]}"
|
|||
|
|
result = analyze_entities_with_llm(text, target_entities)
|
|||
|
|
|
|||
|
|
if 'error' in result:
|
|||
|
|
print(f"\n❌ 分析失败: {result['error']}")
|
|||
|
|
return
|
|||
|
|
|
|||
|
|
print(f"\n=== 内容质量审计 ===")
|
|||
|
|
print(f"集中度评分: {result.get('focus_grade', 'N/A')}")
|
|||
|
|
if result.get('first_paragraph_issue'):
|
|||
|
|
print(f"⚠️ 第一段问题: 主题不够突出（前15字未点题）")
|
|||
|
|
|
|||
|
|
print(f"\n=== 实体显著性 Top 10 ===")
|
|||
|
|
print(f"{'实体':<30} {'显著性':>8} {'类型':<15}")
|
|||
|
|
print("-" * 60)
|
|||
|
|
for e in result.get('entities', [])[:10]:
|
|||
|
|
bar = '█' * int(e['salience'] * 50)
|
|||
|
|
print(f"{e['name']:<30} {e['salience']:.3f} {e.get('type', ''):<15}")
|
|||
|
|
|
|||
|
|
if target_entities:
|
|||
|
|
print(f"\n=== 目标实体检测 ===")
|
|||
|
|
entity_names = {e['name'].lower(): e for e in result.get('entities', [])}
|
|||
|
|
for t in target_entities:
|
|||
|
|
found = None
|
|||
|
|
for name, e in entity_names.items():
|
|||
|
|
if t.lower() in name:
|
|||
|
|
found = e
|
|||
|
|
break
|
|||
|
|
if found:
|
|||
|
|
ok = '✅ 达标' if found['salience'] >= 0.8 else '❌ 不够(需≥0.8)'
|
|||
|
|
print(f" {t}: 显著度 {found['salience']:.3f} {ok}")
|
|||
|
|
else:
|
|||
|
|
print(f" {t}: ❌ 未找到! 需在标题或第一段加入")
|
|||
|
|
|
|||
|
|
if result.get('issues'):
|
|||
|
|
print(f"\n=== 发现的问题 ===")
|
|||
|
|
for issue in result['issues']:
|
|||
|
|
print(f" - {issue}")
|
|||
|
|
|
|||
|
|
if __name__ == '__main__':
|
|||
|
|
import argparse
|
|||
|
|
p = argparse.ArgumentParser(description='GitHub Models 免费实体审计 (替代Google NLP)')
|
|||
|
|
p.add_argument('file', help='HTML 文件路径')
|
|||
|
|
p.add_argument('--targets', nargs='+', help='目标实体: --targets Gold XAUUSD "position sizing"')
|
|||
|
|
args = p.parse_args()
|
|||
|
|
|
|||
|
|
if '1234567890' in GITHUB_TOKEN:
|
|||
|
|
print("❌ 请设置 GITHUB_TOKEN 环境变量 (你的 GitHub Personal Access Token)")
|
|||
|
|
print(" $env:GITHUB_TOKEN='github_pat_xxxx' # PowerShell")
|
|||
|
|
sys.exit(1)
|
|||
|
|
|
|||
|
|
audit_html_file(args.file, args.targets)
|