使用验证码解决进行学术研究网络抓取

抓取 Google Scholar、PubMed 或 IEEE Xplore 时，请求量一高就会跳出 reCAPTCHA 或 Cloudflare Turnstile——这是文献综述、文献计量分析项目最常见的卡点。本文给出一套可复用的方案：

• 识别不同学术数据库弹出的验证码类型
• 用 CaptchaAI 的 API 自动求解，脚本不再卡在验证码页面
• 按各数据库的限流规则控制抓取节奏，降低被封 IP 的概率

学术数据库都用哪种验证码

多数国际学术站点把 reCAPTCHA 脚本部署在 Google 的域名下，国内网络访问这类资源本身就不太稳定，容易和验证码拦截混在一起分不清。下表是几个常用来源触发验证码的场景：

引文抓取脚本怎么写

下面的脚本演示完整流程：请求页面 → 检测 sitekey → 调用 CaptchaAI 求解 → 提交 token 重新抓取。solve_captcha() 把提交任务和轮询结果封装成一次调用；AcademicScraper 把翻页搜索、详情抓取、CSV 导出拆成独立方法。

import requests
import time
import re
from bs4 import BeautifulSoup
import csv

CAPTCHAAI_KEY = "YOUR_API_KEY"
CAPTCHAAI_URL = "https://ocr.captchaai.com"


def solve_captcha(method, sitekey, pageurl, **kwargs):
    data = {
        "key": CAPTCHAAI_KEY, "method": method,
        "googlekey": sitekey, "pageurl": pageurl, "json": 1,
    }
    data.update(kwargs)
    resp = requests.post(f"{CAPTCHAAI_URL}/in.php", data=data)
    task_id = resp.json()["request"]
    for _ in range(60):
        time.sleep(5)
        result = requests.get(f"{CAPTCHAAI_URL}/res.php", params={
            "key": CAPTCHAAI_KEY, "action": "get",
            "id": task_id, "json": 1,
        })
        r = result.json()
        if r["request"] != "CAPCHA_NOT_READY":
            return r["request"]
    raise TimeoutError("Timeout")


class AcademicScraper:
    def __init__(self, proxy=None):
        self.session = requests.Session()
        if proxy:
            self.session.proxies = {"http": proxy, "https": proxy}
        self.session.headers.update({
            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
            "AppleWebKit/537.36 Chrome/126.0.0.0 Safari/537.36",
            "Accept-Language": "en-US,en;q=0.9",
        })

    def search_papers(self, search_url, query, max_pages=10):
        """Search academic database for papers matching query."""
        all_papers = []

        for page in range(max_pages):
            url = f"{search_url}?q={query}&start={page * 10}"
            resp = self.session.get(url, timeout=30)

            # Handle CAPTCHA
            if self._has_captcha(resp.text):
                resp = self._solve_and_retry(resp.text, url)

            papers = self._parse_results(resp.text)
            if not papers:
                break  # No more results

            all_papers.extend(papers)
            print(f"Page {page + 1}: {len(papers)} papers")
            time.sleep(5)  # Respectful delay

        return all_papers

    def get_paper_details(self, paper_url):
        """Get detailed metadata for a single paper."""
        resp = self.session.get(paper_url, timeout=30)

        if self._has_captcha(resp.text):
            resp = self._solve_and_retry(resp.text, paper_url)

        soup = BeautifulSoup(resp.text, "html.parser")
        return {
            "title": self._safe_text(soup, "h1, .article-title"),
            "authors": self._safe_text(soup, ".authors, .author-list"),
            "abstract": self._safe_text(soup, ".abstract, #abstract"),
            "doi": self._safe_text(soup, ".doi, [data-doi]"),
            "journal": self._safe_text(soup, ".journal-name, .publication"),
            "year": self._safe_text(soup, ".pub-date, .year"),
            "citations": self._safe_text(soup, ".citation-count, .cited-by"),
        }

    def export_to_csv(self, papers, filename):
        """Export collected papers to CSV."""
        if not papers:
            return
        keys = papers[0].keys()
        with open(filename, "w", newline="", encoding="utf-8") as f:
            writer = csv.DictWriter(f, fieldnames=keys)
            writer.writeheader()
            writer.writerows(papers)
        print(f"Exported {len(papers)} papers to {filename}")

    def _has_captcha(self, html):
        return any(tag in html.lower() for tag in [
            'data-sitekey', 'g-recaptcha', 'cf-turnstile',
        ])

    def _solve_and_retry(self, html, url):
        match = re.search(r'data-sitekey="([^"]+)"', html)
        if not match:
            return self.session.get(url)

        sitekey = match.group(1)
        if 'cf-turnstile' in html:
            token = solve_captcha("turnstile", sitekey, url)
            return self.session.post(url, data={"cf-turnstile-response": token})
        else:
            token = solve_captcha("userrecaptcha", sitekey, url)
            return self.session.post(url, data={"g-recaptcha-response": token})

    def _parse_results(self, html):
        soup = BeautifulSoup(html, "html.parser")
        papers = []
        for item in soup.select(".gs_r, .search-result, article.result"):
            title_el = item.select_one("h3 a, .result-title a")
            if title_el:
                papers.append({
                    "title": title_el.get_text(strip=True),
                    "url": title_el.get("href", ""),
                    "snippet": self._safe_text(item, ".gs_rs, .abstract-snippet"),
                    "authors": self._safe_text(item, ".gs_a, .author-info"),
                })
        return papers

    def _safe_text(self, soup, selector):
        el = soup.select_one(selector)
        return el.get_text(strip=True) if el else ""


# Usage — Literature review
scraper = AcademicScraper(
    proxy="http://user:[email protected]:5000"
)

papers = scraper.search_papers(
    "https://scholar.example.com/scholar",
    query="machine learning CAPTCHA solving",
    max_pages=5,
)

# Get details for top papers
detailed = []
for paper in papers[:20]:
    if paper["url"]:
        detail = scraper.get_paper_details(paper["url"])
        detailed.append(detail)
        time.sleep(3)

scraper.export_to_csv(detailed, "literature_review.csv")

_has_captcha 只检测 data-sitekey、g-recaptcha、cf-turnstile 三个标记，命中就转 _solve_and_retry 处理，逻辑保持精简。

顺着引用关系做文献计量分析

文献计量分析常要沿着"被引用"链接做多层遍历。bibliometric_analysis() 复用同一个 scraper 实例，按 depth 递归抓取引用网络，每层最多跟进 5 篇，避免节点数量膨胀。

def bibliometric_analysis(scraper, seed_papers, depth=2):
    """Follow citations to build a citation network."""
    visited = set()
    network = []

    def _crawl(paper_url, current_depth):
        if current_depth > depth or paper_url in visited:
            return
        visited.add(paper_url)

        try:
            details = scraper.get_paper_details(paper_url)
            network.append(details)

            # Follow "cited by" links
            resp = scraper.session.get(f"{paper_url}/citations", timeout=30)
            if scraper._has_captcha(resp.text):
                resp = scraper._solve_and_retry(resp.text, f"{paper_url}/citations")

            citations = scraper._parse_results(resp.text)
            for cite in citations[:5]:  # Limit breadth
                if cite["url"]:
                    _crawl(cite["url"], current_depth + 1)
                    time.sleep(3)

        except Exception as e:
            print(f"Error crawling {paper_url}: {e}")

    for paper in seed_papers:
        _crawl(paper["url"], 0)

    return network

各数据库的限流建议

学术站点封 IP 很快，延迟设置偏保守更划算。下表是常见来源的建议延迟和每小时页数上限，仅供参考，请结合自己账号的历史表现调整。只采集公开可访问的元数据（标题、作者、摘要），遵循目标站点的 robots 协议和所在机构的访问授权范围。

抓取学术数据库常遇到的问题

常见问题

CaptchaAI 支持 reCAPTCHA v3 吗？

支持。CaptchaAI 能识别 reCAPTCHA v2、v3 和 Cloudflare Turnstile，Google Scholar 常用的无感 v3 挑战也在覆盖范围内，接口共用同一个 userrecaptcha 方法。

国内网络访问 Google Scholar、PubMed 验证码特别多，为什么？

reCAPTCHA 脚本托管在 Google 域名下，国内网络访问本身就不太稳定，容易和验证码拦截混在一起。先确认页面能否打开、sitekey 能否取到——这是网络可达性问题。

抓取学术数据库的公开元数据算不算合规？

标题、作者、摘要通常可以采集，全文取决于授权协议。PubMed 明确开放了 E-utilities API，优先用官方接口更稳定。

机构代理和 CaptchaAI 可以一起用吗？

可以，两者互不影响。机构代理负责请求来源，CaptchaAI 只负责识别弹出的验证码。

轮询一直拿不到结果，是超时了吗？

solve_captcha() 最多轮询 60 次、每次 5 秒，等满 5 分钟才会抛 TimeoutError。多数任务 15–30 秒内就能拿到 token，长时间拿不到通常是 sitekey 取错了。

相关指南

• 轮换自有服务器基础设施
• 验证码识别的 QA 与授权测试范围

别再对着验证码手动点了——注册 CaptchaAI 拿到 API Key，把学术数据采集接入自动识别流程。