Source code for google_scholar_search_mcp.scholar

from __future__ import annotations

import logging
import re
from collections.abc import Iterator
from typing import TYPE_CHECKING, TypeVar

from scholarly import ProxyGenerator, scholarly

from .block_detector import BlockDetector
from .errors import (
    ScholarBlockedError,
    ScholarInvalidArgumentError,
    ScholarNotFoundError,
)
from .models import PaperDetails, PaperResult
from .session_identity import login_cookie_names, parse_cookies, set_identity
from .throttle import throttle

if TYPE_CHECKING:
    from .config import Config

logger = logging.getLogger(__name__)

T = TypeVar("T")


def _safe_int(value: object) -> int | None:
    """Convert a value to ``int`` without raising.

    Args:
        value (object): Value to convert, e.g. a year string from Scholar.

    Returns:
        int | None: The integer, or ``None`` if ``value`` is ``None`` or
        not convertible.
    """
    if value is None:
        return None
    try:
        return int(value)  # type: ignore[call-overload]
    except (ValueError, TypeError):
        return None


_CITES_PARAM = re.compile(r"[?&]cites=(\d+)")


def _scholar_id_of(pub: dict) -> str | None:
    """Find the Scholar cluster id of a ``scholarly`` publication.

    Search results carry it only inside their "Cited by" link
    (``citedby_url``, ``...?cites=<id>``); profile entries have ``cites_id``.
    A paper nobody cites has neither.

    Args:
        pub (dict): Publication as returned by ``scholarly``.

    Returns:
        str | None: The id, or ``None`` if the publication has none.
    """
    cites_id = pub.get("cites_id")
    if isinstance(cites_id, list) and cites_id:
        return str(cites_id[0])
    found = _CITES_PARAM.search(str(pub.get("citedby_url") or ""))
    return found.group(1) if found else None


[docs] def require_identifier(title: str | None, scholar_id: str | None) -> None: """Ensure at least one way to identify a paper was given. Args: title (str | None): Paper title. scholar_id (str | None): Scholar cluster id (``PaperResult.scholar_id``). Raises: ScholarInvalidArgumentError: If both are empty. """ if not any(v and v.strip() for v in (title, scholar_id)): raise ScholarInvalidArgumentError( "Provide scholar_id (from an earlier search result) or title. IDs are " "more reliable than titles." )
[docs] class ScholarClient: """Synchronous wrapper around ``scholarly`` that returns typed models.""" def __init__(self, config: Config) -> None: """Create a client; ``scholarly`` is configured lazily on first use. Args: config (Config): Timeout, retry, proxy, cookie and cooldown settings. """ self._config = config self._initialized = False self._detector = BlockDetector(config.block_cooldown_seconds) set_identity(config.cookies, config.user_agent) throttle.configure(config.min_delay_seconds, config.max_delay_seconds) risky = login_cookie_names(parse_cookies(config.cookies)) if risky: logger.warning( "GS_COOKIES contains Google LOGIN cookies (%s). Use cookies from a " "private window where you are NOT logged in.", ", ".join(risky), )
[docs] def ensure_available(self) -> None: """Fail fast while Scholar is known to be blocking this client. Raises: ScholarBlockedError: During the cooldown after a detected block. """ remaining = self._detector.cooldown_remaining() if remaining > 0: raise ScholarBlockedError( f"Google Scholar blocked this client recently; not retrying for " f"another {remaining / 60:.0f} min to avoid making it worse " "(GS_BLOCK_COOLDOWN). Do not retry now; tell the user." )
def _raise_if_blocked(self) -> None: """Turn a block seen during this call into :class:`ScholarBlockedError`. Raises: ScholarBlockedError: If a response was a bot-check or sign-in page. """ reason = self._detector.reason if reason is None: return self._detector.trip() if self._config.cookies: advice = ( "The configured browser cookies (GS_COOKIES / GS_COOKIES_FILE) were " "not accepted: they have probably expired. Tell the user to renew " "them with `google-scholar-mcp set-cookies` (README: 'Using " "browser cookies') and restart the server." ) else: advice = ( "Waiting, another network, browser cookies (run " "`google-scholar-mcp set-cookies`) or a JavaScript-capable " "proxy (GS_PROXY_TYPE=scraperapi) may help." ) raise ScholarBlockedError( f"Google Scholar is blocking this client ({reason}); it returns a " f"bot check instead of results. Do not retry now; tell the user. {advice}" ) def _first(self, results: Iterator[T], what: str) -> T: """Return the first item of a ``scholarly`` result iterator. Args: results (Iterator[T]): Lazy search results. what (str): Description of the lookup, used in the error message. Returns: T: The first result. Raises: ScholarBlockedError: If the empty result is a block page. ScholarNotFoundError: If the iterator is empty. A bare ``StopIteration`` must not leave a worker thread, as it cannot propagate through an ``asyncio`` future. """ try: return next(results) except StopIteration: self._raise_if_blocked() raise ScholarNotFoundError( f"No Google Scholar result for {what}. Try the exact title from a " "search result, fewer words, or a different spelling." ) from None def _ensure_initialized(self) -> None: """Apply timeout, retries and proxy to ``scholarly`` once. Returns: None: Does nothing after the first call. """ if self._initialized: return scholarly.set_timeout(self._config.timeout_seconds) scholarly.set_retries(self._config.max_retries) self._setup_proxy() self._initialized = True def _setup_proxy(self) -> None: """Install the proxy selected by ``config.proxy_type`` in ``scholarly``. Returns: None: With ``ProxyType.NONE`` nothing is changed. """ from .config import ProxyType match self._config.proxy_type: case ProxyType.FREE: pg = ProxyGenerator() pg.FreeProxies() scholarly.use_proxy(pg) case ProxyType.SINGLE: pg = ProxyGenerator() pg.SingleProxy( http=self._config.proxy_http, https=self._config.proxy_https, ) scholarly.use_proxy(pg) case ProxyType.SCRAPERAPI: pg = ProxyGenerator() pg.ScraperAPI(self._config.scraperapi_key) scholarly.use_proxy(pg) case _: pass
[docs] def search_pubs( self, query: str, max_results: int = 10, year_low: int | None = None, year_high: int | None = None, sort_by: str = "relevance", patents: bool = True, offset: int = 0, ) -> list[PaperResult]: """Search publications by keyword. Args: query (str): Search query. max_results (int): Maximum number of papers to return. year_low (int | None): Earliest publication year. year_high (int | None): Latest publication year. sort_by (str): ``"relevance"`` or ``"date"``. patents (bool): Whether to include patents. offset (int): Number of leading results to skip (for paging). Returns: list[PaperResult]: Up to ``max_results`` papers, starting at ``offset``; empty when the offset is past the last result. Raises: ScholarBlockedError: If Scholar answers with a bot check instead of results. MaxTriesExceededException: If Scholar blocks the request (raised by ``scholarly``). """ self._ensure_initialized() self._detector.reset() results = scholarly.search_pubs( query, year_low=year_low, year_high=year_high, sort_by=sort_by, patents=patents, start_index=offset, ) papers: list[PaperResult] = [] for i, pub in enumerate(results): if i >= max_results: break papers.append(self._pub_to_paper_result(pub)) if not papers: self._raise_if_blocked() return papers
def _locate_pub( self, title: str | None, scholar_id: str | None, *, fill: bool, ) -> dict: """Resolve a paper from an ID or title, preferring the ID. Args: title (str | None): Paper title; best search hit is used. scholar_id (str | None): Scholar cluster id; the first entry of the cluster (the paper's best version) is used. fill (bool): Fetch the full record. Returns: dict: ``scholarly`` publication. Raises: ScholarInvalidArgumentError: If no identifier is given. ScholarNotFoundError: If nothing matches. """ require_identifier(title, scholar_id) if scholar_id and scholar_id.strip(): cluster = scholar_id.strip() results = scholarly.search_pubs_custom_url( f"/scholar?hl=en&cluster={cluster}" ) pub = self._first(results, f"scholar_id {cluster!r}") else: results = scholarly.search_pubs(title) pub = self._first(results, f"title {title!r}") return scholarly.fill(pub) if fill else pub
[docs] def get_paper_details( self, title: str | None = None, *, scholar_id: str | None = None, ) -> PaperDetails: """Fetch full details and BibTeX of a paper. Args: title (str | None): Paper title (best search hit is used). scholar_id (str | None): Scholar cluster id from a search result. Returns: PaperDetails: Details of the identified paper. Raises: ScholarInvalidArgumentError: If no identifier is given. ScholarNotFoundError: If nothing matches. """ self._ensure_initialized() self._detector.reset() pub = self._locate_pub(title, scholar_id, fill=True) return self._pub_to_paper_details(pub)
[docs] def get_citations( self, title: str | None = None, max_results: int = 10, *, scholar_id: str | None = None, offset: int = 0, ) -> tuple[str | None, int | None, list[PaperResult]]: """Fetch papers that cite a paper. With ``scholar_id`` the cited paper is not fetched (saving a request), so its title is ``None`` and the total comes from the result page header. Args: title (str | None): Title of the cited paper. max_results (int): Maximum number of citing papers to return. scholar_id (str | None): Scholar cluster id from a search result. offset (int): Number of leading citing papers to skip (for paging). Returns: tuple[str | None, int | None, list[PaperResult]]: The cited paper's title (if known), its total citation count (if known) and the citing papers. Raises: ScholarInvalidArgumentError: If no identifier is given. ScholarNotFoundError: If the paper is not found. """ require_identifier(title, scholar_id) self._ensure_initialized() self._detector.reset() cited_title: str | None total: int | None paging = {"start_index": offset} if offset > 0 else {} if scholar_id and scholar_id.strip(): citing = scholarly.search_citedby(scholar_id.strip(), **paging) total = _safe_int(getattr(citing, "total_results", None)) cited_title = None else: pub = self._locate_pub(title, None, fill=False) total = pub.get("num_citations", 0) cited_title = pub.get("bib", {}).get("title", title) cluster = _scholar_id_of(pub) if cluster: # One plain request per page. ``scholarly.citedby`` would walk # the citations year by year for papers cited over 1000 times. citing = scholarly.search_citedby(cluster, **paging) elif offset > 0: citing = iter(()) else: citing = scholarly.citedby(pub) papers: list[PaperResult] = [] for i, cite in enumerate(citing): if i >= max_results: break papers.append(self._pub_to_paper_result(cite)) if not papers: self._raise_if_blocked() return cited_title, total, papers
[docs] def get_bibtex( self, title: str | None = None, *, scholar_id: str | None = None, ) -> str: """Get the BibTeX entry of a paper. Args: title (str | None): Paper title (best search hit is used). scholar_id (str | None): Scholar cluster id from a search result. Returns: str: BibTeX entry. Raises: ScholarInvalidArgumentError: If no identifier is given. ScholarNotFoundError: If nothing matches. """ self._ensure_initialized() self._detector.reset() pub = self._locate_pub(title, scholar_id, fill=True) return scholarly.bibtex(pub)
@staticmethod def _pub_to_paper_result(pub: dict) -> PaperResult: """Convert a ``scholarly`` publication dict to a model. Args: pub (dict): Publication as returned by ``scholarly``. Returns: PaperResult: The publication with missing fields defaulted. """ bib = pub.get("bib", {}) author_list = bib.get("author", []) if isinstance(author_list, str): author_list = [a.strip() for a in author_list.split(" and ")] scholar_id = _scholar_id_of(pub) return PaperResult( title=bib.get("title", "Unknown"), authors=author_list, year=_safe_int(bib.get("pub_year")), abstract=bib.get("abstract"), num_citations=pub.get("num_citations", 0), url=pub.get("pub_url") or pub.get("eprint_url"), pub_url=pub.get("pub_url"), venue=bib.get("venue"), scholar_id=scholar_id, ) def _pub_to_paper_details(self, pub: dict) -> PaperDetails: """Convert a filled ``scholarly`` publication dict to a detailed model. Args: pub (dict): Publication filled via ``scholarly.fill``. Returns: PaperDetails: The paper with citation history, access flag and BibTeX. """ base = self._pub_to_paper_result(pub) public_access_info = pub.get("public_access") public_access = None if isinstance(public_access_info, dict): public_access = public_access_info.get("available") elif isinstance(public_access_info, bool): public_access = public_access_info return PaperDetails( **base.model_dump(), cites_per_year=pub.get("cites_per_year", {}), public_access=public_access, bibtex=scholarly.bibtex(pub), )