pubmed-database

PubMed Database

Overview

PubMed is the U.S. National Library of Medicine's database providing free access to 36M+ biomedical citations from MEDLINE and life sciences journals. This skill covers programmatic access via the E-utilities REST API and advanced search query construction using Boolean operators, MeSH terms, and field tags.

When to Use

• Searching biomedical literature with structured Boolean/MeSH queries
• Building automated literature monitoring or extraction pipelines
• Conducting systematic literature reviews or meta-analyses
• Retrieving article metadata, abstracts, or citation information by PMID/DOI
• Finding related articles or exploring citation networks programmatically
• Batch processing large sets of PubMed records
• For Python-native PubMed access, prefer BioPython (Bio.Entrez) — this skill covers direct REST API usage
• For broader academic search (non-biomedical), use OpenAlex or Semantic Scholar APIs

Prerequisites

pip install requests  # HTTP client for E-utilities API
# Optional: pip install biopython  — Bio.Entrez wrapper (higher-level API)

API Rate Limits:

• Without API key: 3 requests/second
• With API key: 10 requests/second (register at https://www.ncbi.nlm.nih.gov/account/)
• Always include User-Agent header with contact email

Quick Start

import requests
import time

BASE_URL = "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/"
API_KEY = "YOUR_API_KEY"  # Optional but recommended

def pubmed_request(endpoint, params):
    """Reusable helper for E-utilities API calls with rate limiting."""
    params.setdefault("api_key", API_KEY)
    response = requests.get(f"{BASE_URL}{endpoint}", params=params)
    response.raise_for_status()
    time.sleep(0.1 if API_KEY != "YOUR_API_KEY" else 0.34)  # Rate limit
    return response

# Search → Fetch workflow
search_resp = pubmed_request("esearch.fcgi", {
    "db": "pubmed", "term": "CRISPR[tiab] AND 2024[dp]",
    "retmax": 5, "retmode": "json"
})
pmids = search_resp.json()["esearchresult"]["idlist"]
print(f"Found {len(pmids)} articles: {pmids}")

fetch_resp = pubmed_request("efetch.fcgi", {
    "db": "pubmed", "id": ",".join(pmids),
    "rettype": "abstract", "retmode": "text"
})
print(fetch_resp.text[:500])

Core API

1. Search Query Construction

Build PubMed queries using Boolean operators, field tags, and MeSH terms.

# Boolean operators: AND, OR, NOT (must be uppercase)
queries = {
    "basic": "diabetes AND treatment AND 2024[dp]",
    "synonyms": "(metformin OR insulin) AND type 2 diabetes",
    "exclude": "cancer NOT review[pt]",
    "phrase": '"gene expression" AND RNA-seq',
    "field_tags": "smith ja[au] AND cancer[tiab] AND 2023[dp]",
}

# Common field tags:
# [tiab] = title/abstract    [au] = author       [mh] = MeSH term
# [pt]   = publication type   [dp] = date          [ta] = journal
# [1au]  = first author       [lastau] = last author
# [affil] = affiliation       [doi] = DOI          [pmid] = PubMed ID

# Date filtering
date_queries = {
    "single_year": "cancer AND 2024[dp]",
    "range": "cancer AND 2020:2024[dp]",
    "specific": "cancer AND 2024/03/15[dp]",
}

# MeSH terms — controlled vocabulary for precise searching
mesh_queries = {
    # [mh] includes narrower terms automatically
    "broad": "diabetes mellitus[mh]",
    # [majr] limits to major topic focus
    "focused": "diabetes mellitus[majr]",
    # MeSH + subheading
    "therapy": "diabetes mellitus, type 2[mh]/drug therapy",
    # Substance name
    "drug": "metformin[nm] AND diabetes mellitus[mh]",
}

# Common MeSH subheadings:
# /diagnosis  /drug therapy  /epidemiology  /etiology
# /prevention & control  /therapy  /genetics

2. ESearch — Search and Retrieve PMIDs

# Basic search
resp = pubmed_request("esearch.fcgi", {
    "db": "pubmed",
    "term": "CRISPR[tiab] AND genome editing[tiab] AND 2024[dp]",
    "retmax": 100,
    "retmode": "json",
    "sort": "relevance",  # or "pub_date", "first_author"
})
result = resp.json()["esearchresult"]
pmids = result["idlist"]
total = result["count"]
print(f"Total hits: {total}, Retrieved: {len(pmids)}")

# With history server (for large result sets > 500)
resp = pubmed_request("esearch.fcgi", {
    "db": "pubmed",
    "term": "cancer AND 2024[dp]",
    "usehistory": "y",
    "retmode": "json",
})
result = resp.json()["esearchresult"]
webenv = result["webenv"]
query_key = result["querykey"]
total = int(result["count"])
print(f"Stored {total} results on history server")

3. EFetch — Download Full Records

# Fetch abstracts as text
resp = pubmed_request("efetch.fcgi", {
    "db": "pubmed",
    "id": ",".join(pmids[:10]),
    "rettype": "abstract",
    "retmode": "text",
})
print(resp.text[:500])

# Fetch XML for structured parsing
resp = pubmed_request("efetch.fcgi", {
    "db": "pubmed",
    "id": ",".join(pmids[:10]),
    "rettype": "xml",
    "retmode": "xml",
})

# Fetch from history server (batch processing)
batch_size = 500
for start in range(0, total, batch_size):
    resp = pubmed_request("efetch.fcgi", {
        "db": "pubmed",
        "query_key": query_key,
        "WebEnv": webenv,
        "retstart": start,
        "retmax": batch_size,
        "rettype": "xml",
        "retmode": "xml",
    })
    print(f"Fetched records {start}–{start + batch_size}")
    time.sleep(0.5)  # Extra delay for large batches

4. ESummary and ELink — Summaries and Related Articles

# ESummary — lightweight document summaries
resp = pubmed_request("esummary.fcgi", {
    "db": "pubmed",
    "id": ",".join(pmids[:5]),
    "retmode": "json",
})
for uid, data in resp.json()["result"].items():
    if uid == "uids":
        continue
    print(f"PMID {uid}: {data.get('title', '')[:80]}")
    print(f"  Journal: {data.get('fulljournalname', '')}, "
          f"Date: {data.get('pubdate', '')}")

# ELink — find related articles
resp = pubmed_request("elink.fcgi", {
    "dbfrom": "pubmed",
    "db": "pubmed",
    "id": pmids[0],
    "cmd": "neighbor",
    "retmode": "json",
})
# Links to other NCBI databases
resp = pubmed_request("elink.fcgi", {
    "dbfrom": "pubmed",
    "db": "pmc",  # PubMed Central
    "id": pmids[0],
    "retmode": "json",
})

5. Citation Matching and Identifier Lookup

# Search by identifiers
id_queries = {
    "pmid": "12345678[pmid]",
    "doi": "10.1056/NEJMoa123456[doi]",
    "pmc": "PMC123456[pmc]",
}

# ECitMatch — match partial citations to PMIDs
# Format: journal|year|volume|first_page|author_name|key|
citation = "Science|2008|320|5880|1185|key1|"
resp = pubmed_request("ecitmatch.cgi", {
    "db": "pubmed",
    "rettype": "xml",
    "bdata": citation,
})
print(f"Matched PMID: {resp.text.strip()}")

# Batch citation matching
citations = [
    "Nature|2020|580|7801|71|ref1|",
    "Science|2019|366|6463|347|ref2|",
]
resp = pubmed_request("ecitmatch.cgi", {
    "db": "pubmed",
    "rettype": "xml",
    "bdata": "\r".join(citations),
})

6. Publication Filtering

# Filter by publication type
type_filters = {
    "rcts": "randomized controlled trial[pt]",
    "reviews": "systematic review[pt]",
    "meta": "meta-analysis[pt]",
    "guidelines": "guideline[pt]",
    "case_reports": "case reports[pt]",
}

# Filter by text availability
availability = {
    "free_text": "free full text[sb]",
    "has_abstract": "hasabstract[text]",
}

# Combine filters
query = (
    "diabetes mellitus[mh] AND "
    "randomized controlled trial[pt] AND "
    "2023:2024[dp] AND "
    "free full text[sb] AND "
    "english[la]"
)
resp = pubmed_request("esearch.fcgi", {
    "db": "pubmed", "term": query, "retmax": 100, "retmode": "json"
})
print(f"Free RCTs on diabetes (2023-2024): {resp.json()['esearchresult']['count']}")

Key Concepts

E-utilities Endpoint Summary

| Endpoint | Purpose | Key Parameters | |----------|---------|----------------| | esearch.fcgi | Search, return PMIDs |