"""Download one GEO series and write the plain tables the analysis reads. The read counts are NCBI's own: GEO re-processes every human RNA-seq run with one pipeline (GRCh38.p13 genome, NCBI gene annotation), so all our examples start from counts made the same way. python analyses/geo_inputs.py # also writes samples_all.tsv python analyses/geo_inputs.py # plus samples.tsv for the analysis python analyses/geo_inputs.py all # ignore the config's manual exclusions Outputs: counts.tsv gene_id (NCBI GeneID), symbol, gene_type, then one column of raw read counts per sample samples_all.tsv every sample with every characteristic GEO records for it samples.tsv the samples used, with donor and tissue (only with a config) excluded.tsv the samples left out, each with the reason (only with a config) source.json URLs and sha256 checksums of everything downloaded """ import gzip, hashlib, json, re, sys, urllib.request from pathlib import Path # Gene names: NCBI's human gene list, keyed by the same NCBI GeneIDs the counts use. GENE_INFO = "https://ftp.ncbi.nlm.nih.gov/gene/DATA/GENE_INFO/Mammalia/Homo_sapiens.gene_info.gz" def counts_url(acc): return (f"https://www.ncbi.nlm.nih.gov/geo/download/?type=rnaseq_counts&acc={acc}" f"&format=file&file={acc}_raw_counts_GRCh38.p13_NCBI.tsv.gz") def matrix_url(acc): return f"https://ftp.ncbi.nlm.nih.gov/geo/series/{acc[:-3]}nnn/{acc}/matrix/{acc}_series_matrix.txt.gz" def get(url): req = urllib.request.Request(url, headers={"User-Agent": "expressionbench-example-analysis"}) with urllib.request.urlopen(req, timeout=600) as r: return r.read() def cells(line): return [c.strip('"') for c in line.rstrip("\n").split("\t")[1:]] def parse_matrix(text): """One dict per sample: accession, title, source, and each 'key: value' characteristic.""" rows, chars = {}, [] for line in text.splitlines(): key = line.split("\t", 1)[0] if key == "!Sample_characteristics_ch1": chars.append(cells(line)) elif key in ("!Sample_geo_accession", "!Sample_title", "!Sample_source_name_ch1"): rows.setdefault(key, cells(line)) samples = [] for i, gsm in enumerate(rows["!Sample_geo_accession"]): s = {"sample": gsm, "title": rows["!Sample_title"][i], "source": rows["!Sample_source_name_ch1"][i]} for line in chars: if ":" in line[i]: k, v = line[i].split(":", 1) s[k.strip().lower()] = v.strip() samples.append(s) return samples def parse_gene_info(text): """GeneID -> symbol and GeneID -> gene type, from NCBI's Homo_sapiens.gene_info.""" symbols, types = {}, {} for line in text.splitlines(): if line.startswith("#"): continue p = line.split("\t") symbols[p[1]] = p[2] types[p[1]] = p[9] return symbols, types def apply_config(samples, cfg, keep_excluded=False): """Turn GEO's free-text labels into donor and tissue, as the dataset's config says. Returns (used, excluded). With require_pairs, a patient is kept only if they have one sample of every tissue, because the comparison is made within each patient. """ used, excluded = [], [] manual = {} if keep_excluded else cfg.get("exclude", {}) for s in samples: if s["title"] in manual: excluded.append({"sample": s["sample"], "title": s["title"], "reason": manual[s["title"]]}) continue tissue = None for label, rule in cfg["tissue"].items(): if re.search(rule["pattern"], s.get(rule["field"], ""), re.I): tissue = label break if tissue is None: excluded.append({"sample": s["sample"], "title": s["title"], "reason": cfg.get("other_reason", "not one of the compared tissues")}) continue m = re.search(cfg["donor"]["pattern"], s.get(cfg["donor"]["field"], "")) if not m: raise SystemExit(f"no donor for {s['sample']}: {s.get(cfg['donor']['field'])!r}") used.append({"sample": s["sample"], "title": s["title"], "donor": "P" + "".join(m.groups()), "tissue": tissue}) pairs = {(s["donor"], s["tissue"]) for s in used} if len(pairs) != len(used): raise SystemExit("a patient has two samples of the same tissue; check the config") if cfg.get("require_pairs"): tissues = set(cfg["tissue"]) have = {} for s in used: have.setdefault(s["donor"], set()).add(s["tissue"]) keep = [] for s in used: if have[s["donor"]] == tissues: keep.append(s) else: excluded.append({"sample": s["sample"], "title": s["title"], "reason": "no matching sample of the other tissue from the same patient"}) used = keep return used, excluded def main(acc, out, config=None, mode=""): out = Path(out) out.mkdir(parents=True, exist_ok=True) blobs = {"matrix": get(matrix_url(acc)), "counts": get(counts_url(acc)), "gene_info": get(GENE_INFO)} samples = parse_matrix(gzip.decompress(blobs["matrix"]).decode()) symbols, types = parse_gene_info(gzip.decompress(blobs["gene_info"]).decode()) lines = gzip.decompress(blobs["counts"]).decode().splitlines() header = lines[0].split("\t") if sorted(header[1:]) != sorted(s["sample"] for s in samples): raise SystemExit("the count file and the sample list name different samples") keys = [] for s in samples: keys += [k for k in s if k not in keys] with open(out / "samples_all.tsv", "w") as f: f.write("\t".join(keys) + "\n") for s in samples: f.write("\t".join(s.get(k, "") for k in keys) + "\n") with open(out / "counts.tsv", "w") as f: f.write("gene_id\tsymbol\tgene_type\t" + "\t".join(header[1:]) + "\n") for line in lines[1:]: gid, rest = line.split("\t", 1) f.write(f"{gid}\t{symbols.get(gid, gid)}\t{types.get(gid, 'unknown')}\t{rest}\n") used = None if config: cfg = json.loads(Path(config).read_text()) used, excluded = apply_config(samples, cfg, keep_excluded=(mode == "all")) with open(out / "samples.tsv", "w") as f: f.write("sample\ttitle\tdonor\ttissue\n") for s in used: f.write("\t".join(s[k] for k in ("sample", "title", "donor", "tissue")) + "\n") with open(out / "excluded.tsv", "w") as f: f.write("sample\ttitle\treason\n") for s in excluded: f.write("\t".join(s[k] for k in ("sample", "title", "reason")) + "\n") urls = {"matrix": matrix_url(acc), "counts": counts_url(acc), "gene_info": GENE_INFO} source = {"accession": acc, "genome": "GRCh38.p13", "counts": "NCBI-generated raw counts", "files": [{"url": urls[k], "sha256": hashlib.sha256(b).hexdigest()} for k, b in blobs.items()], "genes": len(lines) - 1, "samples_in_series": len(samples), "samples_used": len(used) if used is not None else None} (out / "source.json").write_text(json.dumps(source, indent=1) + "\n") print(json.dumps(source, indent=1)) print("SAMPLES_ALL") print((out / "samples_all.tsv").read_text()) if used is not None: by = {} for s in used: by.setdefault(s["tissue"], []).append(s["donor"]) for t, d in sorted(by.items()): print("TISSUE", t, len(d), " ".join(sorted(d))) if __name__ == "__main__": main(*sys.argv[1:])