From 5c73c0d1e55c6f9cea23faeeb0f4317c9f399f1c Mon Sep 17 00:00:00 2001 From: Anton Goloborodko Date: Thu, 8 Aug 2024 10:57:32 +0200 Subject: [PATCH 1/5] add pybigtool engine to to_bigwig --- bioframe/io/fileops.py | 95 +++++++++++++++++++++++++----------------- 1 file changed, 56 insertions(+), 39 deletions(-) diff --git a/bioframe/io/fileops.py b/bioframe/io/fileops.py index 883cdd8f..f52e1b00 100644 --- a/bioframe/io/fileops.py +++ b/bioframe/io/fileops.py @@ -487,7 +487,7 @@ def read_bigbed(path, chrom, start=None, end=None, engine="auto"): return df -def to_bigwig(df, chromsizes, outpath, value_field=None, path_to_binary=None): +def to_bigwig(df, chromsizes, outpath, value_field=None, engine='kenttools', path_to_binary=None): """ Save a bedGraph-like dataframe as a binary BigWig track. @@ -505,35 +505,11 @@ def to_bigwig(df, chromsizes, outpath, value_field=None, path_to_binary=None): is to use the fourth column. path_to_binary : str, optional Provide system path to the bedGraphToBigWig binary. + engine : {'kenttools', 'pybigtools'}, optional + Engine to use for creating the BigWig file. """ - if path_to_binary is None: - cmd = "bedGraphToBigWig" - try: - assert shutil.which(cmd) is not None - except Exception: - raise ValueError( - "bedGraphToBigWig is not present in the current environment. " - "Pass it as 'path_to_binary' parameter to bioframe.to_bigwig or " - "install it with, for example, conda install -y -c bioconda " - "ucsc-bedgraphtobigwig " - ) from None - elif path_to_binary.endswith("bedGraphToBigWig"): - if not os.path.isfile(path_to_binary) and os.access(path_to_binary, os.X_OK): - raise ValueError( - f"bedGraphToBigWig is absent in the provided path or cannot be " - f"fexecuted: {path_to_binary}. " - ) - cmd = path_to_binary - else: - cmd = os.path.join(path_to_binary, "bedGraphToBigWig") - if not os.path.isfile(cmd) and os.access(cmd, os.X_OK): - raise ValueError( - f"bedGraphToBigWig is absent in the provided path or cannot be " - f"executed: {path_to_binary}. " - ) - is_bedgraph = True for col in ["chrom", "start", "end"]: if col not in df.columns: @@ -552,21 +528,62 @@ def to_bigwig(df, chromsizes, outpath, value_field=None, path_to_binary=None): bg["chrom"] = bg["chrom"].astype(str) bg = bg.sort_values(["chrom", "start", "end"]) - with tempfile.NamedTemporaryFile(suffix=".bg") as f, \ - tempfile.NamedTemporaryFile("wt", suffix=".chrom.sizes") as cs: # fmt: skip - chromsizes.to_csv(cs, sep="\t", header=False) - cs.flush() + if chromsizes is None: + chromsizes = df.groupby('chrom')['end'] + + if engine.lower() == 'kenttools': + if path_to_binary is None: + cmd = "bedGraphToBigWig" + try: + assert shutil.which(cmd) is not None + except Exception: + raise ValueError( + "bedGraphToBigWig is not present in the current environment. " + "Pass it as 'path_to_binary' parameter to bioframe.to_bigwig or " + "install it with, for example, conda install -y -c bioconda " + "ucsc-bedgraphtobigwig " + ) from None + elif path_to_binary.endswith("bedGraphToBigWig"): + if not os.path.isfile(path_to_binary) and os.access(path_to_binary, os.X_OK): + raise ValueError( + f"bedGraphToBigWig is absent in the provided path or cannot be " + f"fexecuted: {path_to_binary}. " + ) + cmd = path_to_binary + else: + cmd = os.path.join(path_to_binary, "bedGraphToBigWig") + if not os.path.isfile(cmd) and os.access(cmd, os.X_OK): + raise ValueError( + f"bedGraphToBigWig is absent in the provided path or cannot be " + f"executed: {path_to_binary}. " + ) - bg.to_csv( - f.name, sep="\t", columns=columns, index=False, header=False, na_rep="nan" - ) + with tempfile.NamedTemporaryFile(suffix=".bg") as f, \ + tempfile.NamedTemporaryFile("wt", suffix=".chrom.sizes") as cs: # fmt: skip + chromsizes.to_csv(cs, sep="\t", header=False) + cs.flush() - p = subprocess.run( - [cmd, f.name, cs.name, outpath], - capture_output=True, - ) - return p + bg.to_csv( + f.name, sep="\t", columns=columns, index=False, header=False, na_rep="nan" + ) + + p = subprocess.run( + [cmd, f.name, cs.name, outpath], + capture_output=True, + ) + return p + + elif engine.lower() == 'pybigtools': + import pybigtools + + f = pybigtools.open(outpath, "w") + if issubclass(type(chromsizes), pd.Series): + chromsizes = chromsizes.astype(int).to_dict() + bg = bg.astype({'chrom':str, "start": int, "end": int, value_field: float}) + f.write(chroms=chromsizes, vals=bg.itertuples(index=False)) + f.close() + def to_bigbed(df, chromsizes, outpath, schema="bed6", path_to_binary=None): """ From 51863f90431da9809f9172f3577204449c62764a Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 8 Aug 2024 08:58:30 +0000 Subject: [PATCH 2/5] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- bioframe/io/fileops.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bioframe/io/fileops.py b/bioframe/io/fileops.py index f52e1b00..f98a05e1 100644 --- a/bioframe/io/fileops.py +++ b/bioframe/io/fileops.py @@ -583,7 +583,7 @@ def to_bigwig(df, chromsizes, outpath, value_field=None, engine='kenttools', pat bg = bg.astype({'chrom':str, "start": int, "end": int, value_field: float}) f.write(chroms=chromsizes, vals=bg.itertuples(index=False)) f.close() - + def to_bigbed(df, chromsizes, outpath, schema="bed6", path_to_binary=None): """ From 43bda60bd33ba4b4ac3bb980fadb7d55b184e40a Mon Sep 17 00:00:00 2001 From: Anton Goloborodko Date: Fri, 9 Aug 2024 10:09:09 +0200 Subject: [PATCH 3/5] rename engines --- bioframe/io/fileops.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bioframe/io/fileops.py b/bioframe/io/fileops.py index f98a05e1..fb5aeea5 100644 --- a/bioframe/io/fileops.py +++ b/bioframe/io/fileops.py @@ -487,7 +487,7 @@ def read_bigbed(path, chrom, start=None, end=None, engine="auto"): return df -def to_bigwig(df, chromsizes, outpath, value_field=None, engine='kenttools', path_to_binary=None): +def to_bigwig(df, chromsizes, outpath, value_field=None, engine='ucsc', path_to_binary=None): """ Save a bedGraph-like dataframe as a binary BigWig track. @@ -505,7 +505,7 @@ def to_bigwig(df, chromsizes, outpath, value_field=None, engine='kenttools', pat is to use the fourth column. path_to_binary : str, optional Provide system path to the bedGraphToBigWig binary. - engine : {'kenttools', 'pybigtools'}, optional + engine : {'ucsc', 'bigtools'}, optional Engine to use for creating the BigWig file. """ @@ -531,7 +531,7 @@ def to_bigwig(df, chromsizes, outpath, value_field=None, engine='kenttools', pat if chromsizes is None: chromsizes = df.groupby('chrom')['end'] - if engine.lower() == 'kenttools': + if engine.lower() == 'ucsc': if path_to_binary is None: cmd = "bedGraphToBigWig" try: @@ -573,7 +573,7 @@ def to_bigwig(df, chromsizes, outpath, value_field=None, engine='kenttools', pat ) return p - elif engine.lower() == 'pybigtools': + elif engine.lower() == 'bigtools': import pybigtools f = pybigtools.open(outpath, "w") From 1eebe94ff2d16cd3432e4d6971bbac53cf71aae5 Mon Sep 17 00:00:00 2001 From: Nezar Abdennur Date: Fri, 18 Oct 2024 18:10:04 -0400 Subject: [PATCH 4/5] Add bigtools bigbed writer --- bioframe/io/bed.py | 95 +++++++++++------- bioframe/io/fileops.py | 215 +++++++++++++++++++++-------------------- 2 files changed, 169 insertions(+), 141 deletions(-) diff --git a/bioframe/io/bed.py b/bioframe/io/bed.py index 79e7525b..06d9d1a8 100644 --- a/bioframe/io/bed.py +++ b/bioframe/io/bed.py @@ -597,50 +597,15 @@ def parse_bed_schema(schema: str) -> tuple[int, bool]: return n, extended -def to_bed( +def to_bed_dataframe( df: pd.DataFrame, - path: str | pathlib.Path | None = None, - *, schema: str = "infer", validate_fields: bool = True, require_sorted: bool = False, chromsizes: dict | pd.Series | None = None, strict_score: bool = False, replace_na: bool = True, - na_rep: str = "nan", -) -> str | None: - """Write a DataFrame to a BED file. - - Parameters - ---------- - df : pd.DataFrame - DataFrame to write. - path : str or Path, optional - Path to write the BED file to. If ``None``, the serialized BED file is - returned as a string. - schema : str, optional [default: "infer"] - BED schema to use. If ``"infer"``, the schema is inferred from the - DataFrame's columns. - validate_fields : bool, optional [default: True] - Whether to validate the fields of the BED file. - require_sorted : bool, optional [default: False] - Whether to require the BED file to be sorted. - chromsizes : dict or pd.Series, optional - Chromosome sizes to validate against. - strict_score : bool, optional [default: False] - Whether to strictly enforce validation of the score field (0-1000). - replace_na : bool, optional [default: True] - Whether to replace null values of standard BED fields with - compliant uninformative values. - na_rep : str, optional [default: "nan"] - String representation of null values if written. - - Returns - ------- - str or None: - The serialized BED file as a string if ``path`` is ``None``, otherwise - ``None``. - """ +) -> pd.DataFrame: if schema == "infer": n, extended = infer_bed_schema(df) else: @@ -712,4 +677,60 @@ def to_bed( if col in custom_cols: bed[col] = df[col] + return bed + + +def to_bed( + df: pd.DataFrame, + path: str | pathlib.Path | None = None, + *, + schema: str = "infer", + validate_fields: bool = True, + require_sorted: bool = False, + chromsizes: dict | pd.Series | None = None, + strict_score: bool = False, + replace_na: bool = True, + na_rep: str = "nan", +) -> str | None: + """Write a DataFrame to a BED file. + + Parameters + ---------- + df : pd.DataFrame + DataFrame to write. + path : str or Path, optional + Path to write the BED file to. If ``None``, the serialized BED file is + returned as a string. + schema : str, optional [default: "infer"] + BED schema to use. If ``"infer"``, the schema is inferred from the + DataFrame's columns. + validate_fields : bool, optional [default: True] + Whether to validate the fields of the BED file. + require_sorted : bool, optional [default: False] + Whether to require the BED file to be sorted. + chromsizes : dict or pd.Series, optional + Chromosome sizes to validate against. + strict_score : bool, optional [default: False] + Whether to strictly enforce validation of the score field (0-1000). + replace_na : bool, optional [default: True] + Whether to replace null values of standard BED fields with + compliant uninformative values. + na_rep : str, optional [default: "nan"] + String representation of null values if written. + + Returns + ------- + str or None: + The serialized BED file as a string if ``path`` is ``None``, otherwise + ``None``. + """ + bed = to_bed_dataframe( + df, + schema=schema, + validate_fields=validate_fields, + require_sorted=require_sorted, + chromsizes=chromsizes, + strict_score=strict_score, + replace_na=replace_na, + ) return bed.to_csv(path, sep="\t", na_rep=na_rep, index=False, header=False) diff --git a/bioframe/io/fileops.py b/bioframe/io/fileops.py index fb5aeea5..1f4b35cd 100644 --- a/bioframe/io/fileops.py +++ b/bioframe/io/fileops.py @@ -245,22 +245,21 @@ def read_alignments(fp, chrom=None, start=None, end=None): raise ImportError("pysam is required to use `read_alignments`") from None ext = os.path.splitext(fp)[1] - if ext == '.sam': - mode = 'r' - elif ext == '.bam': - mode = 'rb' - elif ext == '.cram': - mode = 'rc' + if ext == ".sam": + mode = "r" + elif ext == ".bam": + mode = "rb" + elif ext == ".cram": + mode = "rc" else: - raise ValueError(f'{ext} is not a supported filetype') + raise ValueError(f"{ext} is not a supported filetype") with closing(pysam.AlignmentFile(fp, mode)) as f: records = [] for s in f.fetch(chrom, start, end): # Needed because array.array is not json serializable tags = [ - (k, v.tolist() if isinstance(v, array.array) else v) - for k, v in s.tags + (k, v.tolist() if isinstance(v, array.array) else v) for k, v in s.tags ] records.append( ( @@ -487,9 +486,38 @@ def read_bigbed(path, chrom, start=None, end=None, engine="auto"): return df -def to_bigwig(df, chromsizes, outpath, value_field=None, engine='ucsc', path_to_binary=None): - """ - Save a bedGraph-like dataframe as a binary BigWig track. +def _find_ucsc_binary(path, cmd): + if path is None: + try: + assert shutil.which(cmd) is not None + except Exception: + raise ValueError( + f"{cmd} is not present in the current environment. " + f"Pass it as 'path_to_binary' parameter to bioframe.to_bigwig or " + f"install it with, for example, conda install -y -c bioconda " + f"ucsc-{cmd.lower()} " + ) from None + elif path.endswith(cmd): + if not os.path.isfile(path) and os.access(path, os.X_OK): + raise ValueError( + f"{cmd} is absent in the provided path or cannot be " + f"executed: {path}. " + ) + cmd = path + else: + cmd = os.path.join(path, cmd) + if not os.path.isfile(cmd) and os.access(cmd, os.X_OK): + raise ValueError( + f"{cmd} is absent in the provided path or cannot be " + f"executed: {path}. " + ) + return cmd + + +def to_bigwig( + df, chromsizes, outpath, value_field=None, engine="ucsc", path_to_binary=None +): + """Save a bedGraph-like dataframe as a binary BigWig file. Parameters ---------- @@ -507,7 +535,6 @@ def to_bigwig(df, chromsizes, outpath, value_field=None, engine='ucsc', path_to_ Provide system path to the bedGraphToBigWig binary. engine : {'ucsc', 'bigtools'}, optional Engine to use for creating the BigWig file. - """ is_bedgraph = True @@ -528,43 +555,21 @@ def to_bigwig(df, chromsizes, outpath, value_field=None, engine='ucsc', path_to_ bg["chrom"] = bg["chrom"].astype(str) bg = bg.sort_values(["chrom", "start", "end"]) - if chromsizes is None: - chromsizes = df.groupby('chrom')['end'] - - if engine.lower() == 'ucsc': - if path_to_binary is None: - cmd = "bedGraphToBigWig" - try: - assert shutil.which(cmd) is not None - except Exception: - raise ValueError( - "bedGraphToBigWig is not present in the current environment. " - "Pass it as 'path_to_binary' parameter to bioframe.to_bigwig or " - "install it with, for example, conda install -y -c bioconda " - "ucsc-bedgraphtobigwig " - ) from None - elif path_to_binary.endswith("bedGraphToBigWig"): - if not os.path.isfile(path_to_binary) and os.access(path_to_binary, os.X_OK): - raise ValueError( - f"bedGraphToBigWig is absent in the provided path or cannot be " - f"fexecuted: {path_to_binary}. " - ) - cmd = path_to_binary - else: - cmd = os.path.join(path_to_binary, "bedGraphToBigWig") - if not os.path.isfile(cmd) and os.access(cmd, os.X_OK): - raise ValueError( - f"bedGraphToBigWig is absent in the provided path or cannot be " - f"executed: {path_to_binary}. " - ) + if engine.lower() == "ucsc": + cmd = _find_ucsc_binary(path_to_binary, "bedGraphToBigWig") with tempfile.NamedTemporaryFile(suffix=".bg") as f, \ - tempfile.NamedTemporaryFile("wt", suffix=".chrom.sizes") as cs: # fmt: skip - chromsizes.to_csv(cs, sep="\t", header=False) + tempfile.NamedTemporaryFile("wt", suffix=".chrom.sizes") as cs: # fmt: skip # noqa: E501 + pd.Series(chromsizes).to_csv(cs, sep="\t", header=False) cs.flush() bg.to_csv( - f.name, sep="\t", columns=columns, index=False, header=False, na_rep="nan" + f.name, + sep="\t", + columns=columns, + index=False, + header=False, + na_rep="nan", ) p = subprocess.run( @@ -573,21 +578,27 @@ def to_bigwig(df, chromsizes, outpath, value_field=None, engine='ucsc', path_to_ ) return p - elif engine.lower() == 'bigtools': - import pybigtools + elif engine.lower() == "bigtools": + try: + import pybigtools + except ImportError: + raise ImportError( + "pybigtools is required to use engine='bigtools'" + ) from None f = pybigtools.open(outpath, "w") if issubclass(type(chromsizes), pd.Series): chromsizes = chromsizes.astype(int).to_dict() - bg = bg.astype({'chrom':str, "start": int, "end": int, value_field: float}) + bg = bg.astype({"chrom": str, "start": int, "end": int, value_field: float}) f.write(chroms=chromsizes, vals=bg.itertuples(index=False)) f.close() -def to_bigbed(df, chromsizes, outpath, schema="bed6", path_to_binary=None): - """ - Save a bedGraph-like dataframe as a binary BigWig track. +def to_bigbed( + df, chromsizes, outpath, schema="infer", engine="ucsc", path_to_binary=None +): + """Save a BED-like dataframe as a binary BigBed file. Parameters ---------- @@ -602,63 +613,59 @@ def to_bigbed(df, chromsizes, outpath, schema="bed6", path_to_binary=None): Select the column label of the data frame to generate the track. Default is to use the fourth column. path_to_binary : str, optional - Provide system path to the bedGraphToBigWig binary. - + Provide system path to the bedToBigBed binary. """ + from bioframe.io.bed import infer_bed_schema, parse_bed_schema, to_bed_dataframe - if path_to_binary is None: - cmd = "bedToBigBed" - try: - assert shutil.which(cmd) is not None - except Exception: - raise ValueError( - "bedToBigBed is not present in the current environment. " - "Pass it as 'path_to_binary' parameter to bioframe.to_bigbed or " - "install it with, for example, conda install -y -c bioconda " - "ucsc-bedtobigbed " - ) from None - elif path_to_binary.endswith("bedToBigBed"): - if not os.path.isfile(path_to_binary) and os.access(path_to_binary, os.X_OK): - raise ValueError( - f"bedToBigBed is absent in the provided path or cannot be " - f"executed: {path_to_binary}. " - ) - cmd = path_to_binary + if schema == "infer": + n, _ = infer_bed_schema(df) else: - cmd = os.path.join(path_to_binary, "bedGraphToBigWig") - if not os.path.isfile(cmd) and os.access(cmd, os.X_OK): - raise ValueError( - f"bedToBigBed is absent in the provided path or cannot be " - f"executed: {path_to_binary}. " + n, _ = parse_bed_schema(schema) + + bed = to_bed_dataframe(df, schema=schema) + m = len(bed.columns) - n + schema = f"bed{n}+{m}" if m > 0 else f"bed{n}" + + if engine.lower() == "ucsc": + if path_to_binary is None: + cmd = _find_ucsc_binary(path_to_binary, "bedToBigBed") + + with tempfile.NamedTemporaryFile(suffix=".bed") as f, \ + tempfile.NamedTemporaryFile("wt", suffix=".chrom.sizes") as cs: # fmt: skip # noqa: E501 + pd.Series(chromsizes).to_csv(cs, sep="\t", header=False) + cs.flush() + + bed.to_csv( + f.name, + sep="\t", + columns=bed.columns, + index=False, + header=False, + na_rep="nan", ) - is_bed6 = True - for col in ["chrom", "start", "end", "name", "score", "strand"]: - if col not in df.columns: - is_bed6 = False - if len(df.columns) < 6: - is_bed6 = False - - if not is_bed6: - raise ValueError(f"A bed6-like DataFrame is required, got {df.columns}") - - columns = ["chrom", "start", "end", "name", "score", "strand"] - bed = df[columns].copy() - bed["chrom"] = bed["chrom"].astype(str) - bed = bed.sort_values(["chrom", "start", "end"]) - - with tempfile.NamedTemporaryFile(suffix=".bed") as f, tempfile.NamedTemporaryFile( - "wt", suffix=".chrom.sizes" - ) as cs: - chromsizes.to_csv(cs, sep="\t", header=False) - cs.flush() - - bed.to_csv( - f.name, sep="\t", columns=columns, index=False, header=False, na_rep="nan" - ) + p = subprocess.run( + [cmd, f"-type={schema}", f.name, cs.name, outpath], + capture_output=True, + ) + return p - p = subprocess.run( - [cmd, f"-type={schema}", f.name, cs.name, outpath], - capture_output=True, + elif engine.lower() == "bigtools": + try: + import pybigtools + except ImportError: + raise ImportError( + "pybigtools is required to use engine='bigtools'" + ) from None + + f = pybigtools.open(outpath, "w") + if issubclass(type(chromsizes), pd.Series): + chromsizes = chromsizes.astype(int).to_dict() + + bed = bed.astype({"chrom": str, "start": int, "end": int}) + record_iter = ( + (row[0], row[1], row[2], "\t".join(str(x) for x in row[3:])) + for row in bed.itertuples(index=False) ) - return p + f.write(chroms=chromsizes, vals=record_iter) + f.close() From ee8ce0ca3b210cf8bec95d4b3fc090ee57f578e7 Mon Sep 17 00:00:00 2001 From: Nezar Abdennur Date: Fri, 18 Oct 2024 18:17:37 -0400 Subject: [PATCH 5/5] Remove py38 from tests for lack of numpy 2.0 support --- .github/workflows/ci.yml | 2 +- pyproject.toml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 78449d27..312e2b3e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,7 +17,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [ "3.8", "3.9", "3.10", "3.11" ] + python-version: [ "3.9", "3.10", "3.11", "3.12" ] steps: - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} diff --git a/pyproject.toml b/pyproject.toml index e3a57a84..bb9dbbb7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,11 +28,11 @@ classifiers = [ "Operating System :: OS Independent", "Programming Language :: Python", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", ] readme = "README.md" requires-python = ">=3.8" @@ -126,4 +126,4 @@ docs = "sphinx-autobuild docs docs/_build/html" features = ["dev", "test"] [[tool.hatch.envs.test.matrix]] -python = ["3.8", "3.9", "3.10", "3.11"] +python = ["3.9", "3.10", "3.11", "3.12"]