From 4f39341c13bf820bc7b599033d2cb85745448316 Mon Sep 17 00:00:00 2001 From: wpk Date: Wed, 1 Nov 2023 15:35:34 -0400 Subject: [PATCH 01/36] chore: update pyproject.toml --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 69bbfb1..9f452d1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -288,7 +288,7 @@ ignore = [ dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$" [tool.ruff.per-file-ignores] -"src/pyproject2conda.py" = ["UP006", "UP007"] +"src/pyproject2conda/cli.py" = ["UP006", "UP007"] [tool.ruff.isort] known-first-party = ["pyproject2conda"] From 958e8cb676ef3c24ffb9ebbb562dca128cf821cd Mon Sep 17 00:00:00 2001 From: wpk Date: Wed, 1 Nov 2023 16:05:07 -0400 Subject: [PATCH 02/36] chore: update cruft --- .cruft.json | 2 +- .pre-commit-config.yaml | 20 +++---- CHANGELOG.md | 7 ++- CONTRIBUTING.md | 13 +++-- Makefile | 10 +++- docs/conf.py | 8 ++- noxfile.py | 105 +++++++++++++++++++++++++++------- pyproject.toml | 49 ++++------------ src/pyproject2conda/parser.py | 4 +- tests/test_parser.py | 79 +++++++++++++++++++++++++ tools/create_pythons.py | 3 + 11 files changed, 214 insertions(+), 86 deletions(-) diff --git a/.cruft.json b/.cruft.json index 4c1d0fc..4a3e774 100644 --- a/.cruft.json +++ b/.cruft.json @@ -1,6 +1,6 @@ { "template": "https://github.com/usnistgov/cookiecutter-nist-python.git", - "commit": "7ba78569cba4c2b39c2706d1f95b74c919b310cf", + "commit": "41b5d67426f708650af617193e42e081fe965649", "checkout": "develop", "context": { "cookiecutter": { diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5b38489..a16a15c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -9,7 +9,7 @@ default_install_hook_types: repos: # * Top level - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.4.0 + rev: v4.5.0 hooks: - id: trailing-whitespace - id: end-of-file-fixer @@ -39,7 +39,7 @@ repos: # * Markdown - repo: https://github.com/DavidAnson/markdownlint-cli2 - rev: v0.9.2 + rev: v0.10.0 hooks: - id: markdownlint-cli2 args: ["--style prettier"] @@ -47,11 +47,11 @@ repos: # * Linting - repo: https://github.com/charliermarsh/ruff-pre-commit # Ruff version. - rev: "v0.0.289" + rev: "v0.1.3" hooks: - id: ruff - repo: https://github.com/psf/black - rev: 23.9.1 + rev: 23.10.1 hooks: # NOTE: nbQA for notebook formatting - id: black @@ -60,19 +60,19 @@ repos: hooks: - id: blacken-docs additional_dependencies: - - black==23.9.1 + - black==23.10.1 # exclude: ^README.md - repo: https://github.com/nbQA-dev/nbQA rev: 1.7.0 hooks: - id: nbqa-ruff - additional_dependencies: [ruff==0.0.289] + additional_dependencies: [ruff==0.1.3] - id: nbqa-black - additional_dependencies: [black==23.9.1] + additional_dependencies: [black==23.10.1] # * Commit message - repo: https://github.com/commitizen-tools/commitizen - rev: 3.9.0 + rev: 3.12.0 hooks: - id: commitizen stages: [commit-msg] @@ -85,7 +85,7 @@ repos: - id: isort stages: [manual] - repo: https://github.com/asottile/pyupgrade - rev: v3.10.1 + rev: v3.15.0 hooks: - id: pyupgrade stages: [manual] @@ -113,7 +113,7 @@ repos: stages: [manual] # ** spelling - repo: https://github.com/codespell-project/codespell - rev: v2.2.5 + rev: v2.2.6 hooks: - id: codespell types_or: [python, rst, markdown, cython, c] diff --git a/CHANGELOG.md b/CHANGELOG.md index 844cdea..bfcef24 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,6 @@ + + # Changelog @@ -6,8 +8,9 @@ Changelog for `pyproject2conda` ## Unreleased -See the fragment files in -[changelog.d](https://github.com/usnistgov/pyproject2conda) +[changelog.d]: https://github.com/usnistgov/pyproject2conda + +See the fragment files in [changelog.d] diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e1a4b7f..da0f48f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -7,7 +7,9 @@ You can contribute in many ways: ## Types of Contributions + [issues]: https://github.com/usnistgov/pyproject2conda/issues + ### Report Bugs @@ -31,9 +33,9 @@ and "help wanted" is open to whoever wants to implement it. ### Write Documentation -`pyproject2conda` could always use more documentation, whether as part of the -official `pyproject2conda` docs, in docstrings, or even on the web in blog -posts, articles, and such. +This project could always use more documentation, whether as part of the +official docs, in docstrings, or even on the web in blog posts, articles, and +such. ### Submit Feedback @@ -48,10 +50,9 @@ If you are proposing a feature: ## Making a contribution -Ready to contribute? Here's how to set up `pyproject2conda` for local -development. +Ready to contribute? Here's how to make a contribution. -- Fork the `pyproject2conda` repo on GitHub. +- Fork the repo on GitHub. - Clone your fork locally: diff --git a/Makefile b/Makefile index 4ff8a3e..d05ad2a 100644 --- a/Makefile +++ b/Makefile @@ -141,6 +141,9 @@ version-scm: ## check/update version of package with setuptools-scm version-import: ## check version from python import -python -c 'import pyproject2conda; print(pyproject2conda.__version__)' +version-update: ## update version using nox + nox -s update-version-scm + version: version-scm version-import ################################################################################ @@ -149,9 +152,10 @@ version: version-scm version-import .PHONY: requirements requirements: ## rebuild all requirements/environment files nox -s requirements - -requirements/%.yaml: pyproject.toml requirements -requirements/%.txt: pyproject.toml requirements +requirements/%.yaml: pyproject.toml + nox -s requirements +requirements/%.txt: pyproject.toml + nox -s requirements ################################################################################ # * NOX diff --git a/docs/conf.py b/docs/conf.py index 7d6c757..2f5b230 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -58,7 +58,6 @@ # "sphinx_design" ## myst stuff "myst_nb", - # project specific "sphinx_click", ] @@ -114,7 +113,10 @@ # nb_execution_mode = "auto" # set the kernel name -nb_kernel_rgx_aliases = {"pyproject2conda.*": "python3", "conda.*": "python3"} +nb_kernel_rgx_aliases = { + "pyproject2conda.*": "python3", + "conda.*": "python3", +} nb_execution_allow_errors = True @@ -478,7 +480,9 @@ def linkcode_resolve(domain, info): else: linespec = "" + # fmt: off fn = os.path.relpath(fn, start=os.path.dirname(pyproject2conda.__file__)) + # fmt: on return f"https://github.com/{github_username}/pyproject2conda/blob/{html_context['github_version']}/src/pyproject2conda/{fn}{linespec}" diff --git a/noxfile.py b/noxfile.py index 1cbd226..5a487f9 100644 --- a/noxfile.py +++ b/noxfile.py @@ -4,21 +4,34 @@ import shutil import sys + +# Should only use on python version > 3.10 +assert sys.version_info >= (3, 10) + from dataclasses import replace # noqa from pathlib import Path from typing import ( - Annotated, Any, Callable, Iterator, Sequence, - TypeAlias, TypeVar, cast, ) -import nox -from noxopt import NoxOpt, Option, Session +if sys.version_info > (3, 10): + from typing import TypeAlias +else: + from typing_extensions import TypeAlias + +if sys.version_info > (3, 9): + from typing import Annotated +else: + from typing_extensions import Annotated + + +import nox # type: ignore[unused-ignore,import] +from noxopt import NoxOpt, Option, Session # type: ignore[unused-ignore,import] # fmt: off sys.path.insert(0, ".") @@ -88,6 +101,9 @@ DEFAULT_SESSION_VENV = cast(C[F], group.session(python=PYTHON_DEFAULT_VERSION)) # type: ignore ALL_SESSION_VENV = cast(C[F], group.session(python=PYTHON_ALL_VERSIONS)) # type: ignore +NOPYTHON_SESSION = cast(C[F], group.session(python=False)) # type: ignore +INHERITED_SESSION_VENV = cast(C[F], group.session) # type: ignore + OPTS_OPT = Option(nargs="*", type=str) # SET_KERNEL_OPT = Option(type=bool, help="If True, try to set the kernel name") RUN_OPT = Option( @@ -101,20 +117,20 @@ LOCK_OPT = Option(type=bool, help="If True, use conda-lock") -def opts_annotated(**kwargs: Any): # type: ignore - return Annotated[list[str], replace(OPTS_OPT, **kwargs)] +def opts_annotated(**kwargs: Any): # type: ignore[unused-ignore,no-untyped-def] + return Annotated["list[str]", replace(OPTS_OPT, **kwargs)] -def cmd_annotated(**kwargs: Any): # type: ignore - return Annotated[list[str], replace(CMD_OPT, **kwargs)] +def cmd_annotated(**kwargs: Any): # type: ignore[unused-ignore,no-untyped-def] + return Annotated["list[str]", replace(CMD_OPT, **kwargs)] -def run_annotated(**kwargs: Any): # type: ignore - return Annotated[list[list[str]], replace(RUN_OPT, **kwargs)] +def run_annotated(**kwargs: Any): # type: ignore[unused-ignore,no-untyped-def] + return Annotated["list[list[str]]", replace(RUN_OPT, **kwargs)] LOCK_CLI = Annotated[bool, LOCK_OPT] -RUN_CLI = Annotated[list[list[str]], RUN_OPT] +RUN_CLI = Annotated["list[list[str]]", RUN_OPT] TEST_OPTS_CLI = opts_annotated(help="extra arguments/flags to pytest") DEV_EXTRAS_CLI = cmd_annotated(help="extras included in user dev environment") PYTHON_PATHS_CLI = cmd_annotated(help="python paths to append to PATHS") @@ -192,8 +208,8 @@ def dev_venv( # ** bootstrap -@group.session(python=False) # type: ignore -def bootstrap(session: Session): +@NOPYTHON_SESSION +def bootstrap(session: Session) -> None: """Run config, reqs, and dev""" session.notify("config") @@ -202,7 +218,7 @@ def bootstrap(session: Session): # ** config -@group.session(python=False) # type: ignore +@NOPYTHON_SESSION def config( session: Session, dev_extras: DEV_EXTRAS_CLI = [], # type: ignore # noqa @@ -220,7 +236,7 @@ def config( # ** requirements -@group.session(python=False) # type: ignore +@NOPYTHON_SESSION def pyproject2conda( session: Session, update: UPDATE_CLI = False, @@ -229,7 +245,7 @@ def pyproject2conda( session.notify("requirements") -@group.session +@INHERITED_SESSION_VENV def requirements( session: Session, update: UPDATE_CLI = False, @@ -243,7 +259,7 @@ def requirements( """ pkg_install_venv( session=session, - reqs=["."], + reqs=["pyproject2conda>=0.8.0"], name="reqs", update=update, log_session=log_session, @@ -423,7 +439,7 @@ def _coverage( session_run_commands(session, run) if not cmd and not run and not run_internal: - cmd = ["combine", "report"] + cmd = ["combine", "html"] session.log(f"{cmd}") @@ -755,13 +771,13 @@ def dist_conda( elif command == "recipe-cat-full": import tempfile - with tempfile.TemporaryDirectory() as d: + with tempfile.TemporaryDirectory() as d: # type: ignore[assignment,unused-ignore] session.run( "grayskull", "pypi", sdist_path, "-o", - d, + str(d), ) session.run( "cat", str(Path(d) / PACKAGE_NAME / "meta.yaml"), external=True @@ -778,7 +794,7 @@ def dist_conda( # ** lint -@group.session +@INHERITED_SESSION_VENV def lint( session: nox.Session, lint_run: RUN_CLI = [], # noqa @@ -828,6 +844,15 @@ def _run_info(cmd: str) -> None: session.run("which", cmd, external=True) session.run(cmd, "--version", external=True) + if "clean" in cmd: + cmd = [x for x in cmd if x != "clean"] + + for name in [".mypy_cache", ".pytype"]: + p = Path(session.create_tmp()) / name + if p.exists(): + session.log(f"removing cache {p}") + shutil.rmtree(str(p)) + for c in cmd: if not c.startswith("nbqa"): _run_info(c) @@ -849,6 +874,7 @@ def typing( session: nox.Session, typing_cmd: cmd_annotated( # type: ignore choices=[ + "clean", "mypy", "pyright", "pytype", @@ -891,6 +917,7 @@ def typing_venv( session: nox.Session, typing_cmd: cmd_annotated( # type: ignore choices=[ + "clean", "mypy", "pyright", "pytype", @@ -1046,6 +1073,42 @@ def testdist_pypi_condaenv( ) +@DEFAULT_SESSION_VENV +def update_version_scm( + session: Session, + version: VERSION_CLI = "", + update: UPDATE_CLI = False, +) -> None: + """ + Get current version from setuptools-scm + + Note that the version of editable installs can get stale. + This will show the actual current version. + Avoids need to include setuptools-scm in develop/docs/etc. + """ + + if version: + session.env["SETUPTOOLS_SCM_PRETEND_VERSION"] = version + + pkg_install_venv( + session=session, + name="update-version-scm", + install_package=True, + # reqs=["setuptools_scm"], + update=True, + no_deps=True, + ) + + session.run( + "python", + "-c", + "import sys;" + "sys.path.insert(0, 'src');" + "from pyproject2conda._version import __version__;" + "print(__version__);", + ) + + # * Utilities -------------------------------------------------------------------------- def _create_doc_examples_symlinks(session: nox.Session, clean: bool = True) -> None: """Create symlinks from docs/examples/*.md files to /examples/usage/...""" diff --git a/pyproject.toml b/pyproject.toml index 9f452d1..75efcd2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["setuptools>=61.2", "setuptools_scm[toml]>=7.0"] +requires = ["setuptools>=61.2", "setuptools_scm[toml]>=8.0"] build-backend = "setuptools.build_meta" [project] @@ -20,29 +20,18 @@ classifiers = [ "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", "Topic :: Scientific/Engineering", ] dynamic = ["readme", "version"] requires-python = ">=3.8" -dependencies = [ - "tomli", - "ruamel.yaml", - "tomlkit", - "typer", - # "rich-click", - "packaging", - "typing-extensions; python_version<'3.9'", -] # additional packages +dependencies = ["typer"] [project.urls] homepage = "https://github.com/usnistgov/pyproject2conda" documentation = "https://pages.nist.gov/pyproject2conda/" [project.optional-dependencies] -all = [ - "rich", # - "shellingham", -] test = [ "pytest", # "pytest-xdist", @@ -50,17 +39,15 @@ test = [ "pytest-sugar", ] dev-extras = [ - "setuptools-scm", # + "setuptools-scm>=8.0", # "pytest-accept", # p2c: -p + # "nbval", "ipython", "ipykernel", - # "nbval", - ] typing-extras = [ "pytype; python_version < '3.11'", # "mypy >= 1.4.1", - "types-click", ] typing = [ "pyproject2conda[typing-extras]", # @@ -99,7 +86,6 @@ docs = [ "myst-nb", "sphinx-book-theme", "autodocsumm", - # project specific "sphinx-click", ] # to be parsed with pyproject2conda with --no-base option @@ -149,8 +135,7 @@ envs = ["dist-conda"] style = ["yaml"] [project.scripts] -pyproject2conda = "pyproject2conda.cli:app" -p2c = "pyproject2conda.cli:app" +pyproject2conda = "pyproject2conda.cli:main" ## grayskull still messes some things up, but use scripts/recipe-append.sh for this [tool.setuptools] @@ -173,7 +158,7 @@ readme = { file = [ [tool.setuptools_scm] fallback_version = "999" -write_to = "src/pyproject2conda/_version.py" +version_file = "src/pyproject2conda/_version.py" [tool.aliases] test = "pytest" @@ -284,12 +269,10 @@ ignore = [ # Allow relative imports "TID252", ] +per-file-ignores = { } # Allow unused variables when underscore-prefixed. dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$" -[tool.ruff.per-file-ignores] -"src/pyproject2conda/cli.py" = ["UP006", "UP007"] - [tool.ruff.isort] known-first-party = ["pyproject2conda"] @@ -307,7 +290,7 @@ ignore = [ "E203", # module level import not at top of file "E402", - # line too long - let black worry about that + # line t oo long - let black worry about that "E501", # do not assign a lambda expression, use a def "E731", @@ -343,14 +326,13 @@ use_shortcuts = true [tool.cruft] [tool.mypy] -files = ["src/pyproject2conda", "tests"] +files = ["src", "tests", "noxfile.py", "tools"] show_error_codes = true warn_unused_ignores = true warn_return_any = true warn_unused_configs = true exclude = [".eggs", ".tox", "doc", "docs", ".nox"] check_untyped_defs = true -strict = true [[tool.mypy.overrides]] ignore_missing_imports = true @@ -362,16 +344,7 @@ module = [] [tool.pyright] include = ["src", "tests"] -exclude = [ - "**/__pycache__", - ".tox/**", - ".nox/**", - "**/.mypy_cache", - "**/_version.py" -] -# extraPaths = ["."] -# TODO: add strict to pyright -# strict = ["src"] +exclude = ["**/__pycache__", ".tox/**", ".nox/**", "**/.mypy_cache"] pythonVersion = "3.10" typeCheckingMode = "basic" # enable subset of "strict" diff --git a/src/pyproject2conda/parser.py b/src/pyproject2conda/parser.py index 50b035d..4f58018 100644 --- a/src/pyproject2conda/parser.py +++ b/src/pyproject2conda/parser.py @@ -203,9 +203,7 @@ def _pyproject_to_value_comment_pairs( unique: bool = True, include_base_dependencies: bool = True, ) -> list[tuple[Tstr_opt, Tstr_opt]]: - package_name = cast( - str, get_in(["project", "name"], data, factory=_factory_empty_tomlkit_Array) - ) + package_name = cast(str, get_in(["project", "name"], data, default=None)) if package_name is None: raise ValueError("Must specify `project.package_name` in pyproject.toml") diff --git a/tests/test_parser.py b/tests/test_parser.py index 05e4cf3..26a3bf0 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -189,6 +189,85 @@ def test_output_to_yaml(): assert s == dedent(expected) +def test_infer(): + toml = dedent( + """\ + [project] + name = "hello" + dependencies = [ + "athing", # p2c: -p # a comment + "bthing", # p2c: -s bthing-conda + "cthing; python_version<'3.10'", # p2c: -c conda-forge + ] + + [project.optional-dependencies] + test = [ + "pandas", + "pytest", # p2c: -c conda-forge + ] + dev-extras = [ + # p2c: -s additional-thing # this is an additional conda package + "matplotlib", # p2c: -s conda-matplotlib + ] + dev = [ + "hello[test]", + "hello[dev-extras]", + ] + dist-pypi = [ + "setuptools", + "build", # p2c: -p + ] + + + [tool.pyproject2conda] + channels = ['conda-forge'] + """ + ) + + d = parser.PyProject2Conda.from_string(toml) + with pytest.raises(ValueError): + d.to_conda_yaml(python_include="infer") + + +def test_package_name(): + toml = dedent( + """\ + [project] + requires-python = ">=3.8,<3.11" + dependencies = [ + "athing", # p2c: -p # a comment + "bthing", # p2c: -s bthing-conda + "cthing; python_version<'3.10'", # p2c: -c conda-forge + ] + + [project.optional-dependencies] + test = [ + "pandas", + "pytest", # p2c: -c conda-forge + ] + dev-extras = [ + # p2c: -s additional-thing # this is an additional conda package + "matplotlib", # p2c: -s conda-matplotlib + ] + dev = [ + "hello[test]", + "hello[dev-extras]", + ] + dist-pypi = [ + "setuptools", + "build", # p2c: -p + ] + + + [tool.pyproject2conda] + channels = ['conda-forge'] + """ + ) + d = parser.PyProject2Conda.from_string(toml) + with pytest.raises(ValueError): + d.to_conda_lists() + + def test_complete(): toml = dedent( """\ diff --git a/tools/create_pythons.py b/tools/create_pythons.py index 1be3ec5..c075ea0 100644 --- a/tools/create_pythons.py +++ b/tools/create_pythons.py @@ -1,8 +1,11 @@ """Script to create pythons for use with virtualenvs""" from __future__ import annotations +import sys from functools import lru_cache +assert sys.version_info >= (3, 9) + @lru_cache def conda_cmd() -> str: From a06696b6f7256803819c5d1879bd03b1df393e34 Mon Sep 17 00:00:00 2001 From: wpk Date: Thu, 2 Nov 2023 10:17:57 -0400 Subject: [PATCH 03/36] chore: fixup --- noxfile.py | 18 ++++++++++++++++++ pyproject.toml | 21 ++++++++++++++++++--- requirements/dev-base.txt | 2 +- requirements/dev-complete.txt | 2 +- requirements/dev.txt | 2 +- requirements/py310-dev-base.yaml | 2 +- requirements/py310-dev-complete.yaml | 2 +- 7 files changed, 41 insertions(+), 8 deletions(-) diff --git a/noxfile.py b/noxfile.py index 4d71796..c10b598 100644 --- a/noxfile.py +++ b/noxfile.py @@ -504,6 +504,10 @@ def _docs( if open_page := "open" in cmd: cmd.remove("open") + if serve := "serve" in cmd: + open_webpage(url="http://localhost:8000") + cmd.remove("serve") + if cmd: args = ["make", "-C", "docs"] + combine_list_str(cmd) session.run(*args, external=True) @@ -511,6 +515,18 @@ def _docs( if open_page: open_webpage(path="./docs/_build/html/index.html") + if serve and "livehtml" not in cmd: + session.run( + "python", + "-m", + "http.server", + "-d", + "docs/_build/html", + "-b", + "127.0.0.1", + "8000", + ) + @DEFAULT_SESSION def docs( @@ -527,6 +543,7 @@ def docs( "showlinks", "release", "open", + "serve", ], flags=("--docs-cmd", "-d"), ) = (), @@ -567,6 +584,7 @@ def docs_venv( "showlinks", "release", "open", + "serve", ], flags=("--docs-cmd", "-d"), ) = (), diff --git a/pyproject.toml b/pyproject.toml index 75efcd2..3981fc0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,13 +25,24 @@ classifiers = [ ] dynamic = ["readme", "version"] requires-python = ">=3.8" -dependencies = ["typer"] +dependencies = [ + "tomli", + "ruamel.yaml", + "tomlkit", + "typer", + "packaging", + "typing-extensions; python_version<'3.9'", +] [project.urls] homepage = "https://github.com/usnistgov/pyproject2conda" documentation = "https://pages.nist.gov/pyproject2conda/" [project.optional-dependencies] +all = [ + "rich", # + "shellingham", +] test = [ "pytest", # "pytest-xdist", @@ -48,6 +59,7 @@ dev-extras = [ typing-extras = [ "pytype; python_version < '3.11'", # "mypy >= 1.4.1", + "types-click", ] typing = [ "pyproject2conda[typing-extras]", # @@ -135,7 +147,7 @@ envs = ["dist-conda"] style = ["yaml"] [project.scripts] -pyproject2conda = "pyproject2conda.cli:main" +pyproject2conda = "pyproject2conda.cli:app" ## grayskull still messes some things up, but use scripts/recipe-append.sh for this [tool.setuptools] @@ -269,10 +281,12 @@ ignore = [ # Allow relative imports "TID252", ] -per-file-ignores = { } # Allow unused variables when underscore-prefixed. dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$" +[tool.ruff.per-file-ignores] +"src/pyproject2conda/cli.py" = ["UP006", "UP007"] + [tool.ruff.isort] known-first-party = ["pyproject2conda"] @@ -333,6 +347,7 @@ warn_return_any = true warn_unused_configs = true exclude = [".eggs", ".tox", "doc", "docs", ".nox"] check_untyped_defs = true +strict = true [[tool.mypy.overrides]] ignore_missing_imports = true diff --git a/requirements/dev-base.txt b/requirements/dev-base.txt index 2f29a5f..bee2e1e 100644 --- a/requirements/dev-base.txt +++ b/requirements/dev-base.txt @@ -18,7 +18,7 @@ pytest-sugar pytest-xdist pytype; python_version < '3.11' ruamel.yaml -setuptools-scm +setuptools-scm>=8.0 tomli tomlkit typer diff --git a/requirements/dev-complete.txt b/requirements/dev-complete.txt index ad0d5f7..75e7985 100644 --- a/requirements/dev-complete.txt +++ b/requirements/dev-complete.txt @@ -24,7 +24,7 @@ pytest-xdist pytype; python_version < '3.11' ruamel.yaml scriv -setuptools-scm +setuptools-scm>=8.0 tomli tomlkit typer diff --git a/requirements/dev.txt b/requirements/dev.txt index 03276c3..8a3ffd5 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -20,7 +20,7 @@ pytest-sugar pytest-xdist pytype; python_version < '3.11' ruamel.yaml -setuptools-scm +setuptools-scm>=8.0 tomli tomlkit typer diff --git a/requirements/py310-dev-base.yaml b/requirements/py310-dev-base.yaml index 54f9d7d..c007719 100644 --- a/requirements/py310-dev-base.yaml +++ b/requirements/py310-dev-base.yaml @@ -21,7 +21,7 @@ dependencies: - pytest-xdist - pytype - ruamel.yaml - - setuptools-scm + - setuptools-scm>=8.0 - tomli - tomlkit - typer diff --git a/requirements/py310-dev-complete.yaml b/requirements/py310-dev-complete.yaml index fc3f256..4e55e51 100644 --- a/requirements/py310-dev-complete.yaml +++ b/requirements/py310-dev-complete.yaml @@ -25,7 +25,7 @@ dependencies: - pytest-xdist - pytype - ruamel.yaml - - setuptools-scm + - setuptools-scm>=8.0 - tomli - tomlkit - typer From b0f3336052408ad2209fccd69969d2f17042bbc9 Mon Sep 17 00:00:00 2001 From: wpk Date: Thu, 2 Nov 2023 12:43:11 -0400 Subject: [PATCH 04/36] chore: udpate cruft --- .cruft.json | 2 +- CHANGELOG.md | 6 +++++- pyproject.toml | 2 +- tools/noxtools.py | 4 ++-- 4 files changed, 9 insertions(+), 5 deletions(-) diff --git a/.cruft.json b/.cruft.json index 4a3e774..2869f94 100644 --- a/.cruft.json +++ b/.cruft.json @@ -1,6 +1,6 @@ { "template": "https://github.com/usnistgov/cookiecutter-nist-python.git", - "commit": "41b5d67426f708650af617193e42e081fe965649", + "commit": "ab2f71c9e76533931d52cd32c7091fce972f7626", "checkout": "develop", "context": { "cookiecutter": { diff --git a/CHANGELOG.md b/CHANGELOG.md index bfcef24..c226e1a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,10 +8,14 @@ Changelog for `pyproject2conda` ## Unreleased -[changelog.d]: https://github.com/usnistgov/pyproject2conda +[changelog.d]: https://github.com/usnistgov/pyproject2conda/tree/main/changelog.d See the fragment files in [changelog.d] + + + + ## v0.8.0 — 2023-10-02 diff --git a/pyproject.toml b/pyproject.toml index 3981fc0..769adc8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -304,7 +304,7 @@ ignore = [ "E203", # module level import not at top of file "E402", - # line t oo long - let black worry about that + # line too long - let black worry about that "E501", # do not assign a lambda expression, use a def "E731", diff --git a/tools/noxtools.py b/tools/noxtools.py index 1ad3d8a..cd2426d 100644 --- a/tools/noxtools.py +++ b/tools/noxtools.py @@ -6,7 +6,7 @@ from pathlib import Path from typing import TYPE_CHECKING, Any, Iterable, Literal, Sequence, TextIO, cast -from ruamel.yaml import safe_load +from ruamel.yaml import YAML if TYPE_CHECKING: from collections.abc import Collection @@ -394,7 +394,7 @@ def _get_context(path: str | Path | TextIO) -> TextIO | Path: for path in paths: with _get_context(path) as f: - data = safe_load(f) + data = YAML(typ="safe", pure=True).load(f) channels.update(data.get("channels", [])) name = data.get("name", name) From e94d2c3a34f3d4d9d05e8df80803b341aa066bae Mon Sep 17 00:00:00 2001 From: wpk Date: Mon, 6 Nov 2023 15:33:32 -0500 Subject: [PATCH 05/36] chore: remove _copy_without_render from .cruft.json --- .cruft.json | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/.cruft.json b/.cruft.json index 2869f94..316b6bc 100644 --- a/.cruft.json +++ b/.cruft.json @@ -11,16 +11,7 @@ "conda_channel": "wpk-nist", "project_name": "pyproject2conda", "project_slug": "pyproject2conda", - "_copy_without_render": [ - "*.html", - "docs/_templates/*.rst", - "docs/_templates/autosummary/*.rst", - "docs/_templates/autodocsumm/*.rst", - "docs/_static/css/*", - "docs/_static/js/*", - "changelog.d/templates/*.j2", - "changelog.d/templates/auto-changelog/*.jinja2" - ], + "_copy_without_render": [], "project_short_description": "A script to convert a Python project declared on a pyproject.toml to a conda environment.", "command_line_interface": "Typer", "sphinx_use_autodocsumm": "y", From 2ff50b1b90d635d95d38223a574daa6276764def Mon Sep 17 00:00:00 2001 From: wpk Date: Mon, 6 Nov 2023 15:34:24 -0500 Subject: [PATCH 06/36] chore: udpate cruft --- .cruft.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.cruft.json b/.cruft.json index 316b6bc..74b26cf 100644 --- a/.cruft.json +++ b/.cruft.json @@ -1,6 +1,6 @@ { "template": "https://github.com/usnistgov/cookiecutter-nist-python.git", - "commit": "ab2f71c9e76533931d52cd32c7091fce972f7626", + "commit": "e216a57fbff8458b829095114546bb454c2f51db", "checkout": "develop", "context": { "cookiecutter": { From 604a93b62bc7775a331b7bb14e302cb724c43ed9 Mon Sep 17 00:00:00 2001 From: wpk Date: Thu, 9 Nov 2023 16:27:19 -0500 Subject: [PATCH 07/36] chore: update .cruft before update --- .cruft.json | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.cruft.json b/.cruft.json index 74b26cf..b1aaa09 100644 --- a/.cruft.json +++ b/.cruft.json @@ -16,6 +16,8 @@ "command_line_interface": "Typer", "sphinx_use_autodocsumm": "y", "sphinx_theme": "sphinx_book_theme", + "__year": "2023", + "__answers": "", "_template": "https://github.com/usnistgov/cookiecutter-nist-python.git" } }, From 58be7078b9d1f2f327993431330611e067c93743 Mon Sep 17 00:00:00 2001 From: wpk Date: Thu, 9 Nov 2023 16:32:05 -0500 Subject: [PATCH 08/36] chore: update cruft to new template --- .cruft.json | 2 +- .pre-commit-config.yaml | 2 +- CONTRIBUTING.md | 28 ++++--- noxfile.py | 60 ++++++--------- pyproject.toml | 1 - src/pyproject2conda/__init__.py | 10 +-- tools/noxtools.py | 128 ++++++++++++++++++-------------- 7 files changed, 119 insertions(+), 112 deletions(-) diff --git a/.cruft.json b/.cruft.json index b1aaa09..9e5c409 100644 --- a/.cruft.json +++ b/.cruft.json @@ -1,6 +1,6 @@ { "template": "https://github.com/usnistgov/cookiecutter-nist-python.git", - "commit": "e216a57fbff8458b829095114546bb454c2f51db", + "commit": "bb73e10f6947a5ca80b8d1c4ac83875b965dab6a", "checkout": "develop", "context": { "cookiecutter": { diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a16a15c..cfcd77b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -35,7 +35,7 @@ repos: stages: [commit] additional_dependencies: - prettier-plugin-toml - exclude: ^requirements/lock/.*[.]yml + exclude: ^requirements/lock/.*[.]yml|^.copier-answers.y?ml # * Markdown - repo: https://github.com/DavidAnson/markdownlint-cli2 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index da0f48f..cc317bd 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -279,6 +279,8 @@ where commands can be one of: [ghp-import](https://github.com/c-w/ghp-import)) - livehtml : Live documentation updates - open : open the documentation in a web browser +- serve : Serve the created documentation webpage (Need this to view javescript + in created pages). ## Testing with nox @@ -518,22 +520,23 @@ Before you submit a pull request, check that it meets these guidelines: [setuptools_scm]: https://github.com/pypa/setuptools_scm -Versioning is handled with [setuptools_scm].The package version is set by the +Versioning is handled with [setuptools_scm]. The package version is set by the git tag. For convenience, you can override the version with nox setting `--version ...`. This is useful for updating the docs, etc. -We use the `write_to` option to [setuptools_scm]. This stores the current -version in `_version.py`. Note that if you build the package (or, build docs -with the `--version` flag), this will overwrite information in `_version.py` in -the `src` directory. To refresh the version, run: +Note that the version in a given environment/session can become stale. The +easiest way to update the installed package version version is to reinstall the +package. This can be done using the following: ```bash -make version-scm +pip install -e . --no-deps ``` -Note also that the file `_version.py` SHOULD NOT be tracked by git. It will be -autogenerated when building the package. This scheme avoids having to install -`setuptools-scm` (and `setuptools`) in each environment. +To do this in a given session, use: + +```bash +nox -s {session} -- -P/--update-package +``` ## Serving the documentation @@ -543,4 +546,9 @@ To view to documentation with js headers/footers, you'll need to serve them: python -m http.server -d docs/_build/html ``` -Then open the address `localhost:8000` in a webbrowser. +Then open the address `localhost:8000` in a webbrowser. Alternatively, you can +run: + +```bash +nox -s docs -- -d serve +``` diff --git a/noxfile.py b/noxfile.py index c10b598..229bc96 100644 --- a/noxfile.py +++ b/noxfile.py @@ -144,6 +144,16 @@ def run_annotated(**kwargs: Any): # type: ignore[unused-ignore,no-untyped-def] ), ] + +UPDATE_PACKAGE_CLI = Annotated[ + bool, + Option( + type=bool, + help="If True, and session uses package, reinstall package", + flags=("--update-package", "-P"), + ), +] + VERSION_CLI = Annotated[ str, Option(type=str, help="Version to substitute or check against") ] @@ -165,6 +175,7 @@ def dev( dev_run: RUN_CLI = [], # noqa lock: LOCK_CLI = False, update: UPDATE_CLI = False, + update_package: UPDATE_PACKAGE_CLI = False, log_session: bool = False, ) -> None: """Create dev env using conda.""" @@ -177,6 +188,7 @@ def dev( display_name=f"{PACKAGE_NAME}-dev", install_package=True, update=update, + update_package=update_package, log_session=log_session, ) session_run_commands(session, dev_run) @@ -189,6 +201,7 @@ def dev_venv( dev_run: RUN_CLI = [], # noqa lock: LOCK_CLI = False, update: UPDATE_CLI = False, + update_package: UPDATE_PACKAGE_CLI = False, log_session: bool = False, ) -> None: """Create dev env using virtualenv.""" @@ -202,6 +215,7 @@ def dev_venv( display_name=f"{PACKAGE_NAME}-dev-venv", install_package=True, update=update, + update_package=update_package, log_session=log_session, ) session_run_commands(session, dev_run) @@ -375,6 +389,7 @@ def test( test_run: RUN_CLI = [], # noqa lock: LOCK_CLI = False, update: UPDATE_CLI = False, + update_package: UPDATE_PACKAGE_CLI = False, log_session: bool = False, no_cov: bool = False, ) -> None: @@ -386,6 +401,7 @@ def test( lock=lock, install_package=True, update=update, + update_package=update_package, log_session=log_session, ) @@ -406,6 +422,7 @@ def test_venv( test_run: RUN_CLI = [], # noqa lock: LOCK_CLI = False, # pyright: ignore update: UPDATE_CLI = False, + update_package: UPDATE_PACKAGE_CLI = False, log_session: bool = False, no_cov: bool = False, ) -> None: @@ -417,6 +434,7 @@ def test_venv( install_package=True, requirement_paths="test.txt", update=update, + update_package=update_package, log_session=log_session, ) @@ -458,7 +476,7 @@ def _coverage( session_run_commands(session, run_internal, external=False) -@DEFAULT_SESSION_VENV +@INHERITED_SESSION_VENV def coverage( session: Session, coverage_cmd: cmd_annotated( # type: ignore @@ -550,6 +568,7 @@ def docs( docs_run: RUN_CLI = [], # noqa lock: LOCK_CLI = False, update: UPDATE_CLI = False, + update_package: UPDATE_PACKAGE_CLI = False, version: VERSION_CLI = "", log_session: bool = False, ) -> None: @@ -561,6 +580,7 @@ def docs( display_name=f"{PACKAGE_NAME}-docs", install_package=True, update=update, + update_package=update_package, log_session=log_session, ) @@ -591,6 +611,7 @@ def docs_venv( docs_run: RUN_CLI = [], # noqa lock: LOCK_CLI = False, update: UPDATE_CLI = False, + update_package: UPDATE_PACKAGE_CLI = False, version: VERSION_CLI = "", log_session: bool = False, ) -> None: @@ -602,6 +623,7 @@ def docs_venv( display_name=f"{PACKAGE_NAME}-docs-venv", install_package=True, update=update, + update_package=update_package, log_session=log_session, requirement_paths="docs.txt", ) @@ -1091,42 +1113,6 @@ def testdist_pypi_condaenv( ) -@DEFAULT_SESSION_VENV -def update_version_scm( - session: Session, - version: VERSION_CLI = "", - update: UPDATE_CLI = False, -) -> None: - """ - Get current version from setuptools-scm - - Note that the version of editable installs can get stale. - This will show the actual current version. - Avoids need to include setuptools-scm in develop/docs/etc. - """ - - if version: - session.env["SETUPTOOLS_SCM_PRETEND_VERSION"] = version - - pkg_install_venv( - session=session, - name="update-version-scm", - install_package=True, - # reqs=["setuptools_scm"], - update=True, - no_deps=True, - ) - - session.run( - "python", - "-c", - "import sys;" - "sys.path.insert(0, 'src');" - "from pyproject2conda._version import __version__;" - "print(__version__);", - ) - - # * Utilities -------------------------------------------------------------------------- def _create_doc_examples_symlinks(session: nox.Session, clean: bool = True) -> None: """Create symlinks from docs/examples/*.md files to /examples/usage/...""" diff --git a/pyproject.toml b/pyproject.toml index 769adc8..6fac6ec 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -170,7 +170,6 @@ readme = { file = [ [tool.setuptools_scm] fallback_version = "999" -version_file = "src/pyproject2conda/_version.py" [tool.aliases] test = "pytest" diff --git a/src/pyproject2conda/__init__.py b/src/pyproject2conda/__init__.py index 3f67f00..2f12e24 100644 --- a/src/pyproject2conda/__init__.py +++ b/src/pyproject2conda/__init__.py @@ -2,15 +2,15 @@ Top level API (:mod:`pyproject2conda`) ====================================== """ +from importlib.metadata import PackageNotFoundError +from importlib.metadata import version as _version -from .parser import PyProject2Conda - -# updated versioning scheme try: - from ._version import __version__ -except Exception: # pragma: no cover + __version__ = _version("pyproject2conda") +except PackageNotFoundError: # pragma: no cover __version__ = "999" +from .parser import PyProject2Conda __author__ = """William P. Krekelberg""" __email__ = "wpk@nist.gov" diff --git a/tools/noxtools.py b/tools/noxtools.py index cd2426d..afe84e7 100644 --- a/tools/noxtools.py +++ b/tools/noxtools.py @@ -60,6 +60,7 @@ def pkg_install_condaenv( display_name: str | None = None, install_package: bool = True, update: bool = False, + update_package: bool = False, log_session: bool = False, deps: Collection[str] | None = None, reqs: Collection[str] | None = None, @@ -86,6 +87,7 @@ def check_filename(filename: str | Path) -> str: lockfile=check_filename(filename), display_name=display_name, update=update, + update_package=update_package, install_package=install_package, **kwargs, ) @@ -96,6 +98,7 @@ def check_filename(filename: str | Path) -> str: check_filename(filename), display_name=display_name, update=update, + update_package=update_package, deps=deps, reqs=reqs, channels=channels, @@ -117,6 +120,7 @@ def pkg_install_venv( reqs: Collection[str] | None = None, display_name: str | None = None, update: bool = False, + update_package: bool = False, install_package: bool = False, no_deps: bool = True, log_session: bool = False, @@ -132,6 +136,7 @@ def pkg_install_venv( reqs=reqs, display_name=display_name, update=update, + update_package=update_package, install_package=install_package, no_deps=no_deps, lock=lock, @@ -317,6 +322,7 @@ def session_install_envs_lock( extras: str | list[str] | None = None, display_name: str | None = None, update: bool = False, + update_package: bool = False, install_package: bool = False, ) -> bool: """Install dependencies using conda-lock.""" @@ -327,34 +333,36 @@ def session_install_envs_lock( unchanged, hashes = env_unchanged( session, lockfile, prefix="lock", other=dict(install_package=install_package) ) - if unchanged and not update: - return unchanged - if extras: - if isinstance(extras, str): - extras = extras.split(",") - extras = cast(list[str], sum([["--extras", _] for _ in extras], [])) - else: - extras = [] - - session.run( - "conda-lock", - "install", - "--mamba", - *extras, - "-p", - str(session.virtualenv.location), - str(lockfile), - silent=True, - external=True, - ) + do_dep = update or (not unchanged) + do_pkg = install_package and (do_dep or update_package) - if install_package: - session_install_package(session) + if do_dep: + if extras: + if isinstance(extras, str): + extras = extras.split(",") + extras = cast(list[str], sum([["--extras", _] for _ in extras], [])) + else: + extras = [] + + session.run( + "conda-lock", + "install", + "--mamba", + *extras, + "-p", + str(session.virtualenv.location), + str(lockfile), + silent=True, + external=True, + ) - session_set_ipykernel_display_name(session, display_name) + if do_pkg: + session_install_package(session) - write_hashfile(hashes, session=session, prefix="lock") + if do_dep or do_pkg: + session_set_ipykernel_display_name(session, display_name) + write_hashfile(hashes, session=session, prefix="lock") return unchanged @@ -421,6 +429,7 @@ def session_install_envs( install_kws: dict[str, Any] | None = None, display_name: str | None = None, update: bool = False, + update_package: bool = False, install_package: bool = False, ) -> bool: """Parse and install everything. Pass an already merged yaml file.""" @@ -446,30 +455,32 @@ def session_install_envs( install_package=install_package, ), ) - if unchanged and not update: - return unchanged - if not channels: - channels = "" - if deps: - conda_install_kws = conda_install_kws or {} - conda_install_kws.update(channel=channels) - if update: - deps = ["--update-all"] + list(deps) + do_dep = update or (not unchanged) + do_pkg = install_package and (do_dep or update_package) - session.conda_install(*deps, **(conda_install_kws or {})) + if do_dep: + if not channels: + channels = "" + if deps: + conda_install_kws = conda_install_kws or {} + conda_install_kws.update(channel=channels) + if update: + deps = ["--update-all"] + list(deps) - if reqs: - if update: - reqs = ["--upgrade"] + list(reqs) - session.install(*reqs, **(install_kws or {})) + session.conda_install(*deps, **(conda_install_kws or {})) - if install_package: - session_install_package(session) + if reqs: + if update: + reqs = ["--upgrade"] + list(reqs) + session.install(*reqs, **(install_kws or {})) - session_set_ipykernel_display_name(session, display_name) + if do_pkg: + session_install_package(session) - write_hashfile(hashes, session=session, prefix="env") + if do_dep or do_pkg: + session_set_ipykernel_display_name(session, display_name) + write_hashfile(hashes, session=session, prefix="env") return unchanged @@ -483,6 +494,7 @@ def session_install_pip( reqs: str | Collection[str] | None = None, display_name: str | None = None, update: bool = False, + update_package: bool = False, install_package: bool = False, no_deps: bool = True, lock: bool = False, @@ -545,26 +557,28 @@ def _verify_paths(paths: str | list[str]) -> list[str]: ), ) - if unchanged and not update: - return unchanged + do_dep = update or (not unchanged) + do_pkg = install_package and (do_dep or update_package) - # do install - install_args = ( - prepend_flag("-r", *requirement_paths) - + prepend_flag("-c", *constraint_paths) - + list(reqs) - ) + if do_dep: + # do install + install_args = ( + prepend_flag("-r", *requirement_paths) + + prepend_flag("-c", *constraint_paths) + + list(reqs) + ) - if install_args: - if update: - install_args = ["--upgrade"] + list(install_args) - session.install(*install_args) + if install_args: + if update: + install_args = ["--upgrade"] + list(install_args) + session.install(*install_args) - if install_package: + if do_pkg: session.install(*install_package_args) - session_set_ipykernel_display_name(session, display_name) - write_hashfile(hashes, session=session, prefix="pip") + if do_dep or do_pkg: + session_set_ipykernel_display_name(session, display_name) + write_hashfile(hashes, session=session, prefix="pip") return unchanged From ce17f72d6a1e9102afeddbc784393ee02f1b3bc7 Mon Sep 17 00:00:00 2001 From: wpk Date: Mon, 13 Nov 2023 11:00:04 -0500 Subject: [PATCH 09/36] chore: update .cruft before update --- .cruft.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.cruft.json b/.cruft.json index 9e5c409..404f994 100644 --- a/.cruft.json +++ b/.cruft.json @@ -1,6 +1,6 @@ { "template": "https://github.com/usnistgov/cookiecutter-nist-python.git", - "commit": "bb73e10f6947a5ca80b8d1c4ac83875b965dab6a", + "commit": "e84f5884bfaa90e9397f41549b621ecacf747a3f", "checkout": "develop", "context": { "cookiecutter": { @@ -13,10 +13,10 @@ "project_slug": "pyproject2conda", "_copy_without_render": [], "project_short_description": "A script to convert a Python project declared on a pyproject.toml to a conda environment.", - "command_line_interface": "Typer", - "sphinx_use_autodocsumm": "y", + "command_line_interface": "typer", + "sphinx_use_autodocsumm": true, "sphinx_theme": "sphinx_book_theme", - "__year": "2023", + "year": "2023", "__answers": "", "_template": "https://github.com/usnistgov/cookiecutter-nist-python.git" } From 3bbc295043a54b15d7e98e91904c0e8b4e527acf Mon Sep 17 00:00:00 2001 From: wpk Date: Mon, 13 Nov 2023 11:03:34 -0500 Subject: [PATCH 10/36] chore: update cruft --- .cruft.json | 10 +++++----- .pre-commit-config.yaml | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.cruft.json b/.cruft.json index 404f994..e1ce2ba 100644 --- a/.cruft.json +++ b/.cruft.json @@ -1,23 +1,23 @@ { "template": "https://github.com/usnistgov/cookiecutter-nist-python.git", - "commit": "e84f5884bfaa90e9397f41549b621ecacf747a3f", + "commit": "8fe5103d25de7e7d13d400f599c6653866a69afc", "checkout": "develop", "context": { "cookiecutter": { + "project_name": "pyproject2conda", + "project_slug": "pyproject2conda", + "project_short_description": "A script to convert a Python project declared on a pyproject.toml to a conda environment.", "full_name": "William P. Krekelberg", "email": "wpk@nist.gov", "github_username": "usnistgov", "pypi_username": "conda-forge", "conda_channel": "wpk-nist", - "project_name": "pyproject2conda", - "project_slug": "pyproject2conda", - "_copy_without_render": [], - "project_short_description": "A script to convert a Python project declared on a pyproject.toml to a conda environment.", "command_line_interface": "typer", "sphinx_use_autodocsumm": true, "sphinx_theme": "sphinx_book_theme", "year": "2023", "__answers": "", + "_copy_without_render": [], "_template": "https://github.com/usnistgov/cookiecutter-nist-python.git" } }, diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index cfcd77b..f7037ad 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -35,7 +35,7 @@ repos: stages: [commit] additional_dependencies: - prettier-plugin-toml - exclude: ^requirements/lock/.*[.]yml|^.copier-answers.y?ml + exclude: ^requirements/lock/.*[.]yml|^.copier-answers.ya?ml # * Markdown - repo: https://github.com/DavidAnson/markdownlint-cli2 From 82ec20119c635877d1b587e613dcdac12c6c627e Mon Sep 17 00:00:00 2001 From: wpk Date: Mon, 13 Nov 2023 15:55:40 -0500 Subject: [PATCH 11/36] feat: Remove whitespace now default for dependencies --- ...11_william.krekelberg_ignore_whitespace.md | 44 ++++++++++ pyproject.toml | 1 + requirements/dev-base.txt | 6 +- requirements/dev-complete.txt | 6 +- requirements/dev.txt | 6 +- requirements/docs.txt | 4 +- requirements/py310-dev-base.yaml | 2 +- requirements/py310-dev-complete.yaml | 2 +- requirements/py310-docs.yaml | 2 +- requirements/py310-typing.yaml | 2 +- requirements/py311-typing.yaml | 2 +- requirements/py38-typing.yaml | 2 +- requirements/py39-typing.yaml | 2 +- requirements/test.txt | 2 +- requirements/typing.txt | 6 +- src/pyproject2conda/cli.py | 17 ++++ src/pyproject2conda/config.py | 11 +++ src/pyproject2conda/parser.py | 25 ++++++ tests/test_cli.py | 36 ++++++-- tests/test_config.py | 37 ++++++++ tests/test_parser.py | 85 ++++++++++++++++++- 21 files changed, 272 insertions(+), 28 deletions(-) create mode 100644 changelog.d/20231113_155311_william.krekelberg_ignore_whitespace.md diff --git a/changelog.d/20231113_155311_william.krekelberg_ignore_whitespace.md b/changelog.d/20231113_155311_william.krekelberg_ignore_whitespace.md new file mode 100644 index 0000000..1aab19c --- /dev/null +++ b/changelog.d/20231113_155311_william.krekelberg_ignore_whitespace.md @@ -0,0 +1,44 @@ + + + + + +### Added + +- Default is now to remove whitespace from dependencies. For example, the + dependency `module > 0.1` will become `module>0.1`. To override this + behaviour, pass the option `--no-remove-whitespace`. + + + + + diff --git a/pyproject.toml b/pyproject.toml index 6fac6ec..968aa4f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -148,6 +148,7 @@ style = ["yaml"] [project.scripts] pyproject2conda = "pyproject2conda.cli:app" +p2c = "pyproject2conda.cli:app" ## grayskull still messes some things up, but use scripts/recipe-append.sh for this [tool.setuptools] diff --git a/requirements/dev-base.txt b/requirements/dev-base.txt index bee2e1e..c76fb3b 100644 --- a/requirements/dev-base.txt +++ b/requirements/dev-base.txt @@ -9,18 +9,18 @@ # ipykernel ipython -mypy >= 1.4.1 +mypy>=1.4.1 packaging pytest pytest-accept pytest-cov pytest-sugar pytest-xdist -pytype; python_version < '3.11' +pytype;python_version<'3.11' ruamel.yaml setuptools-scm>=8.0 tomli tomlkit typer types-click -typing-extensions; python_version<'3.9' +typing-extensions;python_version<'3.9' diff --git a/requirements/dev-complete.txt b/requirements/dev-complete.txt index 75e7985..99322cf 100644 --- a/requirements/dev-complete.txt +++ b/requirements/dev-complete.txt @@ -9,7 +9,7 @@ # ipykernel ipython -mypy >= 1.4.1 +mypy>=1.4.1 nbqa nox noxopt @@ -21,7 +21,7 @@ pytest-accept pytest-cov pytest-sugar pytest-xdist -pytype; python_version < '3.11' +pytype;python_version<'3.11' ruamel.yaml scriv setuptools-scm>=8.0 @@ -29,4 +29,4 @@ tomli tomlkit typer types-click -typing-extensions; python_version<'3.9' +typing-extensions;python_version<'3.9' diff --git a/requirements/dev.txt b/requirements/dev.txt index 8a3ffd5..98070d8 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -9,7 +9,7 @@ # ipykernel ipython -mypy >= 1.4.1 +mypy>=1.4.1 nox noxopt packaging @@ -18,11 +18,11 @@ pytest-accept pytest-cov pytest-sugar pytest-xdist -pytype; python_version < '3.11' +pytype;python_version<'3.11' ruamel.yaml setuptools-scm>=8.0 tomli tomlkit typer types-click -typing-extensions; python_version<'3.9' +typing-extensions;python_version<'3.9' diff --git a/requirements/docs.txt b/requirements/docs.txt index 2eca0b3..e99473a 100644 --- a/requirements/docs.txt +++ b/requirements/docs.txt @@ -14,13 +14,13 @@ myst-nb packaging pyenchant ruamel.yaml -sphinx >= 5.3.0 sphinx-autobuild sphinx-book-theme sphinx-click sphinx-copybutton +sphinx>=5.3.0 sphinxcontrib-spelling tomli tomlkit typer -typing-extensions; python_version<'3.9' +typing-extensions;python_version<'3.9' diff --git a/requirements/py310-dev-base.yaml b/requirements/py310-dev-base.yaml index c007719..fd92c2a 100644 --- a/requirements/py310-dev-base.yaml +++ b/requirements/py310-dev-base.yaml @@ -13,7 +13,7 @@ dependencies: - python=3.10 - ipykernel - ipython - - mypy >= 1.4.1 + - mypy>=1.4.1 - packaging - pytest - pytest-cov diff --git a/requirements/py310-dev-complete.yaml b/requirements/py310-dev-complete.yaml index 4e55e51..a46a26d 100644 --- a/requirements/py310-dev-complete.yaml +++ b/requirements/py310-dev-complete.yaml @@ -13,7 +13,7 @@ dependencies: - python=3.10 - ipykernel - ipython - - mypy >= 1.4.1 + - mypy>=1.4.1 - nbqa - nox - packaging diff --git a/requirements/py310-docs.yaml b/requirements/py310-docs.yaml index 40980dc..bda7cfb 100644 --- a/requirements/py310-docs.yaml +++ b/requirements/py310-docs.yaml @@ -18,7 +18,7 @@ dependencies: - packaging - pyenchant - ruamel.yaml - - sphinx >= 5.3.0 + - sphinx>=5.3.0 - sphinx-autobuild - sphinx-book-theme - sphinx-click diff --git a/requirements/py310-typing.yaml b/requirements/py310-typing.yaml index 32f99fb..b286b19 100644 --- a/requirements/py310-typing.yaml +++ b/requirements/py310-typing.yaml @@ -11,7 +11,7 @@ channels: - conda-forge dependencies: - python=3.10 - - mypy >= 1.4.1 + - mypy>=1.4.1 - packaging - pytest - pytest-cov diff --git a/requirements/py311-typing.yaml b/requirements/py311-typing.yaml index 51308cc..6d6d0c1 100644 --- a/requirements/py311-typing.yaml +++ b/requirements/py311-typing.yaml @@ -11,7 +11,7 @@ channels: - conda-forge dependencies: - python=3.11 - - mypy >= 1.4.1 + - mypy>=1.4.1 - packaging - pytest - pytest-cov diff --git a/requirements/py38-typing.yaml b/requirements/py38-typing.yaml index ccc2ec7..21b7c3c 100644 --- a/requirements/py38-typing.yaml +++ b/requirements/py38-typing.yaml @@ -11,7 +11,7 @@ channels: - conda-forge dependencies: - python=3.8 - - mypy >= 1.4.1 + - mypy>=1.4.1 - packaging - pytest - pytest-cov diff --git a/requirements/py39-typing.yaml b/requirements/py39-typing.yaml index 6a5938b..03d436c 100644 --- a/requirements/py39-typing.yaml +++ b/requirements/py39-typing.yaml @@ -11,7 +11,7 @@ channels: - conda-forge dependencies: - python=3.9 - - mypy >= 1.4.1 + - mypy>=1.4.1 - packaging - pytest - pytest-cov diff --git a/requirements/test.txt b/requirements/test.txt index 543fe03..306399d 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -16,4 +16,4 @@ ruamel.yaml tomli tomlkit typer -typing-extensions; python_version<'3.9' +typing-extensions;python_version<'3.9' diff --git a/requirements/typing.txt b/requirements/typing.txt index a6d0985..47af58d 100644 --- a/requirements/typing.txt +++ b/requirements/typing.txt @@ -7,16 +7,16 @@ # You should not manually edit this file. # Instead edit the corresponding pyproject.toml file. # -mypy >= 1.4.1 +mypy>=1.4.1 packaging pytest pytest-cov pytest-sugar pytest-xdist -pytype; python_version < '3.11' +pytype;python_version<'3.11' ruamel.yaml tomli tomlkit typer types-click -typing-extensions; python_version<'3.9' +typing-extensions;python_version<'3.9' diff --git a/src/pyproject2conda/cli.py b/src/pyproject2conda/cli.py index 1dd5d1e..f178056 100644 --- a/src/pyproject2conda/cli.py +++ b/src/pyproject2conda/cli.py @@ -334,6 +334,17 @@ class Overwrite(str, Enum): ) +REMOVE_WHITESPACE_OPTION = typer.Option( + "--remove-whitespace/--no-remove-whitespace", + help=""" + What to do with whitespace in a dependency. The default (`--remove-whitespace`) is + to remove whitespace in a given dependency. For example, the dependency + `package >= 1.0` will be converted to `package>=1.0`. Pass `--no-remove-whitespace` + to keep the the whitespace in the output. + """, +) + + # * Utils ------------------------------------------------------------------------------ def _get_header_cmd( header: Optional[bool], output: Union[str, Path, None] @@ -467,6 +478,7 @@ def yaml( deps: DEPS_CLI = None, reqs: REQS_CLI = None, allow_empty: Annotated[bool, ALLOW_EMPTY_OPTION] = False, + remove_whitespace: Annotated[bool, REMOVE_WHITESPACE_OPTION] = True, ) -> None: """Create yaml file from dependencies and optional-dependencies.""" @@ -500,6 +512,7 @@ def yaml( deps=deps, reqs=reqs, allow_empty=allow_empty, + remove_whitespace=remove_whitespace, ) if not output: print(s, end="") @@ -520,6 +533,7 @@ def requirements( verbose: VERBOSE_CLI = None, # pyright: ignore reqs: REQS_CLI = None, allow_empty: Annotated[bool, ALLOW_EMPTY_OPTION] = False, + remove_whitespace: Annotated[bool, REMOVE_WHITESPACE_OPTION] = True, ) -> None: """Create requirements.txt for pip dependencies.""" @@ -539,6 +553,7 @@ def requirements( sort=sort, reqs=reqs, allow_empty=allow_empty, + remove_whitespace=remove_whitespace, ) if not output: print(s, end="") @@ -562,6 +577,7 @@ def project( dry: DRY_CLI = False, user_config: USER_CONFIG_CLI = "infer", allow_empty: Annotated[Optional[bool], ALLOW_EMPTY_OPTION] = None, + remove_whitespace: Annotated[Optional[bool], REMOVE_WHITESPACE_OPTION] = None, ) -> None: """Create multiple environment files from `pyproject.toml` specification.""" from pyproject2conda.config import Config @@ -580,6 +596,7 @@ def project( overwrite=overwrite.value, verbose=verbose, allow_empty=allow_empty, + remove_whitespace=remove_whitespace, ): if dry: # small header diff --git a/src/pyproject2conda/config.py b/src/pyproject2conda/config.py index 71af17a..b257be1 100644 --- a/src/pyproject2conda/config.py +++ b/src/pyproject2conda/config.py @@ -222,6 +222,15 @@ def allow_empty(self, env_name: str | None = None, default: bool = False) -> boo key="allow_empty", env_name=env_name, default=default ) + def remove_whitespace( + self, env_name: str | None = None, default: bool = True + ) -> bool: + return self._get_value( # type: ignore + key="remove_whitespace", + env_name=env_name, + default=default, + ) + def assign_user_config(self, user: Self) -> Self: """Assign user_config to self.""" from copy import deepcopy @@ -275,6 +284,7 @@ def _iter_yaml( "name", "channels", "allow_empty", + "remove_whitespace", ] data = {k: defaults.get(k, getattr(self, k)(env_name)) for k in keys} @@ -317,6 +327,7 @@ def _iter_reqs( "verbose", "reqs", "allow_empty", + "remove_whitespace", ] output, template, _ = self._get_output_and_templates(env_name, **defaults) diff --git a/src/pyproject2conda/parser.py b/src/pyproject2conda/parser.py index 4f58018..f807a14 100644 --- a/src/pyproject2conda/parser.py +++ b/src/pyproject2conda/parser.py @@ -53,6 +53,13 @@ def _check_allow_empty(allow_empty: bool) -> str: raise ValueError(msg) +_WHITE_SPACE_REGEX = re.compile(r"\s+") + + +def _remove_whitespace(s: str) -> str: + return re.sub(_WHITE_SPACE_REGEX, "", s) + + # --- Default parser ------------------------------------------------------------------- @@ -336,6 +343,7 @@ def pyproject_to_conda_lists( sort: bool = True, deps: Sequence[str] | None = None, reqs: Sequence[str] | None = None, + remove_whitespace: bool = True, ) -> dict[str, Any]: if python_include == "infer": # safer get @@ -374,6 +382,9 @@ def pyproject_to_conda_lists( output["dependencies"], python_version ) + if remove_whitespace: + output = {k: [_remove_whitespace(x) for x in v] for k, v in output.items()} + return output @@ -391,6 +402,7 @@ def pyproject_to_conda( deps: Sequence[str] | None = None, reqs: Sequence[str] | None = None, allow_empty: bool = False, + remove_whitespace: bool = True, ) -> str: output = pyproject_to_conda_lists( data=data, @@ -402,6 +414,7 @@ def pyproject_to_conda( sort=sort, deps=deps, reqs=reqs, + remove_whitespace=remove_whitespace, ) return _output_to_yaml( **output, @@ -550,6 +563,7 @@ def to_conda_yaml( deps: Sequence[str] | None = None, reqs: Sequence[str] | None = None, allow_empty: bool = False, + remove_whitespace: bool = True, ) -> str: self._check_extras(extras) @@ -567,6 +581,7 @@ def to_conda_yaml( deps=deps, reqs=reqs, allow_empty=allow_empty, + remove_whitespace=remove_whitespace, ) def to_conda_lists( @@ -579,6 +594,7 @@ def to_conda_lists( sort: bool = True, deps: Sequence[str] | None = None, reqs: Sequence[str] | None = None, + remove_whitespace: bool = True, ) -> dict[str, Any]: self._check_extras(extras) @@ -592,6 +608,7 @@ def to_conda_lists( sort=sort, deps=deps, reqs=reqs, + remove_whitespace=remove_whitespace, ) def to_requirement_list( @@ -600,6 +617,7 @@ def to_requirement_list( include_base_dependencies: bool = True, sort: bool = True, reqs: Sequence[str] | None = None, + remove_whitespace: bool = True, ) -> list[str]: self._check_extras(extras) @@ -613,6 +631,9 @@ def to_requirement_list( if reqs: out.extend(list(reqs)) + if remove_whitespace: + out = [_remove_whitespace(x) for x in out] + if sort: return sorted(out) else: @@ -627,6 +648,7 @@ def to_requirements( sort: bool = True, reqs: Sequence[str] | None = None, allow_empty: bool = False, + remove_whitespace: bool = True, ) -> str: """Create requirements.txt like file with pip dependencies.""" @@ -637,6 +659,7 @@ def to_requirements( include_base_dependencies=include_base_dependencies, sort=sort, reqs=reqs, + remove_whitespace=remove_whitespace, ) if not reqs: @@ -660,6 +683,7 @@ def to_conda_requirements( sort: bool = True, deps: Sequence[str] | None = None, reqs: Sequence[str] | None = None, + remove_whitespace: bool = True, ) -> tuple[str, str]: output = self.to_conda_lists( extras=extras, @@ -670,6 +694,7 @@ def to_conda_requirements( sort=sort, deps=deps, reqs=reqs, + remove_whitespace=remove_whitespace, ) deps = output.get("dependencies", None) diff --git a/tests/test_cli.py b/tests/test_cli.py index 426d8b4..c92166e 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -353,7 +353,7 @@ def test_requirements(): expected = """\ athing bthing -cthing; python_version < '3.10' +cthing;python_version<'3.10' """ for cmd in ["r", "requirements"]: @@ -363,7 +363,7 @@ def test_requirements(): expected = """\ athing bthing -cthing; python_version < '3.10' +cthing;python_version<'3.10' pandas pytest matplotlib @@ -378,7 +378,7 @@ def test_requirements(): expected = """\ athing bthing -cthing; python_version < '3.10' +cthing;python_version<'3.10' pandas pytest matplotlib @@ -403,7 +403,7 @@ def test_requirements(): expected = """\ athing bthing -cthing; python_version < '3.10' +cthing;python_version<'3.10' matplotlib pandas pytest @@ -418,7 +418,7 @@ def test_requirements(): expected = """\ athing bthing -cthing; python_version < '3.10' +cthing;python_version<'3.10' matplotlib other pandas @@ -439,6 +439,32 @@ def test_requirements(): check_result(result, expected) + # allow whitespace: + expected = """\ +athing +bthing +cthing; python_version < '3.10' +matplotlib +other +pandas +pytest +thing; python_version < '3.10' + """ + + result = do_run( + runner, + "requirements", + "-e", + "dev", + "-r", + "thing; python_version < '3.10'", + "-r", + "other", + "--no-remove-whitespace", + ) + + check_result(result, expected) + expected = """\ setuptools build diff --git a/tests/test_config.py b/tests/test_config.py index b26d5d9..858bd44 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -98,6 +98,7 @@ def test_option_override(): "name": None, "channels": ["conda-forge"], "allow_empty": False, + "remove_whitespace": True, "output": "hello-base.yaml", }, ) @@ -118,6 +119,7 @@ def test_option_override(): "name": None, "channels": ["conda-forge"], "allow_empty": False, + "remove_whitespace": True, "output": "there-base.yaml", }, ) @@ -138,6 +140,35 @@ def test_option_override(): "name": None, "channels": ["conda-forge"], "allow_empty": True, + "remove_whitespace": True, + "output": "there-base.yaml", + }, + ) + + output = list( + d.iter( + envs=["base"], + allow_empty=True, + remove_whitespace=False, + template="there-{env}", + ) + ) + + assert output[0] == ( + "yaml", + { + "extras": [], + "sort": True, + "base": True, + "header": None, + "overwrite": "check", + "verbose": None, + "reqs": None, + "deps": None, + "name": None, + "channels": ["conda-forge"], + "allow_empty": True, + "remove_whitespace": False, "output": "there-base.yaml", }, ) @@ -192,6 +223,7 @@ def test_config_only_default(): "deps": None, "reqs": None, "allow_empty": False, + "remove_whitespace": True, }, ) ] @@ -251,6 +283,7 @@ def test_config_overrides(): "deps": None, "reqs": None, "allow_empty": False, + "remove_whitespace": True, }, ) @@ -308,6 +341,7 @@ def test_config_python_include_version(): "deps": None, "reqs": None, "allow_empty": False, + "remove_whitespace": True, }, ), ( @@ -327,6 +361,7 @@ def test_config_python_include_version(): "deps": None, "reqs": None, "allow_empty": False, + "remove_whitespace": True, }, ), ] @@ -374,6 +409,7 @@ def test_config_user_config(): "deps": None, "reqs": None, "allow_empty": False, + "remove_whitespace": True, }, ), ( @@ -392,6 +428,7 @@ def test_config_user_config(): "deps": None, "reqs": None, "allow_empty": False, + "remove_whitespace": True, }, ), ] diff --git a/tests/test_parser.py b/tests/test_parser.py index 26a3bf0..399c549 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -273,7 +273,7 @@ def test_complete(): """\ [project] name = "hello" - requires-python = ">=3.8,<3.11" + requires-python = ">=3.8, <3.11" dependencies = [ "athing", # p2c: -p # a comment "bthing", # p2c: -s bthing-conda @@ -342,6 +342,22 @@ def test_complete(): """ assert dedent(expected) == d.to_conda_yaml(python_include="infer") + # with remove spaces: + expected = """\ +channels: + - conda-forge +dependencies: + - python>=3.8, <3.11 + - bthing-conda + - conda-forge::cthing + - pip + - pip: + - athing + """ + assert dedent(expected) == d.to_conda_yaml( + python_include="infer", remove_whitespace=False + ) + expected = """\ channels: - conda-forge @@ -595,3 +611,70 @@ def test_missing_dependencies(): f() assert f(allow_empty=True) == "No dependencies for this environment\n" + + +def test_spaces(): + toml = dedent( + """\ + [project] + name = "hello" + requires-python = ">=3.8, <3.11" + dependencies = [ + "athing >0.5", # p2c: -p # a comment + "bthing > 1.0", # p2c: -s 'bthing-conda > 2.0' + "cthing; python_version < '3.10'", # p2c: -c conda-forge + ] + + + [tool.pyproject2conda] + channels = ['conda-forge'] + """ + ) + + d = parser.PyProject2Conda.from_string(toml) + + expected = """\ +channels: + - conda-forge +dependencies: + - python>=3.8, <3.11 + - bthing-conda > 2.0 + - conda-forge::cthing + - pip + - pip: + - athing >0.5 + """ + assert dedent(expected) == d.to_conda_yaml( + python_include="infer", remove_whitespace=False + ) + + expected = """\ +channels: + - conda-forge +dependencies: + - python>=3.8,<3.11 + - bthing-conda>2.0 + - conda-forge::cthing + - pip + - pip: + - athing>0.5 + """ + assert dedent(expected) == d.to_conda_yaml( + python_include="infer", remove_whitespace=True + ) + + expected = """\ + athing >0.5 + bthing > 1.0 + cthing; python_version < '3.10' + """ + + assert dedent(expected) == d.to_requirements(remove_whitespace=False) + + expected = """\ + athing>0.5 + bthing>1.0 + cthing;python_version<'3.10' + """ + + assert dedent(expected) == d.to_requirements(remove_whitespace=True) From 9ed14ce19d79821d5df68f261bacfaf3a30d27ca Mon Sep 17 00:00:00 2001 From: wpk Date: Mon, 13 Nov 2023 15:59:47 -0500 Subject: [PATCH 12/36] feat: bump maximum tested version to 3.12 --- noxfile.py | 2 +- pyproject.toml | 3 ++- requirements/py312-test-extras.yaml | 17 +++++++++++++++++ requirements/py312-test.yaml | 22 ++++++++++++++++++++++ requirements/py312-typing.yaml | 24 ++++++++++++++++++++++++ 5 files changed, 66 insertions(+), 2 deletions(-) create mode 100644 requirements/py312-test-extras.yaml create mode 100644 requirements/py312-test.yaml create mode 100644 requirements/py312-typing.yaml diff --git a/noxfile.py b/noxfile.py index 229bc96..1c86d11 100644 --- a/noxfile.py +++ b/noxfile.py @@ -74,7 +74,7 @@ # * Options ---------------------------------------------------------------------------- -PYTHON_ALL_VERSIONS = ["3.8", "3.9", "3.10", "3.11"] +PYTHON_ALL_VERSIONS = ["3.8", "3.9", "3.10", "3.11", "3.12"] PYTHON_DEFAULT_VERSION = "3.10" # conda/mamba diff --git a/pyproject.toml b/pyproject.toml index 968aa4f..eea517d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,6 +21,7 @@ classifiers = [ "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", "Topic :: Scientific/Engineering", ] dynamic = ["readme", "version"] @@ -140,7 +141,7 @@ base = false [[tool.pyproject2conda.overrides]] envs = ["test", "typing", "test-extras"] -python = ["3.8", "3.9", "3.10", "3.11"] +python = ["3.8", "3.9", "3.10", "3.11", "3.12"] [[tool.pyproject2conda.overrides]] envs = ["dist-conda"] diff --git a/requirements/py312-test-extras.yaml b/requirements/py312-test-extras.yaml new file mode 100644 index 0000000..8b891d3 --- /dev/null +++ b/requirements/py312-test-extras.yaml @@ -0,0 +1,17 @@ +# +# This file is autogenerated by pyproject2conda +# with the following command: +# +# $ pyproject2conda project --verbose +# +# You should not manually edit this file. +# Instead edit the corresponding pyproject.toml file. +# +channels: + - conda-forge +dependencies: + - python=3.12 + - pytest + - pytest-cov + - pytest-sugar + - pytest-xdist diff --git a/requirements/py312-test.yaml b/requirements/py312-test.yaml new file mode 100644 index 0000000..ace8f0c --- /dev/null +++ b/requirements/py312-test.yaml @@ -0,0 +1,22 @@ +# +# This file is autogenerated by pyproject2conda +# with the following command: +# +# $ pyproject2conda project --verbose +# +# You should not manually edit this file. +# Instead edit the corresponding pyproject.toml file. +# +channels: + - conda-forge +dependencies: + - python=3.12 + - packaging + - pytest + - pytest-cov + - pytest-sugar + - pytest-xdist + - ruamel.yaml + - tomli + - tomlkit + - typer diff --git a/requirements/py312-typing.yaml b/requirements/py312-typing.yaml new file mode 100644 index 0000000..c6e3382 --- /dev/null +++ b/requirements/py312-typing.yaml @@ -0,0 +1,24 @@ +# +# This file is autogenerated by pyproject2conda +# with the following command: +# +# $ pyproject2conda project --verbose +# +# You should not manually edit this file. +# Instead edit the corresponding pyproject.toml file. +# +channels: + - conda-forge +dependencies: + - python=3.12 + - mypy>=1.4.1 + - packaging + - pytest + - pytest-cov + - pytest-sugar + - pytest-xdist + - ruamel.yaml + - tomli + - tomlkit + - typer + - types-click From f384970faffe6bcdc5ce2f0c49b17b28dbf5c586 Mon Sep 17 00:00:00 2001 From: wpk Date: Mon, 13 Nov 2023 16:37:38 -0500 Subject: [PATCH 13/36] chore: add strict pyright for the code --- pyproject.toml | 1 + src/pyproject2conda/cli.py | 13 +++++++------ src/pyproject2conda/config.py | 8 +++++--- src/pyproject2conda/parser.py | 9 ++++++--- src/pyproject2conda/utils.py | 6 ++++-- 5 files changed, 23 insertions(+), 14 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index eea517d..9a7eba3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -361,6 +361,7 @@ module = [] [tool.pyright] include = ["src", "tests"] exclude = ["**/__pycache__", ".tox/**", ".nox/**", "**/.mypy_cache"] +strict = ["src/**/*.py"] pythonVersion = "3.10" typeCheckingMode = "basic" # enable subset of "strict" diff --git a/src/pyproject2conda/cli.py b/src/pyproject2conda/cli.py index f178056..7ecc682 100644 --- a/src/pyproject2conda/cli.py +++ b/src/pyproject2conda/cli.py @@ -1,3 +1,4 @@ +# pyright: reportUnknownMemberType=false """ Console script for pyproject2conda (:mod:`~pyproject2conda.cli`) ================================================================ @@ -108,7 +109,7 @@ def main( PYPROJECT_CLI = Annotated[ Path, - typer.Option( + typer.Option( # pyright: ignore[reportUnknownMemberType] "--file", "-f", help="input pyproject.toml file", @@ -116,7 +117,7 @@ def main( ] EXTRAS_CLI = Annotated[ Optional[List[str]], - typer.Option( + typer.Option( # pyright: ignore[reportUnknownMemberType] "--extra", "-e", help="Extra dependencies. Can specify multiple times for multiple extras.", @@ -124,7 +125,7 @@ def main( ] CHANNEL_CLI = Annotated[ Optional[List[str]], - typer.Option( + typer.Option( # pyright: ignore[reportUnknownMemberType] "--channel", "-c", help="conda channel. Can specify. Overrides [tool.pyproject2conda.channels]", @@ -132,7 +133,7 @@ def main( ] NAME_CLI = Annotated[ Optional[str], - typer.Option( + typer.Option( # pyright: ignore[reportUnknownMemberType] "--name", "-n", help="Name of conda env", @@ -140,7 +141,7 @@ def main( ] OUTPUT_CLI = Annotated[ Optional[Path], - typer.Option( + typer.Option( # pyright: ignore[reportUnknownMemberType] "--output", "-o", help="File to output results", @@ -417,7 +418,7 @@ def wrapped(*args: Any, **kwargs: Any) -> R: level = logging.WARN elif verbosity == 1: level = logging.INFO - elif verbosity >= 2: # pragma: no cover + else: # pragma: no cover level = logging.DEBUG logger.setLevel(level) diff --git a/src/pyproject2conda/config.py b/src/pyproject2conda/config.py index b257be1..547dc5a 100644 --- a/src/pyproject2conda/config.py +++ b/src/pyproject2conda/config.py @@ -98,7 +98,7 @@ def _get_value( if not isinstance(value, list): value = [value] - return value + return value # pyright: ignore def channels( self, env_name: str | None = None, inherit: bool = True @@ -250,9 +250,11 @@ def assign_user_config(self, user: Self) -> Self: if u is not None: d = data[key] if isinstance(d, list): - d.extend(u) + assert isinstance(u, list) + d.extend(u) # pyright: ignore elif isinstance(d, dict): - d.update(**u) + assert isinstance(u, dict) + d.update(**u) # pyright: ignore return type(self)(data) diff --git a/src/pyproject2conda/parser.py b/src/pyproject2conda/parser.py index f807a14..3751714 100644 --- a/src/pyproject2conda/parser.py +++ b/src/pyproject2conda/parser.py @@ -1,3 +1,4 @@ +# pyright: reportUnknownMemberType=false, reportGeneralTypeIssues=false """ Parse `pyproject.toml` (:mod:`~pyproject2conda.parser`) ======================================================= @@ -25,6 +26,8 @@ ) if TYPE_CHECKING: + import tomlkit.items + import tomlkit.toml_document from typing_extensions import Self import tomlkit @@ -210,7 +213,7 @@ def _pyproject_to_value_comment_pairs( unique: bool = True, include_base_dependencies: bool = True, ) -> list[tuple[Tstr_opt, Tstr_opt]]: - package_name = cast(str, get_in(["project", "name"], data, default=None)) + package_name = cast("str | None", get_in(["project", "name"], data, default=None)) if package_name is None: raise ValueError("Must specify `project.package_name` in pyproject.toml") @@ -276,7 +279,7 @@ def _limit_deps_by_python_version( r"(?P.*?);\s*python_version\s*(?P[<=>~]*)\s*[\'|\"](?P.*?)[\'|\"]" ) - output = [] + output: list[str] = [] for dep in deps: if match := matcher.match(dep): if not version or version in SpecifierSet( @@ -626,7 +629,7 @@ def to_requirement_list( extras=extras, include_base_dependencies=include_base_dependencies, ) - out = [x for x, y in values if x is not None] + out = [x for x, _ in values if x is not None] if reqs: out.extend(list(reqs)) diff --git a/src/pyproject2conda/utils.py b/src/pyproject2conda/utils.py index 78290b0..8561ebe 100644 --- a/src/pyproject2conda/utils.py +++ b/src/pyproject2conda/utils.py @@ -88,7 +88,7 @@ def update_target( update = True else: # check times - deps_filtered = [] + deps_filtered: list[Path] = [] for d in map(Path, deps): if d.exists(): deps_filtered.append(d) @@ -96,6 +96,8 @@ def update_target( target_time = target.stat().st_mtime update = any(target_time < dep.stat().st_mtime for dep in deps_filtered) + else: + raise ValueError(f"unknown option overwrite={overwrite}") return update @@ -121,7 +123,7 @@ def filename_from_template( if template is None: return None - kws = {} + kws: dict[str, str] = {} if python: py_version = python elif python_version: From b41fa08d8ad6b46a50cbda6e64fb4aca5e2e1f4f Mon Sep 17 00:00:00 2001 From: wpk Date: Mon, 13 Nov 2023 18:57:31 -0500 Subject: [PATCH 14/36] chore: moved python version specific imports for typing to _typing_compat.py --- src/pyproject2conda/_typing.py | 2 +- src/pyproject2conda/_typing_compat.py | 25 +++++++++++++++++++++++++ src/pyproject2conda/cli.py | 7 +------ src/pyproject2conda/config.py | 2 +- src/pyproject2conda/parser.py | 3 ++- 5 files changed, 30 insertions(+), 9 deletions(-) create mode 100644 src/pyproject2conda/_typing_compat.py diff --git a/src/pyproject2conda/_typing.py b/src/pyproject2conda/_typing.py index a096bc5..8fce015 100644 --- a/src/pyproject2conda/_typing.py +++ b/src/pyproject2conda/_typing.py @@ -1,6 +1,6 @@ from typing import Any, Callable, TypeVar -from typing_extensions import TypeAlias +from ._typing_compat import TypeAlias FuncType: TypeAlias = Callable[..., Any] diff --git a/src/pyproject2conda/_typing_compat.py b/src/pyproject2conda/_typing_compat.py new file mode 100644 index 0000000..7425efb --- /dev/null +++ b/src/pyproject2conda/_typing_compat.py @@ -0,0 +1,25 @@ +import sys + +if sys.version_info < (3, 10): + from typing_extensions import TypeAlias +else: + from typing import TypeAlias + + +if sys.version_info < (3, 9): + from typing_extensions import Annotated +else: + from typing import Annotated + + +if sys.version_info < (3, 11): + from typing_extensions import Self +else: + from typing import Self + + +__all__ = [ + "TypeAlias", + "Annotated", + "Self", +] diff --git a/src/pyproject2conda/cli.py b/src/pyproject2conda/cli.py index 7ecc682..f564e68 100644 --- a/src/pyproject2conda/cli.py +++ b/src/pyproject2conda/cli.py @@ -7,18 +7,12 @@ import logging import os -import sys from enum import Enum from functools import lru_cache, wraps from inspect import signature from pathlib import Path from typing import Any, Callable, Iterable, List, Optional, Union, cast -if sys.version_info[:2] < (3, 9): - from typing_extensions import Annotated -else: - from typing import Annotated - # from click import click.Context import click import typer @@ -32,6 +26,7 @@ ) from ._typing import R +from ._typing_compat import Annotated # * Logger ----------------------------------------------------------------------------- diff --git a/src/pyproject2conda/config.py b/src/pyproject2conda/config.py index 547dc5a..dbcf312 100644 --- a/src/pyproject2conda/config.py +++ b/src/pyproject2conda/config.py @@ -14,7 +14,7 @@ if TYPE_CHECKING: from typing import Any, Iterator, Sequence - from typing_extensions import Self + from ._typing_compat import Self # * Utilities diff --git a/src/pyproject2conda/parser.py b/src/pyproject2conda/parser.py index 3751714..beba1e1 100644 --- a/src/pyproject2conda/parser.py +++ b/src/pyproject2conda/parser.py @@ -28,7 +28,8 @@ if TYPE_CHECKING: import tomlkit.items import tomlkit.toml_document - from typing_extensions import Self + + from ._typing_compat import Self import tomlkit from packaging.specifiers import SpecifierSet From 1ce3969d93065f2afe3c9df9168dbb2f89e94e23 Mon Sep 17 00:00:00 2001 From: wpk Date: Mon, 13 Nov 2023 19:00:45 -0500 Subject: [PATCH 15/36] chore: update changelog snippet --- .../20231113_155311_william.krekelberg_ignore_whitespace.md | 1 + 1 file changed, 1 insertion(+) diff --git a/changelog.d/20231113_155311_william.krekelberg_ignore_whitespace.md b/changelog.d/20231113_155311_william.krekelberg_ignore_whitespace.md index 1aab19c..f5055ed 100644 --- a/changelog.d/20231113_155311_william.krekelberg_ignore_whitespace.md +++ b/changelog.d/20231113_155311_william.krekelberg_ignore_whitespace.md @@ -17,6 +17,7 @@ Uncomment the section that is right (remove the HTML comment wrapper). - Default is now to remove whitespace from dependencies. For example, the dependency `module > 0.1` will become `module>0.1`. To override this behaviour, pass the option `--no-remove-whitespace`. +- Now supports python version `>3.8,<=3.12` - + ```bash $ p2c project -f tests/data/test-pyproject.toml --dry diff --git a/changelog.d/20231113_155311_william.krekelberg_ignore_whitespace.md b/changelog.d/20231113_155311_william.krekelberg_ignore_whitespace.md index f5055ed..a47ae32 100644 --- a/changelog.d/20231113_155311_william.krekelberg_ignore_whitespace.md +++ b/changelog.d/20231113_155311_william.krekelberg_ignore_whitespace.md @@ -18,6 +18,10 @@ Uncomment the section that is right (remove the HTML comment wrapper). dependency `module > 0.1` will become `module>0.1`. To override this behaviour, pass the option `--no-remove-whitespace`. - Now supports python version `>3.8,<=3.12` +- Can now specify `extras = false` in pyprojec.toml to skip any extras. The + default (`extras = true`) is the same as `extras = [env_name]` where + `env_name` is the name of the environment (e.g., + `tool.pyproject2conda.envs.env_name`). +## v0.9.0 — 2023-11-14 + +### Added + +- Default is now to remove whitespace from dependencies. For example, the + dependency `module > 0.1` will become `module>0.1`. To override this + behaviour, pass the option `--no-remove-whitespace`. +- Now supports python version `>3.8,<=3.12` +- Can now specify `extras = false` in pyprojec.toml to skip any extras. The + default (`extras = true`) is the same as `extras = [env_name]` where + `env_name` is the name of the environment (e.g., + `tool.pyproject2conda.envs.env_name`). + ## v0.8.0 — 2023-10-02 ### Added diff --git a/changelog.d/20231113_155311_william.krekelberg_ignore_whitespace.md b/changelog.d/20231113_155311_william.krekelberg_ignore_whitespace.md deleted file mode 100644 index a47ae32..0000000 --- a/changelog.d/20231113_155311_william.krekelberg_ignore_whitespace.md +++ /dev/null @@ -1,49 +0,0 @@ - - - - - -### Added - -- Default is now to remove whitespace from dependencies. For example, the - dependency `module > 0.1` will become `module>0.1`. To override this - behaviour, pass the option `--no-remove-whitespace`. -- Now supports python version `>3.8,<=3.12` -- Can now specify `extras = false` in pyprojec.toml to skip any extras. The - default (`extras = true`) is the same as `extras = [env_name]` where - `env_name` is the name of the environment (e.g., - `tool.pyproject2conda.envs.env_name`). - - - - - From 2cc720b27e78950a586c73dc8b6ed610e7fdb04f Mon Sep 17 00:00:00 2001 From: wpk Date: Tue, 14 Nov 2023 12:05:02 -0500 Subject: [PATCH 19/36] chore: rerun cog --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 54d18e4..d512ecb 100644 --- a/README.md +++ b/README.md @@ -515,10 +515,10 @@ style = ["requirements"] # or # extras = false + # # A value of `extras = true` also implies using the environment name # as the extras. - [tool.pyproject2conda.envs."test-extras"] extras = ["test"] style = ["yaml", "requirements"] @@ -630,10 +630,10 @@ style = ["requirements"] # or # extras = false + # # A value of `extras = true` also implies using the environment name # as the extras. - [tool.pyproject2conda.envs."test-extras"] extras = ["test"] style = ["yaml", "requirements"] From ff76897683018ff92c266fa742cac6a1f53638a7 Mon Sep 17 00:00:00 2001 From: wpk Date: Wed, 15 Nov 2023 08:53:12 -0500 Subject: [PATCH 20/36] chore: replace extras extraction using regex with `packaging.requirements` module --- src/pyproject2conda/parser.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/src/pyproject2conda/parser.py b/src/pyproject2conda/parser.py index beba1e1..26677f0 100644 --- a/src/pyproject2conda/parser.py +++ b/src/pyproject2conda/parser.py @@ -38,7 +38,7 @@ from pyproject2conda.utils import get_in -# -- typing ---------------------------------------------------------------------------- +# * Typing ----------------------------------------------------------------------------- Tstr_opt = Optional[str] Tstr_seq_opt = Optional[Union[str, Sequence[str]]] @@ -46,7 +46,7 @@ T = TypeVar("T") -# -- Utilities ------------------------------------------------------------------------- +# * Utilities -------------------------------------------------------------------------- def _check_allow_empty(allow_empty: bool) -> str: @@ -64,7 +64,7 @@ def _remove_whitespace(s: str) -> str: return re.sub(_WHITE_SPACE_REGEX, "", s) -# --- Default parser ------------------------------------------------------------------- +# * Default parser --------------------------------------------------------------------- @lru_cache @@ -153,17 +153,18 @@ def _matches_package_name( If it does, return extras, else return None """ - if not dep: - return None - - pattern = rf"{package_name}\[(.*?)\]" - match = re.match(pattern, dep) + extras = None - if match: - extras = match.group(1).split(",") else: - extras = None + from packaging.requirements import Requirement + + r = Requirement(dep) + + if r.name == package_name and r.extras: + extras = list(r.extras) + else: + extras = None return extras From 0c41eea8296b8dd07c14679ae6870eb24e656ff1 Mon Sep 17 00:00:00 2001 From: wpk Date: Wed, 15 Nov 2023 10:39:15 -0500 Subject: [PATCH 21/36] chore: cleanup how python_version specifier is handled --- src/pyproject2conda/_typing.py | 30 +++- src/pyproject2conda/parser.py | 242 +++++++++++++++------------------ tests/test_parser.py | 2 +- 3 files changed, 139 insertions(+), 135 deletions(-) diff --git a/src/pyproject2conda/_typing.py b/src/pyproject2conda/_typing.py index 8fce015..3220b98 100644 --- a/src/pyproject2conda/_typing.py +++ b/src/pyproject2conda/_typing.py @@ -1,10 +1,30 @@ -from typing import Any, Callable, TypeVar +# from typing import Any, Callable, TypeVar -from ._typing_compat import TypeAlias +# from ._typing_compat import TypeAlias + +# FuncType: TypeAlias = Callable[..., Any] + +# F = TypeVar("F", bound=FuncType) +# R = TypeVar("R") + +# Dec: TypeAlias = Callable[[F], F] + + +# Tstr_opt = Optional[str] +# Tstr_seq_opt = Optional[Union[str, Sequence[str]]] + +# T = TypeVar("T") + +from __future__ import annotations + +from typing import TYPE_CHECKING, Sequence, TypeVar + +if TYPE_CHECKING: + from ._typing_compat import TypeAlias -FuncType: TypeAlias = Callable[..., Any] -F = TypeVar("F", bound=FuncType) R = TypeVar("R") +T = TypeVar("T") -Dec: TypeAlias = Callable[[F], F] +OptStr: TypeAlias = str | None +OptStrSeq: TypeAlias = str | Sequence[str] | None diff --git a/src/pyproject2conda/parser.py b/src/pyproject2conda/parser.py index 26677f0..1cee115 100644 --- a/src/pyproject2conda/parser.py +++ b/src/pyproject2conda/parser.py @@ -17,11 +17,8 @@ Any, Generator, Iterable, - Optional, Sequence, TextIO, - TypeVar, - Union, cast, ) @@ -29,23 +26,15 @@ import tomlkit.items import tomlkit.toml_document + from ._typing import OptStr, OptStrSeq, T from ._typing_compat import Self import tomlkit -from packaging.specifiers import SpecifierSet -from packaging.version import Version +from packaging.requirements import Requirement from ruamel.yaml import YAML from pyproject2conda.utils import get_in -# * Typing ----------------------------------------------------------------------------- - -Tstr_opt = Optional[str] -Tstr_seq_opt = Optional[Union[str, Sequence[str]]] - -T = TypeVar("T") - - # * Utilities -------------------------------------------------------------------------- @@ -64,6 +53,29 @@ def _remove_whitespace(s: str) -> str: return re.sub(_WHITE_SPACE_REGEX, "", s) +def _unique_list(values: Iterable[T]) -> list[T]: + """ + Return only unique values in list. + Unlike using set(values), this preserves order. + """ + output: list[T] = [] + for v in values: + if v not in output: + output.append(v) + return output + + +def _list_to_str(values: Iterable[str] | None, eol: bool = True) -> str: + if values: + output = "\n".join(values) + if eol: + output += "\n" + else: + output = "" + + return output + + # * Default parser --------------------------------------------------------------------- @@ -101,32 +113,9 @@ def _factory_empty_tomlkit_Array() -> tomlkit.items.Array: return tomlkit.items.Array([], tomlkit.items.Trivia()) -def _unique_list(values: Iterable[T]) -> list[T]: - """ - Return only unique values in list. - Unlike using set(values), this preserves order. - """ - output: list[T] = [] - for v in values: - if v not in output: - output.append(v) - return output - - -def _list_to_str(values: Iterable[str] | None, eol: bool = True) -> str: - if values: - output = "\n".join(values) - if eol: - output += "\n" - else: - output = "" - - return output - - def _iter_value_comment_pairs( array: tomlkit.items.Array, # pyright: ignore -) -> Generator[tuple[Tstr_opt, Tstr_opt], None, None]: +) -> Generator[tuple[OptStr, OptStr], None, None]: """Extract value and comments from array""" for v in array._value: # pyright: ignore if v.value is not None and not isinstance( @@ -145,7 +134,7 @@ def _iter_value_comment_pairs( def _matches_package_name( - dep: Tstr_opt, + dep: OptStr, package_name: str, ) -> list[str] | None: """ @@ -157,8 +146,6 @@ def _matches_package_name( extras = None else: - from packaging.requirements import Requirement - r = Requirement(dep) if r.name == package_name and r.extras: @@ -171,10 +158,10 @@ def _matches_package_name( def _get_value_comment_pairs( package_name: str, deps: tomlkit.items.Array, - extras: Tstr_seq_opt = None, + extras: OptStrSeq = None, opts: tomlkit.items.Table | None = None, include_base_dependencies: bool = True, -) -> list[tuple[Tstr_opt, Tstr_opt]]: +) -> list[tuple[OptStr, OptStr]]: """Recursively build dependency, comment pairs from deps and extras.""" if include_base_dependencies: out = list(_iter_value_comment_pairs(deps)) @@ -211,10 +198,10 @@ def _get_value_comment_pairs( def _pyproject_to_value_comment_pairs( data: tomlkit.toml_document.TOMLDocument, - extras: Tstr_seq_opt = None, + extras: OptStrSeq = None, unique: bool = True, include_base_dependencies: bool = True, -) -> list[tuple[Tstr_opt, Tstr_opt]]: +) -> list[tuple[OptStr, OptStr]]: package_name = cast("str | None", get_in(["project", "name"], data, default=None)) if package_name is None: @@ -243,7 +230,7 @@ def _pyproject_to_value_comment_pairs( return value_comment_list -def _match_p2c_comment(comment: Tstr_opt) -> Tstr_opt: +def _match_p2c_comment(comment: OptStr) -> OptStr: if not comment or not (match := re.match(r".*?#\s*p2c:\s*([^\#]*)", comment)): return None elif re.match(r".*?##\s*p2c:", comment): @@ -253,7 +240,7 @@ def _match_p2c_comment(comment: Tstr_opt) -> Tstr_opt: return match.group(1).strip() -def _parse_p2c(match: Tstr_opt) -> dict[str, Any] | None: +def _parse_p2c(match: OptStr) -> dict[str, Any] | None: """Parse match from _match_p2c_comment""" if match: @@ -262,42 +249,49 @@ def _parse_p2c(match: Tstr_opt) -> dict[str, Any] | None: return None -def parse_p2c_comment(comment: Tstr_opt) -> dict[str, Any] | None: +def parse_p2c_comment(comment: OptStr) -> dict[str, Any] | None: if match := _match_p2c_comment(comment): return _parse_p2c(match) else: return None -def _limit_deps_by_python_version( - deps: list[str], python_version: Tstr_opt = None -) -> list[str]: - if python_version: - version = Version(python_version) +def _limit_dep_by_python_version( + dep: str, + python_version: str | None = None, +) -> str | None: + """Limit by python version""" + r = Requirement(dep) + + if ( + r.marker + and python_version + and (not r.marker.evaluate({"python_version": python_version})) + ): + return None else: - version = None + r.marker = None + r.extras = None + return str(r) - matcher = re.compile( - r"(?P.*?);\s*python_version\s*(?P[<=>~]*)\s*[\'|\"](?P.*?)[\'|\"]" - ) - output: list[str] = [] +def _limit_deps_by_python_version( + deps: Iterable[str], + python_version: str | None = None, +) -> list[str]: + out: list[str] = [] for dep in deps: - if match := matcher.match(dep): - if not version or version in SpecifierSet( - match["token"] + match["version"] - ): - output.append(match["dep"]) - else: - output.append(dep) - return output + if (v := _limit_dep_by_python_version(dep, python_version)) is not None: + out.append(v) + return out def value_comment_pairs_to_conda( - value_comment_list: list[tuple[Tstr_opt, Tstr_opt]], + value_comment_list: list[tuple[OptStr, OptStr]], sort: bool = True, deps: Sequence[str] | None = None, reqs: Sequence[str] | None = None, + python_version: str | None = None, ) -> dict[str, Any]: """Convert raw value/comment pairs to install lines""" @@ -315,18 +309,20 @@ def _check_value(value: Any) -> None: pip_deps.append(value) # type: ignore elif not parsed["skip"]: _check_value(value) - - if parsed["channel"]: - conda_deps.append("{}::{}".format(parsed["channel"], value)) - else: - conda_deps.append(value) # type: ignore - - conda_deps.extend(parsed["package"]) + if v := _limit_dep_by_python_version(value, python_version): + if parsed["channel"]: + v = "{}::{}".format(parsed["channel"], v) + conda_deps.append(v) + + conda_deps.extend( + _limit_deps_by_python_version(parsed["package"], python_version) + ) elif value: - conda_deps.append(value) + if v := _limit_dep_by_python_version(value, python_version): + conda_deps.append(v) if deps: - conda_deps.extend(list(deps)) + conda_deps.extend(_limit_deps_by_python_version(deps, python_version)) if reqs: pip_deps.extend(list(reqs)) @@ -340,10 +336,10 @@ def _check_value(value: Any) -> None: def pyproject_to_conda_lists( data: tomlkit.toml_document.TOMLDocument, - extras: Tstr_seq_opt = None, - channels: Tstr_seq_opt = None, - python_include: Tstr_opt = None, - python_version: Tstr_opt = None, + extras: OptStrSeq = None, + channels: OptStrSeq = None, + python_include: OptStr = None, + python_version: OptStr = None, include_base_dependencies: bool = True, sort: bool = True, deps: Sequence[str] | None = None, @@ -375,6 +371,7 @@ def pyproject_to_conda_lists( sort=sort, deps=deps, reqs=reqs, + python_version=python_version, ) if python_include: @@ -382,10 +379,10 @@ def pyproject_to_conda_lists( if channels: output["channels"] = channels - # limit python version/remove python_verions <=> part - output["dependencies"] = _limit_deps_by_python_version( - output["dependencies"], python_version - ) + # # limit python version/remove python_verions <=> part + # output["dependencies"] = _limit_deps_by_python_version( + # output["dependencies"], python_version + # ) if remove_whitespace: output = {k: [_remove_whitespace(x) for x in v] for k, v in output.items()} @@ -395,14 +392,14 @@ def pyproject_to_conda_lists( def pyproject_to_conda( data: tomlkit.toml_document.TOMLDocument, - extras: Tstr_seq_opt = None, - channels: Tstr_seq_opt = None, - name: Tstr_opt = None, - python_include: Tstr_opt = None, + extras: OptStrSeq = None, + channels: OptStrSeq = None, + name: OptStr = None, + python_include: OptStr = None, stream: str | Path | TextIO | None = None, - python_version: Tstr_opt = None, + python_version: OptStr = None, include_base_dependencies: bool = True, - header_cmd: Tstr_opt = None, + header_cmd: OptStr = None, sort: bool = True, deps: Sequence[str] | None = None, reqs: Sequence[str] | None = None, @@ -511,7 +508,7 @@ def _output_to_yaml( dependencies: list[str] | None, channels: list[str] | None = None, pip: list[str] | None = None, - name: Tstr_opt = None, + name: OptStr = None, stream: str | Path | TextIO | None = None, header_cmd: str | None = None, allow_empty: bool = False, @@ -545,9 +542,9 @@ class PyProject2Conda: def __init__( self, data: tomlkit.toml_document.TOMLDocument, - name: Tstr_opt = None, - channels: Tstr_seq_opt = None, - python_include: Tstr_opt = None, + name: OptStr = None, + channels: OptStrSeq = None, + python_include: OptStr = None, ) -> None: self.data = data self.name = name @@ -556,12 +553,12 @@ def __init__( def to_conda_yaml( self, - extras: Tstr_seq_opt = None, - name: Tstr_opt = None, - channels: Tstr_seq_opt = None, - python_include: Tstr_opt = None, + extras: OptStrSeq = None, + name: OptStr = None, + channels: OptStrSeq = None, + python_include: OptStr = None, stream: str | Path | TextIO | None = None, - python_version: Tstr_opt = None, + python_version: OptStr = None, include_base_dependencies: bool = True, header_cmd: str | None = None, sort: bool = True, @@ -591,10 +588,10 @@ def to_conda_yaml( def to_conda_lists( self, - extras: Tstr_seq_opt = None, - channels: Tstr_seq_opt = None, - python_include: Tstr_opt = None, - python_version: Tstr_opt = None, + extras: OptStrSeq = None, + channels: OptStrSeq = None, + python_include: OptStr = None, + python_version: OptStr = None, include_base_dependencies: bool = True, sort: bool = True, deps: Sequence[str] | None = None, @@ -618,7 +615,7 @@ def to_conda_lists( def to_requirement_list( self, - extras: Tstr_seq_opt = None, + extras: OptStrSeq = None, include_base_dependencies: bool = True, sort: bool = True, reqs: Sequence[str] | None = None, @@ -646,7 +643,7 @@ def to_requirement_list( def to_requirements( self, - extras: Tstr_seq_opt = None, + extras: OptStrSeq = None, include_base_dependencies: bool = True, header_cmd: str | None = None, stream: str | Path | TextIO | None = None, @@ -676,15 +673,15 @@ def to_requirements( def to_conda_requirements( self, - extras: Tstr_seq_opt = None, - channels: Tstr_seq_opt = None, - python_include: Tstr_opt = None, - python_version: Tstr_opt = None, + extras: OptStrSeq = None, + channels: OptStrSeq = None, + python_include: OptStr = None, + python_version: OptStr = None, prepend_channel: bool = False, stream_conda: str | Path | TextIO | None = None, stream_pip: str | Path | TextIO | None = None, include_base_dependencies: bool = True, - header_cmd: Tstr_opt = None, + header_cmd: OptStr = None, sort: bool = True, deps: Sequence[str] | None = None, reqs: Sequence[str] | None = None, @@ -724,7 +721,7 @@ def to_conda_requirements( return deps_str, reqs_str - def _check_extras(self, extras: Tstr_seq_opt) -> None: + def _check_extras(self, extras: OptStrSeq) -> None: if extras is None: return elif isinstance(extras, str): @@ -752,9 +749,9 @@ def list_extras(self) -> list[str]: def from_string( cls, toml_string: str, - name: Tstr_opt = None, - channels: Tstr_seq_opt = None, - python_include: Tstr_opt = None, + name: OptStr = None, + channels: OptStrSeq = None, + python_include: OptStr = None, ) -> Self: data = tomlkit.parse(toml_string) return cls( @@ -765,9 +762,9 @@ def from_string( def from_path( cls, path: str | Path, - name: Tstr_opt = None, - channels: Tstr_seq_opt = None, - python_include: Tstr_opt = None, + name: OptStr = None, + channels: OptStrSeq = None, + python_include: OptStr = None, ) -> Self: path = Path(path) @@ -779,16 +776,3 @@ def from_path( return cls( data=data, name=name, channels=channels, python_include=python_include ) - - -# def _list_to_stream(values, stream=None): -# value = "\n".join(values) -# if isinstance(stream, (str, Path)): -# with open(stream, "w") as f: -# f.write(value) - -# elif stream is None: -# return value - -# else: -# stream.write(value) diff --git a/tests/test_parser.py b/tests/test_parser.py index 399c549..1c43f38 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -638,7 +638,7 @@ def test_spaces(): - conda-forge dependencies: - python>=3.8, <3.11 - - bthing-conda > 2.0 + - bthing-conda>2.0 - conda-forge::cthing - pip - pip: From 7083e7372fcb8ce85c80b0ec91a9529988574879 Mon Sep 17 00:00:00 2001 From: wpk Date: Wed, 15 Nov 2023 12:35:31 -0500 Subject: [PATCH 22/36] chore: further cleanup parsing --- src/pyproject2conda/parser.py | 102 ++++++++++++++++++++++------------ tests/test_parser.py | 4 +- 2 files changed, 68 insertions(+), 38 deletions(-) diff --git a/src/pyproject2conda/parser.py b/src/pyproject2conda/parser.py index 1cee115..509df98 100644 --- a/src/pyproject2conda/parser.py +++ b/src/pyproject2conda/parser.py @@ -256,42 +256,70 @@ def parse_p2c_comment(comment: OptStr) -> dict[str, Any] | None: return None -def _limit_dep_by_python_version( - dep: str, - python_version: str | None = None, -) -> str | None: - """Limit by python version""" - r = Requirement(dep) - - if ( - r.marker - and python_version - and (not r.marker.evaluate({"python_version": python_version})) - ): - return None - else: - r.marker = None - r.extras = None - return str(r) +def _clean_pip_reqs(reqs: list[str]) -> list[str]: + return [str(Requirement(r)) for r in reqs] -def _limit_deps_by_python_version( - deps: Iterable[str], - python_version: str | None = None, -) -> list[str]: +def _clean_conda_deps(deps: list[str], python_version: str | None = None) -> list[str]: out: list[str] = [] for dep in deps: - if (v := _limit_dep_by_python_version(dep, python_version)) is not None: - out.append(v) + # if have a channel, take it out + if "::" in dep: + channel, d = dep.split("::") + else: + channel, d = None, dep + r = Requirement(d) + + if ( + r.marker + and python_version + and (not r.marker.evaluate({"python_version": python_version})) + ): + continue + else: + r.marker = None + r.extras = set() + if channel: + r.name = f"{channel}::{r.name}" + out.append(str(r)) return out +# def _limit_dep_by_python_version( +# dep: str, +# python_version: str | None = None, +# ) -> str | None: +# """Limit by python version""" +# r = Requirement(dep) + +# if ( +# r.marker +# and python_version +# and (not r.marker.evaluate({"python_version": python_version})) +# ): +# return None +# else: +# r.marker = None +# r.extras = None +# return str(r) + + +# def _limit_deps_by_python_version( +# deps: Iterable[str], +# python_version: str | None = None, +# ) -> list[str]: +# out: list[str] = [] +# for dep in deps: +# if (v := _limit_dep_by_python_version(dep, python_version)) is not None: +# out.append(v) +# return out + + def value_comment_pairs_to_conda( value_comment_list: list[tuple[OptStr, OptStr]], sort: bool = True, deps: Sequence[str] | None = None, reqs: Sequence[str] | None = None, - python_version: str | None = None, ) -> dict[str, Any]: """Convert raw value/comment pairs to install lines""" @@ -309,20 +337,17 @@ def _check_value(value: Any) -> None: pip_deps.append(value) # type: ignore elif not parsed["skip"]: _check_value(value) - if v := _limit_dep_by_python_version(value, python_version): - if parsed["channel"]: - v = "{}::{}".format(parsed["channel"], v) - conda_deps.append(v) - - conda_deps.extend( - _limit_deps_by_python_version(parsed["package"], python_version) - ) + if parsed["channel"]: + conda_deps.append("{}::{}".format(parsed["channel"], value)) + else: + conda_deps.append(value) # type: ignore + + conda_deps.extend(parsed["package"]) elif value: - if v := _limit_dep_by_python_version(value, python_version): - conda_deps.append(v) + conda_deps.append(value) if deps: - conda_deps.extend(_limit_deps_by_python_version(deps, python_version)) + conda_deps.extend(list(deps)) if reqs: pip_deps.extend(list(reqs)) @@ -371,9 +396,14 @@ def pyproject_to_conda_lists( sort=sort, deps=deps, reqs=reqs, - python_version=python_version, ) + # clean up + output["dependencies"] = _clean_conda_deps( + output["dependencies"], python_version=python_version + ) + output["pip"] = _clean_pip_reqs(output["pip"]) + if python_include: output["dependencies"].insert(0, python_include) if channels: diff --git a/tests/test_parser.py b/tests/test_parser.py index 1c43f38..cf37798 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -510,7 +510,7 @@ def test_complete(): - pip - pip: - athing - - req;python_version<'3.10' + - req;python_version<"3.10" """ assert dedent(expected) == d.to_conda_yaml( deps=["dep;python_version<'3.10'"], @@ -642,7 +642,7 @@ def test_spaces(): - conda-forge::cthing - pip - pip: - - athing >0.5 + - athing>0.5 """ assert dedent(expected) == d.to_conda_yaml( python_include="infer", remove_whitespace=False From 97156fd10bfc1e837b936916ceea68bc30705804 Mon Sep 17 00:00:00 2001 From: wpk Date: Wed, 15 Nov 2023 12:50:48 -0500 Subject: [PATCH 23/36] chore: now run to_requirements through Requirements --- src/pyproject2conda/_typing.py | 10 +++------- src/pyproject2conda/parser.py | 3 +++ tests/test_cli.py | 18 +++++++++--------- tests/test_parser.py | 8 ++++---- 4 files changed, 19 insertions(+), 20 deletions(-) diff --git a/src/pyproject2conda/_typing.py b/src/pyproject2conda/_typing.py index 3220b98..f7d2aaa 100644 --- a/src/pyproject2conda/_typing.py +++ b/src/pyproject2conda/_typing.py @@ -17,14 +17,10 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Sequence, TypeVar - -if TYPE_CHECKING: - from ._typing_compat import TypeAlias - +from typing import Optional, Sequence, TypeVar, Union R = TypeVar("R") T = TypeVar("T") -OptStr: TypeAlias = str | None -OptStrSeq: TypeAlias = str | Sequence[str] | None +OptStr = Optional[str] +OptStrSeq = Union[str, Sequence[str], None] diff --git a/src/pyproject2conda/parser.py b/src/pyproject2conda/parser.py index 509df98..e264ffd 100644 --- a/src/pyproject2conda/parser.py +++ b/src/pyproject2conda/parser.py @@ -663,6 +663,9 @@ def to_requirement_list( if reqs: out.extend(list(reqs)) + # cleanup reqs: + out = _clean_pip_reqs(out) + if remove_whitespace: out = [_remove_whitespace(x) for x in out] diff --git a/tests/test_cli.py b/tests/test_cli.py index c92166e..abafd02 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -353,7 +353,7 @@ def test_requirements(): expected = """\ athing bthing -cthing;python_version<'3.10' +cthing;python_version<"3.10" """ for cmd in ["r", "requirements"]: @@ -363,7 +363,7 @@ def test_requirements(): expected = """\ athing bthing -cthing;python_version<'3.10' +cthing;python_version<"3.10" pandas pytest matplotlib @@ -378,11 +378,11 @@ def test_requirements(): expected = """\ athing bthing -cthing;python_version<'3.10' +cthing;python_version<"3.10" pandas pytest matplotlib -thing;python_version<'3.10' +thing;python_version<"3.10" other """ @@ -403,7 +403,7 @@ def test_requirements(): expected = """\ athing bthing -cthing;python_version<'3.10' +cthing;python_version<"3.10" matplotlib pandas pytest @@ -418,12 +418,12 @@ def test_requirements(): expected = """\ athing bthing -cthing;python_version<'3.10' +cthing;python_version<"3.10" matplotlib other pandas pytest -thing;python_version<'3.10' +thing;python_version<"3.10" """ result = do_run( @@ -443,12 +443,12 @@ def test_requirements(): expected = """\ athing bthing -cthing; python_version < '3.10' +cthing; python_version < "3.10" matplotlib other pandas pytest -thing; python_version < '3.10' +thing; python_version < "3.10" """ result = do_run( diff --git a/tests/test_parser.py b/tests/test_parser.py index cf37798..273cd1b 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -664,9 +664,9 @@ def test_spaces(): ) expected = """\ - athing >0.5 - bthing > 1.0 - cthing; python_version < '3.10' + athing>0.5 + bthing>1.0 + cthing; python_version < "3.10" """ assert dedent(expected) == d.to_requirements(remove_whitespace=False) @@ -674,7 +674,7 @@ def test_spaces(): expected = """\ athing>0.5 bthing>1.0 - cthing;python_version<'3.10' + cthing;python_version<"3.10" """ assert dedent(expected) == d.to_requirements(remove_whitespace=True) From 2a46b240973703b3892ae10be133e32430412a8e Mon Sep 17 00:00:00 2001 From: wpk Date: Wed, 15 Nov 2023 13:30:43 -0500 Subject: [PATCH 24/36] chore: separate out logic for parsing deps --- src/pyproject2conda/parser.py | 100 ++++++++++++++-------------------- 1 file changed, 42 insertions(+), 58 deletions(-) diff --git a/src/pyproject2conda/parser.py b/src/pyproject2conda/parser.py index e264ffd..db12959 100644 --- a/src/pyproject2conda/parser.py +++ b/src/pyproject2conda/parser.py @@ -76,6 +76,35 @@ def _list_to_str(values: Iterable[str] | None, eol: bool = True) -> str: return output +def _clean_pip_reqs(reqs: list[str]) -> list[str]: + return [str(Requirement(r)) for r in reqs] + + +def _clean_conda_deps(deps: list[str], python_version: str | None = None) -> list[str]: + out: list[str] = [] + for dep in deps: + # if have a channel, take it out + if "::" in dep: + channel, d = dep.split("::") + else: + channel, d = None, dep + r = Requirement(d) + + if ( + r.marker + and python_version + and (not r.marker.evaluate({"python_version": python_version})) + ): + continue + else: + r.marker = None + r.extras = set() + if channel: + r.name = f"{channel}::{r.name}" + out.append(str(r)) + return out + + # * Default parser --------------------------------------------------------------------- @@ -230,6 +259,7 @@ def _pyproject_to_value_comment_pairs( return value_comment_list +# * Parsing p2c comments def _match_p2c_comment(comment: OptStr) -> OptStr: if not comment or not (match := re.match(r".*?#\s*p2c:\s*([^\#]*)", comment)): return None @@ -256,65 +286,19 @@ def parse_p2c_comment(comment: OptStr) -> dict[str, Any] | None: return None -def _clean_pip_reqs(reqs: list[str]) -> list[str]: - return [str(Requirement(r)) for r in reqs] - - -def _clean_conda_deps(deps: list[str], python_version: str | None = None) -> list[str]: - out: list[str] = [] - for dep in deps: - # if have a channel, take it out - if "::" in dep: - channel, d = dep.split("::") - else: - channel, d = None, dep - r = Requirement(d) - - if ( - r.marker - and python_version - and (not r.marker.evaluate({"python_version": python_version})) - ): - continue - else: - r.marker = None - r.extras = set() - if channel: - r.name = f"{channel}::{r.name}" - out.append(str(r)) +def _pyproject_to_value_parsed_pairs( + value_comment_list: list[tuple[OptStr, OptStr]], +) -> list[tuple[str | None, dict[str, Any]]]: + out = [] + for value, comment in value_comment_list: + if comment and (parsed := parse_p2c_comment(comment)): + out.append((value, parsed)) + elif value: + out.append((value, {})) return out -# def _limit_dep_by_python_version( -# dep: str, -# python_version: str | None = None, -# ) -> str | None: -# """Limit by python version""" -# r = Requirement(dep) - -# if ( -# r.marker -# and python_version -# and (not r.marker.evaluate({"python_version": python_version})) -# ): -# return None -# else: -# r.marker = None -# r.extras = None -# return str(r) - - -# def _limit_deps_by_python_version( -# deps: Iterable[str], -# python_version: str | None = None, -# ) -> list[str]: -# out: list[str] = [] -# for dep in deps: -# if (v := _limit_dep_by_python_version(dep, python_version)) is not None: -# out.append(v) -# return out - - +# * To dependency list def value_comment_pairs_to_conda( value_comment_list: list[tuple[OptStr, OptStr]], sort: bool = True, @@ -330,8 +314,8 @@ def _check_value(value: Any) -> None: if not value: raise ValueError("trying to add value that does not exist") - for value, comment in value_comment_list: - if comment and (parsed := parse_p2c_comment(comment)): + for value, parsed in _pyproject_to_value_parsed_pairs(value_comment_list): + if parsed: if parsed["pip"]: _check_value(value) pip_deps.append(value) # type: ignore From 43b2db24432e5f9d51c504d2df25f7346ba5c3cb Mon Sep 17 00:00:00 2001 From: wpk Date: Wed, 15 Nov 2023 15:39:39 -0500 Subject: [PATCH 25/36] chore: separate out logic for parsing deps --- README.md | 40 +++++- src/pyproject2conda/parser.py | 63 +++++++++- tests/data/test-pyproject-alt.toml | 69 ++++++++++ tests/test_cli.py | 158 +++++++++++++++++------ tests/test_parser.py | 194 ++++++++++++++++++++--------- 5 files changed, 416 insertions(+), 108 deletions(-) create mode 100644 tests/data/test-pyproject-alt.toml diff --git a/README.md b/README.md index d512ecb..7ebd233 100644 --- a/README.md +++ b/README.md @@ -134,12 +134,12 @@ Note the comment lines `# p2c:...`. These are special tokens that ```bash -usage: -c [-h] [-c CHANNEL] [-p] [-s] [package ...] +usage: -c [-h] [-c CHANNEL] [-p] [-s] [packages ...] Parser searches for comments '# p2c: [OPTIONS] CONDA-PACKAGES positional arguments: - package + packages options: -h, --help show this help message and exit @@ -195,6 +195,40 @@ dependencies: +### Alternate syntax: using table instead of comments + +While using comments to mark options has the convenience of placing the changes +right next to the dependency, it can becore a bit cumbersome. If you feel this +way, then you can use an alternative method to map `pip` dependencies to `conda` +dependencies. For this, use the `tool.pyproject2.conda.dependencies` table. For +example, we can do the same thing as above with: + + + + +```toml +# ... +[tool.pyproject2conda.dependencies] +athing = { pip = true } +bthing = { skip = true, packages = "bthing-conda" } +cthing = { channel = "conda-forge" } +pytest = { channel = "conda-forge" } +matplotlib = { skip = true, packages = [ + "additional-thing; python_version < '3.9'", + "conda-matplotlib" +] } +build = { channel = "pip" } +# ... + +``` + +Note that the table keys are the package name to be adjusted (no version +modifiers in the name), and the following dictionary is similar to the options +above. Also, you can specify `channel = "pip"` to force the package to be +installed with pip (same as setting `pip = true`). + + + ### Specify python version To specify a specific value of python in the output, pass a value with: @@ -661,7 +695,7 @@ $ p2c project -f tests/data/test-pyproject.toml --dry # Creating requirements base.txt athing bthing -cthing;python_version<'3.10' +cthing;python_version<"3.10" # -------------------- # Creating yaml py310-test-extras.yaml channels: diff --git a/src/pyproject2conda/parser.py b/src/pyproject2conda/parser.py index db12959..0d0e24f 100644 --- a/src/pyproject2conda/parser.py +++ b/src/pyproject2conda/parser.py @@ -133,7 +133,7 @@ def _default_parser() -> argparse.ArgumentParser: help="If specified skip pyproject dependency on this line", ) - parser.add_argument("package", nargs="*") + parser.add_argument("packages", nargs="*") return parser @@ -295,6 +295,47 @@ def _pyproject_to_value_parsed_pairs( out.append((value, parsed)) elif value: out.append((value, {})) + return out # pyright: ignore + + +def _format_override_table(override_table: dict[str, Any]) -> dict[str, Any]: + out: dict[str, Any] = {} + for name, v in override_table.items(): + new = { + "pip": v.get("pip", False), + "skip": v.get("skip", False), + "channel": v.get("channel", None), + "packages": v.get("packages", []), + } + + if new["channel"] and new["channel"].strip() == "pip": + new["pip"] = True + new["channel"] = None + + if isinstance(new["packages"], str): + new["packages"] = [new["packages"]] + + out[name] = new + + return out + + +def _apply_override_table( + value_parsed_list: list[tuple[str | None, dict[str, Any]]], + override_table: dict[str, Any], +) -> list[tuple[str | None, dict[str, Any]]]: + out: list[tuple[str | None, dict[str, Any]]] = [] + + override_table = _format_override_table(override_table) + + for value, parsed in value_parsed_list: + if value and (name := Requirement(value).name) in override_table: + new_parsed = dict(override_table[name], **parsed) + else: + new_parsed = parsed + + out.append((value, new_parsed)) + return out @@ -304,6 +345,7 @@ def value_comment_pairs_to_conda( sort: bool = True, deps: Sequence[str] | None = None, reqs: Sequence[str] | None = None, + override_table: dict[str, Any] | None = None, ) -> dict[str, Any]: """Convert raw value/comment pairs to install lines""" @@ -314,7 +356,12 @@ def _check_value(value: Any) -> None: if not value: raise ValueError("trying to add value that does not exist") - for value, parsed in _pyproject_to_value_parsed_pairs(value_comment_list): + value_parsed_list = _pyproject_to_value_parsed_pairs(value_comment_list) + + if override_table: + value_parsed_list = _apply_override_table(value_parsed_list, override_table) + + for value, parsed in value_parsed_list: if parsed: if parsed["pip"]: _check_value(value) @@ -326,7 +373,7 @@ def _check_value(value: Any) -> None: else: conda_deps.append(value) # type: ignore - conda_deps.extend(parsed["package"]) + conda_deps.extend(parsed["packages"]) elif value: conda_deps.append(value) @@ -354,6 +401,7 @@ def pyproject_to_conda_lists( deps: Sequence[str] | None = None, reqs: Sequence[str] | None = None, remove_whitespace: bool = True, + override_table: dict[str, Any] | None = None, ) -> dict[str, Any]: if python_include == "infer": # safer get @@ -366,6 +414,14 @@ def pyproject_to_conda_lists( channels_doc = get_in(["tool", "pyproject2conda", "channels"], data, None) if channels_doc: channels = channels_doc.unwrap() + + if override_table is None: + override_table_ = get_in( + ["tool", "pyproject2conda", "dependencies"], data, None + ) + if override_table_: + override_table = override_table_.unwrap() + if isinstance(channels, str): channels = [channels] @@ -380,6 +436,7 @@ def pyproject_to_conda_lists( sort=sort, deps=deps, reqs=reqs, + override_table=override_table, ) # clean up diff --git a/tests/data/test-pyproject-alt.toml b/tests/data/test-pyproject-alt.toml new file mode 100644 index 0000000..f19837e --- /dev/null +++ b/tests/data/test-pyproject-alt.toml @@ -0,0 +1,69 @@ +[project] +name = "hello" +requires-python = ">=3.8,<3.11" +dependencies = [ + "athing", # + "bthing", + "cthing; python_version < '3.10'", +] + +[project.optional-dependencies] +test = [ + "pandas", # + "pytest", +] +dev-extras = ["matplotlib"] +dev = ["hello[test]", "hello[dev-extras]"] +dist-pypi = [ + # this is intended to be parsed with --no-base option + "setuptools", + "build", +] + +[tool.pyproject2conda] +channels = ['conda-forge'] +# these are the same as the default values of `p2c project` +template_python = "py{py}-{env}" +template = "{env}" +style = "yaml" +# options +python = ["3.10"] +# Note that this is relative to the location of pyproject.toml +user_config = "config/userconfig.toml" +default_envs = ["test", "dev", "dist-pypi"] + +# overrides of dependencies +[tool.pyproject2conda.dependencies] +athing = { pip = true } +bthing = { skip = true, packages = "bthing-conda" } +cthing = { channel = "conda-forge" } +pytest = { channel = "conda-forge" } +matplotlib = { skip = true, packages = [ + "additional-thing; python_version < '3.9'", + "conda-matplotlib" +] } +build = { channel = "pip" } + +[tool.pyproject2conda.envs.base] +style = ["requirements"] +# Note that the default value for `extras` is the name of the environment. +# To have no extras, either pass +# extras = [] +# or +# +extras = false + +# +# A value of `extras = true` also implies using the environment name +# as the extras. +[tool.pyproject2conda.envs."test-extras"] +extras = ["test"] +style = ["yaml", "requirements"] + +[[tool.pyproject2conda.overrides]] +envs = ['test-extras', "dist-pypi"] +base = false + +[[tool.pyproject2conda.overrides]] +envs = ["test", "test-extras"] +python = ["3.10", "3.11"] diff --git a/tests/test_cli.py b/tests/test_cli.py index abafd02..2bb9dde 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -13,12 +13,17 @@ import sys +import pytest + + ROOT = Path(__file__).resolve().parent / "data" def do_run(runner, command, *opts, filename=None, must_exist=False, **kwargs): if filename is None: - filename = str(ROOT / "test-pyproject.toml") + raise ValueError + # if filename is None: + # filename = str(ROOT / "test-pyproject.toml") filename = Path(filename) if must_exist and not filename.exists(): raise ValueError(f"filename {filename} does not exist") @@ -32,11 +37,16 @@ def check_result(result, expected): assert result.output == dedent(expected) -def test_list(): +@pytest.fixture(params=["test-pyproject.toml", "test-pyproject-alt.toml"]) +def filename(request): + return ROOT / request.param + + +def test_list(filename): runner = CliRunner() for cmd in ["l", "list"]: - result = do_run(runner, cmd) + result = do_run(runner, cmd, filename=filename) expected = """\ Extras: ======= @@ -47,13 +57,13 @@ def test_list(): """ check_result(result, expected) - result = do_run(runner, "list", "-v") + result = do_run(runner, "list", "-v", filename=filename) # TODO: the logger writes to output # assert result.output == f"filename: {ROOT / 'test-pyproject.toml'} [pyproject2conda - INFO]" check_result(result, expected) -def test_create(): +def test_create(filename): runner = CliRunner() # test unknown file @@ -73,7 +83,7 @@ def test_create(): - athing """ - result = do_run(runner, "yaml") + result = do_run(runner, "yaml", filename=filename) check_result(result, expected) @@ -99,7 +109,7 @@ def test_create(): - athing """ - result = do_run(runner, "yaml", "--header") + result = do_run(runner, "yaml", "--header", filename=filename) check_result(result, expected) @@ -118,7 +128,7 @@ def test_create(): for opt in [("--python-include", "infer")]: for cmd in ["y", "yaml"]: - result = do_run(runner, cmd, *opt) + result = do_run(runner, cmd, *opt, filename=filename) check_result(result, expected) # -p python=3.8 @@ -138,7 +148,7 @@ def test_create(): ("--python", "3.8"), ("-p", "3.8"), ]: - result = do_run(runner, "yaml", *opts) + result = do_run(runner, "yaml", *opts, filename=filename) check_result(result, expected) # dev @@ -158,11 +168,13 @@ def test_create(): """ for opt in ["-e", "--extra"]: - result = do_run(runner, "yaml", opt, "dev", "--no-sort") + result = do_run(runner, "yaml", opt, "dev", "--no-sort", filename=filename) check_result(result, expected) # test if add in "test" gives same answer - result = do_run(runner, "yaml", "-e", "dev", "-e", "test", "--no-sort") + result = do_run( + runner, "yaml", "-e", "dev", "-e", "test", "--no-sort", filename=filename + ) check_result(result, expected) expected = """\ @@ -181,11 +193,11 @@ def test_create(): """ for opt in ["-e", "--extra"]: - result = do_run(runner, "yaml", opt, "dev") + result = do_run(runner, "yaml", opt, "dev", filename=filename) check_result(result, expected) # test if add in "test" gives same answer - result = do_run(runner, "yaml", "-e", "dev", "-e", "test") + result = do_run(runner, "yaml", "-e", "dev", "-e", "test", filename=filename) check_result(result, expected) # different ordering @@ -282,7 +294,7 @@ def test_create(): """ for opt in ["-c", "--channel"]: - result = do_run(runner, "yaml", opt, "hello") + result = do_run(runner, "yaml", opt, "hello", filename=filename) check_result(result, expected) expected = """\ @@ -298,7 +310,7 @@ def test_create(): """ for opt in ["-c", "--channel"]: - result = do_run(runner, "yaml", opt, "hello", opt, "there") + result = do_run(runner, "yaml", opt, "hello", opt, "there", filename=filename) check_result(result, expected) # test @@ -314,7 +326,7 @@ def test_create(): - pip: - athing """ - result = do_run(runner, "yaml", "-e", "test", "--no-sort") + result = do_run(runner, "yaml", "-e", "test", "--no-sort", filename=filename) check_result(result, expected) expected = """\ @@ -329,7 +341,7 @@ def test_create(): - pip: - athing """ - result = do_run(runner, "yaml", "-e", "test") + result = do_run(runner, "yaml", "-e", "test", filename=filename) check_result(result, expected) # isolated @@ -343,11 +355,13 @@ def test_create(): - build """ for opt in ["-e", "--extra"]: - result = do_run(runner, "yaml", opt, "dist-pypi", "--no-base") + result = do_run( + runner, "yaml", opt, "dist-pypi", "--no-base", filename=filename + ) check_result(result, expected) -def test_requirements(): +def test_requirements(filename): runner = CliRunner() expected = """\ @@ -357,7 +371,7 @@ def test_requirements(): """ for cmd in ["r", "requirements"]: - result = do_run(runner, cmd) + result = do_run(runner, cmd, filename=filename) check_result(result, expected) expected = """\ @@ -369,10 +383,19 @@ def test_requirements(): matplotlib """ - result = do_run(runner, "requirements", "-e", "dev", "--no-sort") + result = do_run(runner, "requirements", "-e", "dev", "--no-sort", filename=filename) check_result(result, expected) - result = do_run(runner, "requirements", "-e", "dev", "-e", "test", "--no-sort") + result = do_run( + runner, + "requirements", + "-e", + "dev", + "-e", + "test", + "--no-sort", + filename=filename, + ) check_result(result, expected) expected = """\ @@ -396,6 +419,7 @@ def test_requirements(): "thing;python_version<'3.10'", "-r", "other", + filename=filename, ) check_result(result, expected) @@ -409,10 +433,12 @@ def test_requirements(): pytest """ - result = do_run(runner, "requirements", "-e", "dev") + result = do_run(runner, "requirements", "-e", "dev", filename=filename) check_result(result, expected) - result = do_run(runner, "requirements", "-e", "dev", "-e", "test") + result = do_run( + runner, "requirements", "-e", "dev", "-e", "test", filename=filename + ) check_result(result, expected) expected = """\ @@ -435,6 +461,7 @@ def test_requirements(): "thing;python_version<'3.10'", "-r", "other", + filename=filename, ) check_result(result, expected) @@ -461,6 +488,7 @@ def test_requirements(): "-r", "other", "--no-remove-whitespace", + filename=filename, ) check_result(result, expected) @@ -470,7 +498,15 @@ def test_requirements(): build """ - result = do_run(runner, "requirements", "-e", "dist-pypi", "--no-base", "--no-sort") + result = do_run( + runner, + "requirements", + "-e", + "dist-pypi", + "--no-base", + "--no-sort", + filename=filename, + ) check_result(result, expected) expected = """\ @@ -478,7 +514,9 @@ def test_requirements(): setuptools """ - result = do_run(runner, "requirements", "-e", "dist-pypi", "--no-base") + result = do_run( + runner, "requirements", "-e", "dist-pypi", "--no-base", filename=filename + ) check_result(result, expected) @@ -489,19 +527,19 @@ def check_results_conda_req(path, expected): assert result == dedent(expected) -def test_conda_requirements(): +def test_conda_requirements(filename): runner = CliRunner() - result = do_run(runner, "c", "hello.txt") + result = do_run(runner, "c", "hello.txt", filename=filename) assert isinstance(result.exception, ValueError) - result = do_run(runner, "c", "--prefix", "hello", "a", "b") + result = do_run(runner, "c", "--prefix", "hello", "a", "b", filename=filename) assert isinstance(result.exception, ValueError) # stdout - result = do_run(runner, "c") + result = do_run(runner, "c", filename=filename) expected = """\ #conda requirements @@ -526,7 +564,14 @@ def test_conda_requirements(): """ for cmd in ["c", "conda-requirements"]: - do_run(runner, cmd, "--prefix", str(d / "hello-"), "--no-header") + do_run( + runner, + cmd, + "--prefix", + str(d / "hello-"), + "--no-header", + filename=filename, + ) check_results_conda_req(d / "hello-conda.txt", expected_conda) check_results_conda_req(d / "hello-pip.txt", expected_pip) @@ -537,6 +582,7 @@ def test_conda_requirements(): str(d / "conda-output.txt"), str(d / "pip-output.txt"), "--no-header", + filename=filename, ) check_results_conda_req(d / "conda-output.txt", expected_conda) @@ -551,6 +597,7 @@ def test_conda_requirements(): "-c", "achannel", "--no-header", + filename=filename, ) expected_conda = """\ @@ -562,11 +609,11 @@ def test_conda_requirements(): check_results_conda_req(d / "hello-pip.txt", expected_pip) -def test_json(): +def test_json(filename): runner = CliRunner() # stdout - result = do_run(runner, "j") + result = do_run(runner, "j", filename=filename) expected = """\ {"dependencies": ["bthing-conda", "conda-forge::cthing"], "pip": ["athing"], "channels": ["conda-forge"]} @@ -589,7 +636,7 @@ def check_results(path, expected): "channels": ["conda-forge"], } - do_run(runner, "json", "-o", str(d / "hello.json")) + do_run(runner, "json", "-o", str(d / "hello.json"), filename=filename) check_results(d / "hello.json", expected) @@ -606,7 +653,16 @@ def check_results(path, expected): "channels": ["conda-forge"], } - do_run(runner, "json", "-o", str(d / "there.json"), "-e", "dev", "--no-sort") + do_run( + runner, + "json", + "-o", + str(d / "there.json"), + "-e", + "dev", + "--no-sort", + filename=filename, + ) check_results(d / "there.json", expected) @@ -623,15 +679,17 @@ def check_results(path, expected): "channels": ["conda-forge"], } - do_run(runner, "json", "-o", str(d / "there.json"), "-e", "dev") + do_run( + runner, "json", "-o", str(d / "there.json"), "-e", "dev", filename=filename + ) check_results(d / "there.json", expected) -def test_alias(): +def test_alias(filename): runner = CliRunner() - result = do_run(runner, "q") + result = do_run(runner, "q", filename=filename) assert isinstance(result.exception, BaseException) @@ -646,7 +704,7 @@ def test_alias(): # check_result(result, expected) -def test_overwrite(): +def test_overwrite(filename): runner = CliRunner(mix_stderr=True) with tempfile.TemporaryDirectory() as d_tmp: @@ -657,7 +715,15 @@ def test_overwrite(): assert not path.exists() result = do_run( - runner, "yaml", "-o", str(path), "-v", "-w", "force", catch_exceptions=False + runner, + "yaml", + "-o", + str(path), + "-v", + "-w", + "force", + catch_exceptions=False, + filename=filename, ) # assert result.output.strip() == f"# Creating yaml {d_tmp}/out.yaml" @@ -673,6 +739,7 @@ def test_overwrite(): "-w", cmd, catch_exceptions=False, + filename=filename, ) if cmd == "force": @@ -697,6 +764,7 @@ def test_overwrite(): "-w", "force", catch_exceptions=False, + filename=filename, ) orig_time = path.stat().st_mtime @@ -704,7 +772,15 @@ def test_overwrite(): for cmd in ["check", "skip", "force"]: result = do_run( - runner, "r", "-o", str(path), "-v", "-w", cmd, catch_exceptions=False + runner, + "r", + "-o", + str(path), + "-v", + "-w", + cmd, + catch_exceptions=False, + filename=filename, ) if cmd == "force": diff --git a/tests/test_parser.py b/tests/test_parser.py index 273cd1b..6146e24 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -49,14 +49,14 @@ def test_match_p2c_comment(): def test_parse_p2c(): - def get_expected(pip=False, skip=False, channel=None, package=None): - if package is None: - package = [] + def get_expected(pip=False, skip=False, channel=None, packages=None): + if packages is None: + packages = [] return { "pip": pip, "skip": skip, "channel": channel, - "package": package, + "packages": packages, } assert parser._parse_p2c(None) is None @@ -72,15 +72,15 @@ def get_expected(pip=False, skip=False, channel=None, package=None): ) assert parser._parse_p2c("athing>=0.3,<0.2 ") == get_expected( - package=["athing>=0.3,<0.2"] + packages=["athing>=0.3,<0.2"] ) assert parser._parse_p2c("athing>=0.3,<0.2 bthing ") == get_expected( - package=["athing>=0.3,<0.2", "bthing"] + packages=["athing>=0.3,<0.2", "bthing"] ) assert parser._parse_p2c("'athing >= 0.3, <0.2' bthing ") == get_expected( - package=["athing >= 0.3, <0.2", "bthing"] + packages=["athing >= 0.3, <0.2", "bthing"] ) @@ -268,42 +268,89 @@ def test_package_name(): d.to_conda_lists() -def test_complete(): - toml = dedent( - """\ - [project] - name = "hello" - requires-python = ">=3.8, <3.11" - dependencies = [ - "athing", # p2c: -p # a comment - "bthing", # p2c: -s bthing-conda - "cthing; python_version<'3.10'", # p2c: -c conda-forge - ] - - [project.optional-dependencies] - test = [ - "pandas", - "pytest", # p2c: -c conda-forge - ] - dev-extras = [ - # p2c: -s additional-thing # this is an additional conda package - "matplotlib", # p2c: -s conda-matplotlib - ] - dev = [ - "hello[test]", - "hello[dev-extras]", - ] - dist-pypi = [ - "setuptools", - "build", # p2c: -p - ] - - - [tool.pyproject2conda] - channels = ['conda-forge'] - """ - ) - +@pytest.mark.parametrize( + "toml", + [ + # comment syntax + dedent( + """\ + [project] + name = "hello" + requires-python = ">=3.8, <3.11" + dependencies = [ + "athing", # p2c: -p # a comment + "bthing", # p2c: -s bthing-conda + "cthing; python_version<'3.10'", # p2c: -c conda-forge + ] + + [project.optional-dependencies] + test = [ + "pandas", + "pytest", # p2c: -c conda-forge + ] + dev-extras = [ + # p2c: -s additional-thing # this is an additional conda package + "matplotlib", # p2c: -s conda-matplotlib + ] + dev = [ + "hello[test]", + "hello[dev-extras]", + ] + dist-pypi = [ + "setuptools", + "build", # p2c: -p + ] + + + [tool.pyproject2conda] + channels = ['conda-forge'] + """ + ), + # table syntax + dedent( + """\ + [project] + name = "hello" + requires-python = ">=3.8, <3.11" + dependencies = [ + "athing", + "bthing", + "cthing; python_version<'3.10'", + ] + + [project.optional-dependencies] + test = [ + "pandas", + "pytest", + ] + dev-extras = [ + "matplotlib", + ] + dev = [ + "hello[test]", + "hello[dev-extras]", + ] + dist-pypi = [ + "setuptools", + "build", + ] + + + [tool.pyproject2conda] + channels = ['conda-forge'] + + [tool.pyproject2conda.dependencies] + athing = {pip = true} + bthing = {skip = true, packages = "bthing-conda"} + cthing = {channel = "conda-forge"} + pytest = {channel = "conda-forge"} + matplotlib = {skip = true, packages = ["additional-thing", "conda-matplotlib"]} + build = { channel = "pip" } + """ + ), + ], +) +def test_complete(toml): d = parser.PyProject2Conda.from_string(toml) # test unknown extra @@ -613,24 +660,49 @@ def test_missing_dependencies(): assert f(allow_empty=True) == "No dependencies for this environment\n" -def test_spaces(): - toml = dedent( - """\ - [project] - name = "hello" - requires-python = ">=3.8, <3.11" - dependencies = [ - "athing >0.5", # p2c: -p # a comment - "bthing > 1.0", # p2c: -s 'bthing-conda > 2.0' - "cthing; python_version < '3.10'", # p2c: -c conda-forge - ] - - - [tool.pyproject2conda] - channels = ['conda-forge'] - """ - ) - +@pytest.mark.parametrize( + "toml", + [ + dedent( + """\ + [project] + name = "hello" + requires-python = ">=3.8, <3.11" + dependencies = [ + "athing >0.5", # p2c: -p # a comment + "bthing > 1.0", # p2c: -s 'bthing-conda > 2.0' + "cthing; python_version < '3.10'", # p2c: -c conda-forge + ] + + + [tool.pyproject2conda] + channels = ['conda-forge'] + """ + ), + dedent( + """\ + [project] + name = "hello" + requires-python = ">=3.8, <3.11" + dependencies = [ + "athing >0.5", + "bthing > 1.0", + "cthing; python_version < '3.10'", + ] + + + [tool.pyproject2conda] + channels = ['conda-forge'] + + [tool.pyproject2conda.dependencies] + athing = {channel = "pip"} + bthing = {skip = true, packages = "bthing-conda > 2.0"} + cthing = {channel = "conda-forge"} + """ + ), + ], +) +def test_spaces(toml): d = parser.PyProject2Conda.from_string(toml) expected = """\ From d6f659415dd74b03ce208ec6adfc534e05ab728d Mon Sep 17 00:00:00 2001 From: wpk Date: Wed, 15 Nov 2023 15:43:33 -0500 Subject: [PATCH 26/36] chore: add changelog snippet --- ...krekelberg_add_pyproject_table_override.md | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 changelog.d/20231115_154156_william.krekelberg_add_pyproject_table_override.md diff --git a/changelog.d/20231115_154156_william.krekelberg_add_pyproject_table_override.md b/changelog.d/20231115_154156_william.krekelberg_add_pyproject_table_override.md new file mode 100644 index 0000000..70d5232 --- /dev/null +++ b/changelog.d/20231115_154156_william.krekelberg_add_pyproject_table_override.md @@ -0,0 +1,43 @@ + + + + + +### Added + +- Can now specify conda changes using `tool.pyproject2conda.dependencies` table. + This is an alternative to using `# p2c:` comments. + + + + + From c97e18cb02125ce02fdae3ddbf2f7d6788c0c2b2 Mon Sep 17 00:00:00 2001 From: wpk Date: Thu, 16 Nov 2023 22:02:09 -0500 Subject: [PATCH 27/36] chore: add requirements.py. Work needed to integrate/test --- README.md | 2 +- src/pyproject2conda/_typing.py | 16 +- src/pyproject2conda/parser.py | 283 ++------- src/pyproject2conda/requirements.py | 879 ++++++++++++++++++++++++++++ src/pyproject2conda/utils.py | 85 ++- tests/test_parser.py | 44 +- 6 files changed, 1037 insertions(+), 272 deletions(-) create mode 100644 src/pyproject2conda/requirements.py diff --git a/README.md b/README.md index 7ebd233..64c3766 100644 --- a/README.md +++ b/README.md @@ -132,7 +132,7 @@ Note the comment lines `# p2c:...`. These are special tokens that - + ```bash usage: -c [-h] [-c CHANNEL] [-p] [-s] [packages ...] diff --git a/src/pyproject2conda/_typing.py b/src/pyproject2conda/_typing.py index f7d2aaa..3c11a25 100644 --- a/src/pyproject2conda/_typing.py +++ b/src/pyproject2conda/_typing.py @@ -17,10 +17,24 @@ from __future__ import annotations -from typing import Optional, Sequence, TypeVar, Union +from typing import TYPE_CHECKING, Optional, Sequence, TypeVar, Union R = TypeVar("R") T = TypeVar("T") + OptStr = Optional[str] OptStrSeq = Union[str, Sequence[str], None] + +if TYPE_CHECKING: + from typing import Literal, Tuple + + from packaging.requirements import Requirement + + from .requirements import OverrideDeps + from .utils import _Missing + + MISSING_TYPE = Literal[_Missing.MISSING] + + RequirementCommentPair = Tuple[Optional[Requirement], Optional[str]] + RequirementOverridePair = Tuple[Optional[Requirement], Optional[OverrideDeps]] diff --git a/src/pyproject2conda/parser.py b/src/pyproject2conda/parser.py index 0d0e24f..72192a6 100644 --- a/src/pyproject2conda/parser.py +++ b/src/pyproject2conda/parser.py @@ -7,159 +7,51 @@ """ from __future__ import annotations -import argparse -import re -import shlex -from functools import lru_cache from pathlib import Path from typing import ( TYPE_CHECKING, - Any, - Generator, - Iterable, - Sequence, - TextIO, cast, ) if TYPE_CHECKING: + from typing import ( + Any, + Sequence, + TextIO, + ) + + import tomlkit.container import tomlkit.items import tomlkit.toml_document - from ._typing import OptStr, OptStrSeq, T + from ._typing import OptStr, OptStrSeq from ._typing_compat import Self import tomlkit from packaging.requirements import Requirement from ruamel.yaml import YAML -from pyproject2conda.utils import get_in - -# * Utilities -------------------------------------------------------------------------- - - -def _check_allow_empty(allow_empty: bool) -> str: - msg = "No dependencies for this environment\n" - if allow_empty: - return msg - else: - raise ValueError(msg) - - -_WHITE_SPACE_REGEX = re.compile(r"\s+") - - -def _remove_whitespace(s: str) -> str: - return re.sub(_WHITE_SPACE_REGEX, "", s) - - -def _unique_list(values: Iterable[T]) -> list[T]: - """ - Return only unique values in list. - Unlike using set(values), this preserves order. - """ - output: list[T] = [] - for v in values: - if v not in output: - output.append(v) - return output - - -def _list_to_str(values: Iterable[str] | None, eol: bool = True) -> str: - if values: - output = "\n".join(values) - if eol: - output += "\n" - else: - output = "" - - return output - - -def _clean_pip_reqs(reqs: list[str]) -> list[str]: - return [str(Requirement(r)) for r in reqs] - - -def _clean_conda_deps(deps: list[str], python_version: str | None = None) -> list[str]: - out: list[str] = [] - for dep in deps: - # if have a channel, take it out - if "::" in dep: - channel, d = dep.split("::") - else: - channel, d = None, dep - r = Requirement(d) - - if ( - r.marker - and python_version - and (not r.marker.evaluate({"python_version": python_version})) - ): - continue - else: - r.marker = None - r.extras = set() - if channel: - r.name = f"{channel}::{r.name}" - out.append(str(r)) - return out - - -# * Default parser --------------------------------------------------------------------- - - -@lru_cache -def _default_parser() -> argparse.ArgumentParser: - parser = argparse.ArgumentParser( - description="Parser searches for comments '# p2c: [OPTIONS] CONDA-PACKAGES" - ) - - parser.add_argument( - "-c", - "--channel", - type=str, - help="Channel to add to the pyproject requirement", - ) - parser.add_argument( - "-p", - "--pip", - action="store_true", - help="If specified, install pyproject dependency with pip", - ) - parser.add_argument( - "-s", - "--skip", - action="store_true", - help="If specified skip pyproject dependency on this line", - ) - - parser.add_argument("packages", nargs="*") - - return parser - - -def _factory_empty_tomlkit_Array() -> tomlkit.items.Array: - return tomlkit.items.Array([], tomlkit.items.Trivia()) - +from pyproject2conda.utils import ( + get_in, + list_to_str, + unique_list, +) +from pyproject2conda.utils import ( + remove_whitespace_list as _remove_whitespace_list, +) -def _iter_value_comment_pairs( - array: tomlkit.items.Array, # pyright: ignore -) -> Generator[tuple[OptStr, OptStr], None, None]: - """Extract value and comments from array""" - for v in array._value: # pyright: ignore - if v.value is not None and not isinstance( - v.value, tomlkit.items.Null - ): # pyright: ignore - value = str(v.value) # pyright: ignore - else: - value = None - if v.comment: # pyright: ignore - comment = v.comment.as_string() - else: - comment = None - if value is None and comment is None: - continue - yield (value, comment) +from .requirements import ( + OverrideDict, + _add_header, + _check_allow_empty, + _clean_conda_strings, + _clean_pip_reqs, + _factory_empty_tomlkit_Array, + _factory_empty_tomlkit_Table, + _iter_value_comment_pairs, + _optional_write, + parse_p2c_comment, +) def _matches_package_name( @@ -248,47 +140,20 @@ def _pyproject_to_value_comment_pairs( opts=get_in( ["project", "optional-dependencies"], data, - factory=_factory_empty_tomlkit_Array, + factory=_factory_empty_tomlkit_Table, ), include_base_dependencies=include_base_dependencies, ) if unique: - value_comment_list = _unique_list(value_comment_list) + value_comment_list = unique_list(value_comment_list) return value_comment_list -# * Parsing p2c comments -def _match_p2c_comment(comment: OptStr) -> OptStr: - if not comment or not (match := re.match(r".*?#\s*p2c:\s*([^\#]*)", comment)): - return None - elif re.match(r".*?##\s*p2c:", comment): - # This checks for double ##. If found, ignore line - return None - else: - return match.group(1).strip() - - -def _parse_p2c(match: OptStr) -> dict[str, Any] | None: - """Parse match from _match_p2c_comment""" - - if match: - return vars(_default_parser().parse_args(shlex.split(match))) - else: - return None - - -def parse_p2c_comment(comment: OptStr) -> dict[str, Any] | None: - if match := _match_p2c_comment(comment): - return _parse_p2c(match) - else: - return None - - def _pyproject_to_value_parsed_pairs( value_comment_list: list[tuple[OptStr, OptStr]], -) -> list[tuple[str | None, dict[str, Any]]]: +) -> list[tuple[str | None, OverrideDict]]: out = [] for value, comment in value_comment_list: if comment and (parsed := parse_p2c_comment(comment)): @@ -298,17 +163,19 @@ def _pyproject_to_value_parsed_pairs( return out # pyright: ignore -def _format_override_table(override_table: dict[str, Any]) -> dict[str, Any]: +def _format_override_table( + override_table: dict[str, OverrideDict] +) -> dict[str, OverrideDict]: out: dict[str, Any] = {} for name, v in override_table.items(): - new = { + new: OverrideDict = { "pip": v.get("pip", False), "skip": v.get("skip", False), "channel": v.get("channel", None), "packages": v.get("packages", []), } - if new["channel"] and new["channel"].strip() == "pip": + if new["channel"] is not None and new["channel"].strip() == "pip": new["pip"] = True new["channel"] = None @@ -321,16 +188,17 @@ def _format_override_table(override_table: dict[str, Any]) -> dict[str, Any]: def _apply_override_table( - value_parsed_list: list[tuple[str | None, dict[str, Any]]], - override_table: dict[str, Any], -) -> list[tuple[str | None, dict[str, Any]]]: - out: list[tuple[str | None, dict[str, Any]]] = [] + value_parsed_list: list[tuple[str | None, OverrideDict]], + override_table: dict[str, OverrideDict], +) -> list[tuple[str | None, OverrideDict]]: + out: list[tuple[str | None, OverrideDict]] = [] override_table = _format_override_table(override_table) + new_parsed: OverrideDict for value, parsed in value_parsed_list: if value and (name := Requirement(value).name) in override_table: - new_parsed = dict(override_table[name], **parsed) + new_parsed = dict(override_table[name], **parsed) # type: ignore else: new_parsed = parsed @@ -440,7 +308,7 @@ def pyproject_to_conda_lists( ) # clean up - output["dependencies"] = _clean_conda_deps( + output["dependencies"] = _clean_conda_strings( output["dependencies"], python_version=python_version ) output["pip"] = _clean_pip_reqs(output["pip"]) @@ -456,7 +324,7 @@ def pyproject_to_conda_lists( # ) if remove_whitespace: - output = {k: [_remove_whitespace(x) for x in v] for k, v in output.items()} + output = {k: _remove_whitespace_list(v) for k, v in output.items()} return output @@ -498,49 +366,6 @@ def pyproject_to_conda( ) -def _create_header(cmd: str | None = None) -> str: - from textwrap import dedent - - if cmd: - header = dedent( - f""" - This file is autogenerated by pyproject2conda - with the following command: - - $ {cmd} - - You should not manually edit this file. - Instead edit the corresponding pyproject.toml file. - """ - ) - else: - header = dedent( - """ - This file is autogenerated by pyproject2conda. - You should not manually edit this file. - Instead edit the corresponding pyproject.toml file. - """ - ) - - # prepend '# ' - lines = [] - for line in header.split("\n"): - if len(line.strip()) == 0: - lines.append("#") - else: - lines.append("# " + line) - header = "\n".join(lines) - # header = "\n".join(["# " + line for line in header.split("\n")]) - return header - - -def _add_header(string: str, header_cmd: str | None) -> str: - if header_cmd is not None: - return _create_header(header_cmd) + "\n" + string - else: - return string - - def _yaml_to_string( data: dict[str, Any], yaml: Any = None, @@ -563,18 +388,6 @@ def _yaml_to_string( return _add_header(val.decode("utf-8"), header_cmd) -def _optional_write( - string: str, stream: str | Path | TextIO | None, mode: str = "w" -) -> None: - if stream is None: - return - if isinstance(stream, (str, Path)): - with open(stream, mode) as f: - f.write(string) - else: - stream.write(string) - - def _output_to_yaml( dependencies: list[str] | None, channels: list[str] | None = None, @@ -708,7 +521,7 @@ def to_requirement_list( out = _clean_pip_reqs(out) if remove_whitespace: - out = [_remove_whitespace(x) for x in out] + out = _remove_whitespace_list(out) if sort: return sorted(out) @@ -741,7 +554,7 @@ def to_requirements( if not reqs: return _check_allow_empty(allow_empty) else: - s = _add_header(_list_to_str(reqs), header_cmd) + s = _add_header(list_to_str(reqs), header_cmd) _optional_write(s, stream) return s @@ -784,8 +597,8 @@ def to_conda_requirements( if deps: deps = [dep if "::" in dep else f"{channel}::{dep}" for dep in deps] - deps_str = _add_header(_list_to_str(deps), header_cmd) - reqs_str = _add_header(_list_to_str(reqs), header_cmd) + deps_str = _add_header(list_to_str(deps), header_cmd) + reqs_str = _add_header(list_to_str(reqs), header_cmd) if stream_conda and deps_str: _optional_write(deps_str, stream_conda) diff --git a/src/pyproject2conda/requirements.py b/src/pyproject2conda/requirements.py new file mode 100644 index 0000000..1ad3ca9 --- /dev/null +++ b/src/pyproject2conda/requirements.py @@ -0,0 +1,879 @@ +""" +Requirements parsing (:mod:`~pyproject2conda.requirements`) +=========================================================== +""" + +from __future__ import annotations + +import argparse +import re +import shlex +from functools import cached_property, lru_cache +from pathlib import Path +from typing import TYPE_CHECKING, TypedDict, cast + +if TYPE_CHECKING: + from typing import ( + Any, + Callable, + Generator, + Iterable, + Sequence, + TextIO, + ) + + import tomlkit.container + import tomlkit.items + import tomlkit.toml_document + + from ._typing import ( + MISSING_TYPE, + OptStr, + OptStrSeq, + RequirementCommentPair, + RequirementOverridePair, + ) + from ._typing_compat import Self + +from copy import copy + +import tomlkit +from packaging.markers import Marker +from packaging.requirements import Requirement +from packaging.specifiers import SpecifierSet + +from pyproject2conda.utils import ( + MISSING, + get_in, + list_to_str, + unique_list, +) +from pyproject2conda.utils import ( + remove_whitespace_list as _remove_whitespace_list, +) + + +# * Utilities -------------------------------------------------------------------------- +def _check_allow_empty(allow_empty: bool) -> str: + msg = "No dependencies for this environment\n" + if allow_empty: + return msg + else: + raise ValueError(msg) + + +# ** Requirement +def _clean_pip_reqs(reqs: list[str]) -> list[str]: + return [str(Requirement(r)) for r in reqs] + + +def _clean_conda_requirement( + requirement: Requirement, + python_version: str | None = None, + channel: str | None = None, +) -> Requirement | None: + if ( + python_version + and requirement.marker + and (not requirement.marker.evaluate({"python_version": python_version})) + ): + return None + else: + requirement = _update_requirement(requirement, marker=None, extras=None) + if channel: + requirement.name = f"{channel}::{requirement.name}" + return requirement + + +def _clean_conda_strings( + deps: list[str], python_version: str | None = None +) -> list[str]: + out: list[str] = [] + for dep in deps: + # if have a channel, take it out + if "::" in dep: + channel, d = dep.split("::") + else: + channel, d = None, dep + + r = _clean_conda_requirement( + Requirement(d), python_version=python_version, channel=channel + ) + if r is not None: + out.append(str(r)) + return out + + +def _update_requirement( + requirement: str | Requirement, + name: str | MISSING_TYPE = MISSING, + url: str | None | MISSING_TYPE = MISSING, + extras: str | Iterable[str] | None | MISSING_TYPE = MISSING, + specifier: str | SpecifierSet | None | MISSING_TYPE = MISSING, + marker: str | Marker | None | MISSING_TYPE = MISSING, +) -> Requirement: + if isinstance(requirement, str): + requirement = Requirement(requirement) + else: + requirement = copy(requirement) + + if name is not MISSING: + requirement.name = name + + if url is not MISSING: + requirement.url = url + + if extras is not MISSING: + if extras is None: + extras = set() + elif isinstance(extras, str): + extras = {extras} + else: + extras = set(extras) + + requirement.extras = extras + + if specifier is not MISSING: + if specifier is None: + specifier = SpecifierSet() + elif isinstance(specifier, str): + specifier = SpecifierSet(specifier) + + requirement.specifier = specifier + + if marker is not MISSING: + if isinstance(marker, str): + marker = Marker(marker) + + requirement.marker = marker + + return requirement + + +# ** Dependencices +def _iter_value_comment_pairs( + array: tomlkit.items.Array, # pyright: ignore +) -> Generator[tuple[OptStr, OptStr], None, None]: + """Extract value and comments from array""" + for v in array._value: # pyright: ignore + if v.value is not None and not isinstance( + v.value, tomlkit.items.Null + ): # pyright: ignore + value = str(v.value) # pyright: ignore + else: + value = None + if v.comment: # pyright: ignore + comment = v.comment.as_string() + else: + comment = None + if value is None and comment is None: + continue + yield (value, comment) + + +def _requirement_comment_pairs( + array: tomlkit.items.Array, +) -> list[RequirementCommentPair]: + out = [] + for value, comment in _iter_value_comment_pairs(array): + if value is None: + r = None + else: + r = Requirement(value) + out.append((r, comment)) + return out + + +def _resolve_extras( + extras: str | Iterable[str], + package_name: str, + mapping_requirement_comment_pairs: dict[str, list[RequirementCommentPair]], +) -> list[RequirementCommentPair]: + if isinstance(extras, str): + extras = [extras] + + out = [] + for extra in extras: + for requirement, comment in mapping_requirement_comment_pairs[extra]: + if requirement is not None and requirement.name == package_name: + out.extend( + _resolve_extras( + extras=requirement.extras, + package_name=package_name, + mapping_requirement_comment_pairs=mapping_requirement_comment_pairs, + ) + ) + else: + out.append((requirement, comment)) + return out + + +# ** factories +def _factory_empty_tomlkit_Array() -> tomlkit.items.Array: + return tomlkit.items.Array([], tomlkit.items.Trivia()) + + +def _factory_empty_tomlkit_Table() -> tomlkit.items.Table: + return tomlkit.items.Table( + value=tomlkit.container.Container(), + trivia=tomlkit.items.Trivia(), + is_aot_element=False, + ) + + +# * Parsing p2c commets ---------------------------------------------------------------- +@lru_cache +def p2c_argparser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description="Parser searches for comments '# p2c: [OPTIONS] CONDA-PACKAGES" + ) + + parser.add_argument( + "-c", + "--channel", + type=str, + help="Channel to add to the pyproject requirement", + ) + parser.add_argument( + "-p", + "--pip", + action="store_true", + help="If specified, install pyproject dependency with pip", + ) + parser.add_argument( + "-s", + "--skip", + action="store_true", + help="If specified skip pyproject dependency on this line", + ) + + parser.add_argument("packages", nargs="*") + + return parser + + +class OverrideDict(TypedDict, total=False): + """Dict for storing override options.""" + + pip: bool + skip: bool + channel: str | None + packages: str | list[str] + + +def _match_p2c_comment(comment: OptStr) -> OptStr: + if not comment or not (match := re.match(r".*?#\s*p2c:\s*([^\#]*)", comment)): + return None + elif re.match(r".*?##\s*p2c:", comment): + # This checks for double ##. If found, ignore line + return None + else: + return match.group(1).strip() + + +def _parse_p2c(match: OptStr) -> OverrideDict | None: + """Parse match from _match_p2c_comment""" + + if match: + return cast(OverrideDict, vars(p2c_argparser().parse_args(shlex.split(match)))) + else: + return None + + +def parse_p2c_comment(comment: OptStr) -> OverrideDict | None: + if match := _match_p2c_comment(comment): + return _parse_p2c(match) + else: + return None + + +class OverrideDeps: + """Class to work with overrides from comment or table""" + + def __init__( + self, + pip: bool = False, + skip: bool = False, + packages: str | list[str] | None = None, + channel: str | None = None, + ): + if channel is not None and channel.strip() in ["pip", "pypi"]: + channel = None + pip = True + + self.pip = pip + self.skip = skip + self.channel = channel + + if packages is None: + packages = [] + elif isinstance(packages, str): + packages = [packages] + self.packages = packages + + def __repr__(self) -> str: + return repr(self.__dict__) + + @classmethod + def from_comment( + cls, comment: str | None, default: OverrideDict | None = None + ) -> Self | None: + parsed = parse_p2c_comment(comment) + + kws: OverrideDict + if parsed is None: + if default is None: + return None + else: + kws = default + else: + if default: + kws = dict(default, **parsed) # type: ignore + else: + kws = parsed + + return cls(**kws) + + @classmethod + def requirement_comment_to_override_pairs( + cls, + requirement_comment_pairs: list[RequirementCommentPair], + override_table: dict[str, OverrideDict], + ) -> list[RequirementOverridePair]: + out = [] + for requirement, comment in requirement_comment_pairs: + if requirement is not None: + default = override_table.get(requirement.name, None) + else: + default = None + + if ( + override := cls.from_comment(comment=comment, default=default) + ) is not None or requirement is not None: + out.append((requirement, override)) + return out + + +# * Parser ----------------------------------------------------------------------------- +def _conda_yaml( + name: str | None = None, + channels: str | Iterable[str] | None = None, + conda_deps: str | Iterable[str] | None = None, + pip_deps: str | Iterable[str] | None = None, + add_file_eol: bool = True, +) -> str: + def _as_list(x: str | Iterable[str]) -> Iterable[str]: + if isinstance(x, str): + return [x] + else: + return x + + if not conda_deps and not pip_deps: + raise ValueError + + out = [] + if name is not None: + out.append(f"name: {name}") + + if channels is not None: + out.append("channels:") + + for channel in _as_list(channels): + out.append(f" - {channel}") + + out.append("dependencies:") + + if conda_deps is not None: + for dep in _as_list(conda_deps): + out.append(f" - {dep}") + + if pip_deps is not None: + out.append(" - pip") + out.append(" - pip:") + + for dep in _as_list(pip_deps): + out.append(f" - {dep}") + + s = "\n".join(out) + + if add_file_eol: + s += "\n" + + return s + + +def _create_header(cmd: str | None = None) -> str: + from textwrap import dedent + + if cmd: + header = dedent( + f""" + This file is autogenerated by pyproject2conda + with the following command: + + $ {cmd} + + You should not manually edit this file. + Instead edit the corresponding pyproject.toml file. + """ + ) + else: + header = dedent( + """ + This file is autogenerated by pyproject2conda. + You should not manually edit this file. + Instead edit the corresponding pyproject.toml file. + """ + ) + + # prepend '# ' + lines = [] + for line in header.split("\n"): + if len(line.strip()) == 0: + lines.append("#") + else: + lines.append("# " + line) + header = "\n".join(lines) + # header = "\n".join(["# " + line for line in header.split("\n")]) + return header + + +def _add_header(string: str, header_cmd: str | None) -> str: + if header_cmd is not None: + return _create_header(header_cmd) + "\n" + string + else: + return string + + +def _optional_write( + string: str, stream: str | Path | TextIO | None, mode: str = "w" +) -> None: + if stream is None: + return + if isinstance(stream, (str, Path)): + with open(stream, mode) as f: + f.write(string) + else: + stream.write(string) + + +class ParseDepends: + """ + Parse pyproject.toml file for dependencies + + Parameters + ---------- + data : tomlkit + """ + + def __init__(self, data: tomlkit.toml_document.TOMLDocument): + self.data = data + + def get_in( + self, *keys: str, default: Any = None, factory: Callable[[], Any] | None = None + ) -> Any: + return get_in( + keys=keys, nested_dict=self.data, default=default, factory=factory + ) + + @cached_property + def package_name(self) -> str: + if (out := self.get_in("project", "name")) is None: + raise ValueError("Must specify `project.name`") + return cast(str, out) + + @cached_property + def dependencies(self) -> tomlkit.items.Array: + """project.dependencies""" + return cast( + "tomlkit.items.Array", + self.get_in( + "project", "dependencies", factory=_factory_empty_tomlkit_Array + ), + ) + + @cached_property + def optional_dependencies(self) -> tomlkit.items.Table: + """project.optional-dependencies""" + return cast( + "tomlkit.items.Table", + self.get_in( + "project", "optional-dependencies", factory=_factory_empty_tomlkit_Table + ), + ) + + @cached_property + def override_table(self) -> dict[str, OverrideDict]: + out = self.get_in("tool", "pyproject2conda", "dependencies", default=MISSING) + if out is MISSING: + out = {} + else: + out = out.unwrap() + return cast(dict[str, OverrideDict], out) + + @cached_property + def channels(self) -> list[str]: + channels_doc = self.get_in("tool", "pyproject2conda", "channels") + if channels_doc: + channels = channels_doc.unwrap() + if isinstance(channels, str): + channels = [channels] + else: + channels = list(channels) + else: + channels = [] + + return channels # type: ignore + + @property + def extras(self) -> list[str]: + return list(self.optional_dependencies.keys()) + + @cached_property + def _requirement_override_pairs_base( + self, + ) -> list[RequirementOverridePair]: + return OverrideDeps.requirement_comment_to_override_pairs( + requirement_comment_pairs=_requirement_comment_pairs(self.dependencies), + override_table=self.override_table, + ) + + @cached_property + def _requirement_override_pairs_extras( + self, + ) -> dict[str, list[RequirementOverridePair]]: + """ + Mapping[extra_name] -> [(requirement, comment)] + + Note that this also resolves self references of the form + "package_name[extra,..]" to the actual dependencies. + """ + unresolved: dict[str, list[RequirementCommentPair]] = { + k: _requirement_comment_pairs(v) + for k, v in self.optional_dependencies.items() + } + + resolved = { + k: _resolve_extras( + extras=k, + package_name=self.package_name, + mapping_requirement_comment_pairs=unresolved, + ) + for k, v in unresolved.items() + } + + # comments -> overrides + return { + k: OverrideDeps.requirement_comment_to_override_pairs( + requirement_comment_pairs=v, override_table=self.override_table + ) + for k, v in resolved.items() + } + + @staticmethod + def _cleanup( + values: list[str], + remove_whitespace: bool = True, + unique: bool = True, + sort: bool = True, + ) -> list[str]: + if remove_whitespace: + values = _remove_whitespace_list(values) + + if unique: + values = unique_list(values) + + if sort: + values = sorted(values) + + return values + + def _get_requirement_override_pairs( + self, extras: str | Iterable[str] | None = None, include_base: bool = True + ) -> list[RequirementOverridePair]: + out: list[RequirementOverridePair] = [] + + if include_base: + out.extend(self._requirement_override_pairs_base) + + if extras is not None: + if isinstance(extras, str): + extras = [extras] + for extra in extras: + out.extend(self._requirement_override_pairs_extras[extra]) + + return out + + def _check_extras(self, extras: str | Iterable[str] | None) -> None: + if extras is None: + return + elif isinstance(extras, str): + extras = [extras] + + for extra in extras: + if extra not in self.extras: + raise ValueError(f"{extras} not in {self.extras}") + + def pip_requirements( + self, + extras: str | Iterable[str] | None = None, + include_base: bool = True, + pip_deps: str | Iterable[str] | None = None, + unique: bool = True, + remove_whitespace: bool = True, + sort: bool = True, + ) -> list[str]: + self._check_extras(extras) + + out: list[str] = [ + str(requirement) + for requirement, _ in self._get_requirement_override_pairs( + extras=extras, include_base=include_base + ) + if requirement is not None + ] + + # TODO: extra checks? + if pip_deps: + if isinstance(pip_deps, str): + pip_deps = [pip_deps] + else: + pip_deps = list(pip_deps) + out.extend(_clean_pip_reqs(pip_deps)) + + return self._cleanup( + out, remove_whitespace=remove_whitespace, unique=unique, sort=sort + ) + + def conda_pip_requirements( + self, + extras: str | Iterable[str] | None = None, + include_base: bool = True, + pip_deps: str | Iterable[str] | None = None, + conda_deps: str | Iterable[str] | None = None, + unique: bool = True, + remove_whitespace: bool = True, + sort: bool = True, + python_version: str | None = None, + python_include: str | None = None, + ) -> tuple[list[str], list[str]]: + self._check_extras(extras) + + if python_include == "infer": + # safer get + if (x := self.get_in("project", "requires-python")) is None: + raise ValueError( + "No value for `requires-python` in pyproject.toml file" + ) + else: + python_include = "python" + str(x) + + def _init_deps(deps: str | Iterable[str] | None) -> list[str]: + if deps is None: + return [] + elif isinstance(deps, str): + return [deps] + else: + return list(deps) + + pip_deps = _clean_pip_reqs(_init_deps(pip_deps)) + conda_deps = _clean_conda_strings( + _init_deps(conda_deps), python_version=python_version + ) + + for requirement, override in self._get_requirement_override_pairs( + extras=extras, include_base=include_base + ): + if override is not None: + if override.pip: + assert requirement is not None + pip_deps.append(str(requirement)) + elif not override.skip: + if requirement is None: + print(requirement, override) + assert requirement is not None + conda_deps.append( + str( + _clean_conda_requirement( + requirement, + python_version=python_version, + channel=override.channel, + ) + ) + ) + + conda_deps.extend( + _clean_conda_strings( + override.packages, python_version=python_version + ) + ) + + elif requirement: + conda_deps.append( + str( + _clean_conda_requirement( + requirement, python_version=python_version + ) + ) + ) + + pip_deps, conda_deps = ( + self._cleanup( + x, remove_whitespace=remove_whitespace, unique=unique, sort=sort + ) + for x in (pip_deps, conda_deps) + ) + + if python_include is not None: + conda_deps = ( + self._cleanup([python_include], remove_whitespace=remove_whitespace) + + conda_deps + ) + + return conda_deps, pip_deps + + def to_conda_yaml( + self, + extras: OptStrSeq = None, + pip_deps: str | Iterable[str] | None = None, + conda_deps: str | Iterable[str] | None = None, + name: OptStr = None, + channels: OptStrSeq = None, + python_include: OptStr = None, + python_version: OptStr = None, + include_base: bool = True, + header_cmd: str | None = None, + stream: str | Path | TextIO | None = None, + sort: bool = True, + remove_whitespace: bool = True, + unique: bool = True, + allow_empty: bool = False, + ) -> str: + self._check_extras(extras) + + conda_deps, pip_deps = self.conda_pip_requirements( + extras=extras, + include_base=include_base, + pip_deps=pip_deps, + conda_deps=conda_deps, + unique=unique, + remove_whitespace=remove_whitespace, + sort=sort, + python_include=python_include, + python_version=python_version, + ) + + if not conda_deps and not pip_deps: + return _check_allow_empty(allow_empty) + + out = _conda_yaml( + name=name, + channels=channels or self.channels, + conda_deps=conda_deps, + pip_deps=pip_deps, + ) + + out = _add_header(out, header_cmd) + + _optional_write(out, stream) + + return out + + def to_requirements( + self, + extras: OptStrSeq = None, + include_base: bool = True, + header_cmd: str | None = None, + stream: str | Path | TextIO | None = None, + sort: bool = True, + pip_deps: Sequence[str] | None = None, + allow_empty: bool = False, + remove_whitespace: bool = True, + ) -> str: + pip_deps = self.pip_requirements( + extras=extras, + include_base=include_base, + pip_deps=pip_deps, + remove_whitespace=remove_whitespace, + sort=sort, + ) + + if not pip_deps: + return _check_allow_empty(allow_empty) + + out = _add_header(list_to_str(pip_deps), header_cmd) + + _optional_write(out, stream) + return out + + def to_conda_requirements( + self, + extras: str | Iterable[str] | None = None, + channels: str | Iterable[str] | None = None, + python_include: str | None = None, + python_version: str | None = None, + prepend_channel: bool = False, + stream_conda: str | Path | TextIO | None = None, + stream_pip: str | Path | TextIO | None = None, + include_base: bool = True, + header_cmd: str | None = None, + sort: bool = True, + unique: bool = True, + conda_deps: str | Iterable[str] | None = None, + pip_deps: str | Iterable[str] | None = None, + remove_whitespace: bool = True, + ) -> tuple[str, str]: + conda_deps, pip_deps = self.conda_pip_requirements( + extras=extras, + include_base=include_base, + pip_deps=pip_deps, + conda_deps=conda_deps, + unique=unique, + remove_whitespace=remove_whitespace, + sort=sort, + python_include=python_include, + python_version=python_version, + ) + + if channels: + if isinstance(channels, str): + channels = [channels] + else: + channels = list(channels) + else: + channels = self.channels + + if conda_deps and channels and prepend_channel: + assert len(channels) == 1 + channel = channels[0] + # add in channel if none exists + conda_deps = [ + dep if "::" in dep else f"{channel}::{dep}" for dep in conda_deps + ] + + conda_deps_str = _add_header(list_to_str(conda_deps), header_cmd) + pip_deps_str = _add_header(list_to_str(pip_deps), header_cmd) + + if stream_conda and conda_deps_str: + _optional_write(conda_deps_str, stream_conda) + + if stream_pip and pip_deps_str: + _optional_write(pip_deps_str, stream_pip) + + return conda_deps_str, pip_deps_str + + @classmethod + def from_string( + cls, + toml_string: str, + ) -> Self: + data = tomlkit.parse(toml_string) + return cls(data=data) + + @classmethod + def from_path(cls, path: str | Path) -> Self: + with open(path, "rb") as f: + data = tomlkit.load(f) + return cls(data=data) + + # Output diff --git a/src/pyproject2conda/utils.py b/src/pyproject2conda/utils.py index 8561ebe..60ee6eb 100644 --- a/src/pyproject2conda/utils.py +++ b/src/pyproject2conda/utils.py @@ -5,11 +5,42 @@ from __future__ import annotations +import enum +import re from pathlib import Path from typing import TYPE_CHECKING if TYPE_CHECKING: - from typing import Any, Callable, Mapping, Sequence + from typing import Any, Callable, Iterable, Mapping, Sequence + + from ._typing import T + + +# taken from https://github.com/python-attrs/attrs/blob/main/src/attr/_make.py +class _Missing(enum.Enum): + """ + Sentinel to indicate the lack of a value when ``None`` is ambiguous. + + If extending attrs, you can use ``typing.Literal[MISSING]`` to show + that a value may be ``MISSING``. + + .. versionchanged:: 21.1.0 ``bool(MISSING)`` is now False. + .. versionchanged:: 22.2.0 ``MISSING`` is now an ``enum.Enum`` variant. + """ + + MISSING = enum.auto() + + def __repr__(self) -> str: + return "MISSING" # pragma: no cover + + def __bool__(self) -> bool: + return False # pragma: no cover + + +MISSING = _Missing.MISSING +""" +Sentinel to indicate the lack of a value when ``None`` is ambiguous. +""" # taken from https://github.com/conda/conda-lock/blob/main/conda_lock/common.py @@ -17,7 +48,7 @@ def get_in( keys: Sequence[Any], nested_dict: Mapping[Any, Any], default: Any = None, - factory: Callable[..., Any] | None = None, + factory: Callable[[], Any] | None = None, ) -> Any: """ >>> foo = {'a': {'b': {'c': 1}}} @@ -37,22 +68,6 @@ def get_in( return default -# def compose_decorators(*decs: Dec[F]) -> Dec[F]: -# from functools import reduce -# def wrapper(func: F) -> F: -# return reduce(lambda x, f: f(x), reversed(decs), func) -# return wrapper - - -# def compose_decorators(*decs: Dec[F]) -> Dec[F]: -# def wrapper(func: F) -> F: -# for d in reversed(decs): -# func = d(func) -# return func - -# return wrapper - - def parse_pythons( python_include: str | None, python_version: str | None, @@ -147,3 +162,37 @@ def filename_from_template( template = template + f".{ext}" return template.format(**kws) + + +_WHITE_SPACE_REGEX = re.compile(r"\s+") + + +def remove_whitespace(s: str) -> str: + return re.sub(_WHITE_SPACE_REGEX, "", s) + + +def remove_whitespace_list(s: Iterable[str]) -> list[str]: + return [remove_whitespace(x) for x in s] + + +def unique_list(values: Iterable[T]) -> list[T]: + """ + Return only unique values in list. + Unlike using set(values), this preserves order. + """ + output: list[T] = [] + for v in values: + if v not in output: + output.append(v) + return output + + +def list_to_str(values: Iterable[str] | None, eol: bool = True) -> str: + if values: + output = "\n".join(values) + if eol: + output += "\n" + else: + output = "" + + return output diff --git a/tests/test_parser.py b/tests/test_parser.py index 6146e24..cee6375 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -17,12 +17,16 @@ def test_get_in(): def test_list_to_string(): - assert parser._list_to_str(["a", "b"], eol=True) == "a\nb\n" - assert parser._list_to_str(["a", "b"], eol=False) == "a\nb" - assert parser._list_to_str(None) == "" + from pyproject2conda.utils import list_to_str + + assert list_to_str(["a", "b"], eol=True) == "a\nb\n" + assert list_to_str(["a", "b"], eol=False) == "a\nb" + assert list_to_str(None) == "" def test_match_p2c_comment(): + from pyproject2conda.requirements import _match_p2c_comment + expected = "-c -d" for comment in [ "#p2c: -c -d", @@ -31,7 +35,7 @@ def test_match_p2c_comment(): "# p2c: -c -d # some other thing", "# some other thing # p2c: -c -d # another thing", ]: - match = parser._match_p2c_comment(comment) + match = _match_p2c_comment(comment) assert match == expected @@ -43,7 +47,7 @@ def test_match_p2c_comment(): "## p2c: -c -d # some other thing", "# some other thing ## p2c: -c -d # another thing", ]: - match = parser._match_p2c_comment(comment) + match = _match_p2c_comment(comment) assert match is None @@ -59,27 +63,29 @@ def get_expected(pip=False, skip=False, channel=None, packages=None): "packages": packages, } - assert parser._parse_p2c(None) is None + from pyproject2conda.requirements import _parse_p2c + + assert _parse_p2c(None) is None - assert parser._parse_p2c("--pip") == get_expected(pip=True) - assert parser._parse_p2c("-p") == get_expected(pip=True) + assert _parse_p2c("--pip") == get_expected(pip=True) + assert _parse_p2c("-p") == get_expected(pip=True) - assert parser._parse_p2c("--skip") == get_expected(skip=True) - assert parser._parse_p2c("-s") == get_expected(skip=True) + assert _parse_p2c("--skip") == get_expected(skip=True) + assert _parse_p2c("-s") == get_expected(skip=True) - assert parser._parse_p2c("-s -c conda-forge") == get_expected( + assert _parse_p2c("-s -c conda-forge") == get_expected( skip=True, channel="conda-forge" ) - assert parser._parse_p2c("athing>=0.3,<0.2 ") == get_expected( + assert _parse_p2c("athing>=0.3,<0.2 ") == get_expected( packages=["athing>=0.3,<0.2"] ) - assert parser._parse_p2c("athing>=0.3,<0.2 bthing ") == get_expected( + assert _parse_p2c("athing>=0.3,<0.2 bthing ") == get_expected( packages=["athing>=0.3,<0.2", "bthing"] ) - assert parser._parse_p2c("'athing >= 0.3, <0.2' bthing ") == get_expected( + assert _parse_p2c("'athing >= 0.3, <0.2' bthing ") == get_expected( packages=["athing >= 0.3, <0.2", "bthing"] ) @@ -98,6 +104,8 @@ def test_value_comment_pairs(): def test_header(): + from pyproject2conda.requirements import _create_header + expected = dedent( """\ # @@ -107,10 +115,10 @@ def test_header(): #""" ) - assert expected == parser._create_header() + assert expected == _create_header() cmd = "hello" - out = parser._create_header(cmd=cmd) + out = _create_header(cmd=cmd) header = dedent( f"""\ @@ -154,12 +162,14 @@ def test_yaml_to_str(): def test_optional_write(): from pathlib import Path + from pyproject2conda.requirements import _optional_write + s = "hello" with tempfile.TemporaryDirectory() as d: p = Path(d) / "tmp.txt" with open(p, "w") as f: - parser._optional_write(s, f) + _optional_write(s, f) with open(p, "r") as f: test = f.read() From 7aea4bbe63bda7f24a8c8b6014758d59228bce1e Mon Sep 17 00:00:00 2001 From: wpk Date: Fri, 17 Nov 2023 09:33:59 -0500 Subject: [PATCH 28/36] feat: refactor parsing -> requirements Cleared up a bunch of sutff. This will hopefully make it easier to make changes going forward. --- README.md | 17 +- src/pyproject2conda/__init__.py | 2 - src/pyproject2conda/_typing.py | 7 +- src/pyproject2conda/cli.py | 54 ++- src/pyproject2conda/overrides.py | 150 +++++++ src/pyproject2conda/parser.py | 665 ---------------------------- src/pyproject2conda/requirements.py | 195 ++------ src/pyproject2conda/utils.py | 2 +- tests/test_cli.py | 2 +- tests/test_parser.py | 237 ++++++++-- 10 files changed, 410 insertions(+), 921 deletions(-) create mode 100644 src/pyproject2conda/overrides.py delete mode 100644 src/pyproject2conda/parser.py diff --git a/README.md b/README.md index 64c3766..ad43905 100644 --- a/README.md +++ b/README.md @@ -132,7 +132,7 @@ Note the comment lines `# p2c:...`. These are special tokens that - + ```bash usage: -c [-h] [-c CHANNEL] [-p] [-s] [packages ...] @@ -222,11 +222,6 @@ build = { channel = "pip" } ``` -Note that the table keys are the package name to be adjusted (no version -modifiers in the name), and the following dictionary is similar to the options -above. Also, you can specify `channel = "pip"` to force the package to be -installed with pip (same as setting `pip = true`). - ### Specify python version @@ -488,8 +483,8 @@ dependencies: `pyproject2conda` can also be used within python: ```pycon ->>> from pyproject2conda import PyProject2Conda ->>> p = PyProject2Conda.from_path("./tests/data/test-pyproject.toml") +>>> from pyproject2conda.requirements import ParseDepends +>>> p = ParseDepends.from_path("./tests/data/test-pyproject.toml") # Basic environment >>> print(p.to_conda_yaml(python_include="infer").strip()) @@ -617,11 +612,11 @@ dependencies: or ```pycon ->>> from pyproject2conda import PyProject2Conda ->>> p = PyProject2Conda.from_path("./tests/data/test-pyproject.toml") +>>> from pyproject2conda.requirements import ParseDepends +>>> p = ParseDepends.from_path("./tests/data/test-pyproject.toml") # Basic environment ->>> print(p.to_conda_yaml(extras='dist-pypi', include_base_dependencies=False).strip()) +>>> print(p.to_conda_yaml(extras='dist-pypi', include_base=False).strip()) channels: - conda-forge dependencies: diff --git a/src/pyproject2conda/__init__.py b/src/pyproject2conda/__init__.py index 2f12e24..c7a7135 100644 --- a/src/pyproject2conda/__init__.py +++ b/src/pyproject2conda/__init__.py @@ -10,7 +10,6 @@ except PackageNotFoundError: # pragma: no cover __version__ = "999" -from .parser import PyProject2Conda __author__ = """William P. Krekelberg""" __email__ = "wpk@nist.gov" @@ -18,5 +17,4 @@ __all__ = [ "__version__", - "PyProject2Conda", ] diff --git a/src/pyproject2conda/_typing.py b/src/pyproject2conda/_typing.py index 3c11a25..22f4849 100644 --- a/src/pyproject2conda/_typing.py +++ b/src/pyproject2conda/_typing.py @@ -17,22 +17,21 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Optional, Sequence, TypeVar, Union +from typing import TYPE_CHECKING, Optional, TypeVar R = TypeVar("R") T = TypeVar("T") OptStr = Optional[str] -OptStrSeq = Union[str, Sequence[str], None] if TYPE_CHECKING: from typing import Literal, Tuple from packaging.requirements import Requirement - from .requirements import OverrideDeps - from .utils import _Missing + from .overrides import OverrideDeps + from .utils import _Missing # pyright: ignore MISSING_TYPE = Literal[_Missing.MISSING] diff --git a/src/pyproject2conda/cli.py b/src/pyproject2conda/cli.py index 4a0e093..c74bc97 100644 --- a/src/pyproject2conda/cli.py +++ b/src/pyproject2conda/cli.py @@ -19,7 +19,9 @@ from typer.core import TyperGroup from pyproject2conda import __version__ -from pyproject2conda.parser import PyProject2Conda + +# from pyproject2conda.parser import PyProject2Conda +from pyproject2conda.requirements import ParseDepends from pyproject2conda.utils import ( parse_pythons, update_target, @@ -365,8 +367,8 @@ def _get_header_cmd( @lru_cache -def _get_pyproject2conda(filename: Union[str, Path]) -> PyProject2Conda: - return PyProject2Conda.from_path(filename) +def _get_requirement_parser(filename: Union[str, Path]) -> ParseDepends: + return ParseDepends.from_path(filename) def _log_skipping( @@ -450,12 +452,12 @@ def create_list( logger.info(f"filename: {filename}") - d = _get_pyproject2conda(filename) + d = _get_requirement_parser(filename) print("Extras:") print("=======") - for extra in d.list_extras(): + for extra in d.extras: print("*", extra) @@ -497,7 +499,7 @@ def yaml( python=python, ) - d = _get_pyproject2conda(filename) + d = _get_requirement_parser(filename) _log_creating(logger, "yaml", output) @@ -508,11 +510,11 @@ def yaml( stream=output, python_include=python_include, python_version=python_version, - include_base_dependencies=base, + include_base=base, header_cmd=_get_header_cmd(header, output), sort=sort, - deps=deps, - reqs=reqs, + conda_deps=deps, + pip_deps=reqs, allow_empty=allow_empty, remove_whitespace=remove_whitespace, ) @@ -543,17 +545,17 @@ def requirements( _log_skipping(logger, "requirements", output) return - d = _get_pyproject2conda(filename) + d = _get_requirement_parser(filename) _log_creating(logger, "requirements", output) s = d.to_requirements( extras=extras, stream=output, - include_base_dependencies=base, + include_base=base, header_cmd=_get_header_cmd(header, output), sort=sort, - reqs=reqs, + pip_deps=reqs, allow_empty=allow_empty, remove_whitespace=remove_whitespace, ) @@ -676,7 +678,7 @@ def conda_requirements( path_conda = prefix + "conda.txt" path_pip = prefix + "pip.txt" - d = _get_pyproject2conda(filename) + d = _get_requirement_parser(filename) _get_header_cmd(header, path_conda) @@ -688,11 +690,11 @@ def conda_requirements( prepend_channel=prepend_channel, stream_conda=path_conda, stream_pip=path_pip, - include_base_dependencies=base, + include_base=base, header_cmd=_get_header_cmd(header, path_conda), sort=sort, - deps=deps, - reqs=reqs, + conda_deps=deps, + pip_deps=reqs, ) if not path_conda: @@ -729,7 +731,7 @@ def to_json( import json - d = _get_pyproject2conda(filename) + d = _get_requirement_parser(filename) python_include, python_version = parse_pythons( python_include=python_include, @@ -737,17 +739,25 @@ def to_json( python=python, ) - result = d.to_conda_lists( + conda_deps, pip_deps = d.conda_pip_requirements( extras=extras, - channels=channels, python_include=python_include, python_version=python_version, - include_base_dependencies=base, + include_base=base, sort=sort, - deps=deps, - reqs=reqs, + conda_deps=deps, + pip_deps=reqs, ) + result = { + "dependencies": conda_deps, + "pip": pip_deps, + } + + channels = channels or d.channels + if channels: + result["channels"] = channels + if output: with open(output, "w") as f: json.dump(result, f) diff --git a/src/pyproject2conda/overrides.py b/src/pyproject2conda/overrides.py new file mode 100644 index 0000000..035cfd6 --- /dev/null +++ b/src/pyproject2conda/overrides.py @@ -0,0 +1,150 @@ +""" +Override dependencies (:mod:`~pyproject2conda.overrides`) +========================================================= +""" + +from __future__ import annotations + +import argparse +import re +import shlex +from functools import lru_cache +from typing import TYPE_CHECKING, TypedDict, cast + +if TYPE_CHECKING: + from ._typing import OptStr, RequirementCommentPair, RequirementOverridePair + from ._typing_compat import Self + + +# * Comment parsing -------------------------------------------------------------------- +@lru_cache +def p2c_argparser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description="Parser searches for comments '# p2c: [OPTIONS] CONDA-PACKAGES" + ) + + parser.add_argument( + "-c", + "--channel", + type=str, + help="Channel to add to the pyproject requirement", + ) + parser.add_argument( + "-p", + "--pip", + action="store_true", + help="If specified, install pyproject dependency with pip", + ) + parser.add_argument( + "-s", + "--skip", + action="store_true", + help="If specified skip pyproject dependency on this line", + ) + + parser.add_argument("packages", nargs="*") + + return parser + + +def _match_p2c_comment(comment: OptStr) -> OptStr: + if not comment or not (match := re.match(r".*?#\s*p2c:\s*([^\#]*)", comment)): + return None + elif re.match(r".*?##\s*p2c:", comment): + # This checks for double ##. If found, ignore line + return None + else: + return match.group(1).strip() + + +def _parse_p2c(match: OptStr) -> OverrideDict | None: + """Parse match from _match_p2c_comment""" + + if match: + return cast(OverrideDict, vars(p2c_argparser().parse_args(shlex.split(match)))) + else: + return None + + +def _parse_p2c_comment(comment: OptStr) -> OverrideDict | None: + if match := _match_p2c_comment(comment): + return _parse_p2c(match) + else: + return None + + +# * Main classes ----------------------------------------------------------------------- +class OverrideDict(TypedDict, total=False): + """Dict for storing override options.""" + + pip: bool + skip: bool + channel: str | None + packages: str | list[str] + + +class OverrideDeps: + """Class to work with overrides from comment or table""" + + def __init__( + self, + pip: bool = False, + skip: bool = False, + packages: str | list[str] | None = None, + channel: str | None = None, + ): + if channel is not None and channel.strip() in ["pip", "pypi"]: + channel = None + pip = True + + self.pip = pip + self.skip = skip + self.channel = channel + + if packages is None: + packages = [] + elif isinstance(packages, str): + packages = [packages] + self.packages = packages + + def __repr__(self) -> str: # pragma: no cover + return repr(self.__dict__) + + @classmethod + def from_comment( + cls, comment: str | None, default: OverrideDict | None = None + ) -> Self | None: + parsed = _parse_p2c_comment(comment) + + kws: OverrideDict + if parsed is None: + if default is None: + return None + else: + kws = default + else: + if default: + kws = dict(default, **parsed) # type: ignore + else: + kws = parsed + + return cls(**kws) + + @classmethod + def requirement_comment_to_override_pairs( + cls, + requirement_comment_pairs: list[RequirementCommentPair], + override_table: dict[str, OverrideDict], + ) -> list[RequirementOverridePair]: + out: list[RequirementOverridePair] = [] + for requirement, comment in requirement_comment_pairs: + if requirement is not None: + default = override_table.get(requirement.name, None) + else: + default = None + + if ( + override := cls.from_comment(comment=comment, default=default) + ) is not None or requirement is not None: + out.append((requirement, override)) + return out diff --git a/src/pyproject2conda/parser.py b/src/pyproject2conda/parser.py deleted file mode 100644 index 72192a6..0000000 --- a/src/pyproject2conda/parser.py +++ /dev/null @@ -1,665 +0,0 @@ -# pyright: reportUnknownMemberType=false, reportGeneralTypeIssues=false -""" -Parse `pyproject.toml` (:mod:`~pyproject2conda.parser`) -======================================================= - -Main parser to turn `pyproject.toml` to other formats. -""" -from __future__ import annotations - -from pathlib import Path -from typing import ( - TYPE_CHECKING, - cast, -) - -if TYPE_CHECKING: - from typing import ( - Any, - Sequence, - TextIO, - ) - - import tomlkit.container - import tomlkit.items - import tomlkit.toml_document - - from ._typing import OptStr, OptStrSeq - from ._typing_compat import Self - -import tomlkit -from packaging.requirements import Requirement -from ruamel.yaml import YAML - -from pyproject2conda.utils import ( - get_in, - list_to_str, - unique_list, -) -from pyproject2conda.utils import ( - remove_whitespace_list as _remove_whitespace_list, -) - -from .requirements import ( - OverrideDict, - _add_header, - _check_allow_empty, - _clean_conda_strings, - _clean_pip_reqs, - _factory_empty_tomlkit_Array, - _factory_empty_tomlkit_Table, - _iter_value_comment_pairs, - _optional_write, - parse_p2c_comment, -) - - -def _matches_package_name( - dep: OptStr, - package_name: str, -) -> list[str] | None: - """ - Check if `dep` matches pattern {package_name}[extra,..] - - If it does, return extras, else return None - """ - if not dep: - extras = None - - else: - r = Requirement(dep) - - if r.name == package_name and r.extras: - extras = list(r.extras) - else: - extras = None - return extras - - -def _get_value_comment_pairs( - package_name: str, - deps: tomlkit.items.Array, - extras: OptStrSeq = None, - opts: tomlkit.items.Table | None = None, - include_base_dependencies: bool = True, -) -> list[tuple[OptStr, OptStr]]: - """Recursively build dependency, comment pairs from deps and extras.""" - if include_base_dependencies: - out = list(_iter_value_comment_pairs(deps)) - else: - out = [] - - if extras is None: - return out - else: - assert opts is not None - - if isinstance(extras, str): - extras = [extras] - - for extra in extras: - for value, comment in _iter_value_comment_pairs( - cast(tomlkit.items.Array, opts[extra]) - ): - if new_extras := _matches_package_name(value, package_name): - out.extend( - _get_value_comment_pairs( - package_name=package_name, - extras=new_extras, - deps=deps, - opts=opts, - include_base_dependencies=False, - ) - ) - else: - out.append((value, comment)) - - return out - - -def _pyproject_to_value_comment_pairs( - data: tomlkit.toml_document.TOMLDocument, - extras: OptStrSeq = None, - unique: bool = True, - include_base_dependencies: bool = True, -) -> list[tuple[OptStr, OptStr]]: - package_name = cast("str | None", get_in(["project", "name"], data, default=None)) - - if package_name is None: - raise ValueError("Must specify `project.package_name` in pyproject.toml") - - deps = cast( - tomlkit.items.Array, - get_in(["project", "dependencies"], data, factory=_factory_empty_tomlkit_Array), - ) - - value_comment_list = _get_value_comment_pairs( - package_name=package_name, - extras=extras, - deps=deps, - opts=get_in( - ["project", "optional-dependencies"], - data, - factory=_factory_empty_tomlkit_Table, - ), - include_base_dependencies=include_base_dependencies, - ) - - if unique: - value_comment_list = unique_list(value_comment_list) - - return value_comment_list - - -def _pyproject_to_value_parsed_pairs( - value_comment_list: list[tuple[OptStr, OptStr]], -) -> list[tuple[str | None, OverrideDict]]: - out = [] - for value, comment in value_comment_list: - if comment and (parsed := parse_p2c_comment(comment)): - out.append((value, parsed)) - elif value: - out.append((value, {})) - return out # pyright: ignore - - -def _format_override_table( - override_table: dict[str, OverrideDict] -) -> dict[str, OverrideDict]: - out: dict[str, Any] = {} - for name, v in override_table.items(): - new: OverrideDict = { - "pip": v.get("pip", False), - "skip": v.get("skip", False), - "channel": v.get("channel", None), - "packages": v.get("packages", []), - } - - if new["channel"] is not None and new["channel"].strip() == "pip": - new["pip"] = True - new["channel"] = None - - if isinstance(new["packages"], str): - new["packages"] = [new["packages"]] - - out[name] = new - - return out - - -def _apply_override_table( - value_parsed_list: list[tuple[str | None, OverrideDict]], - override_table: dict[str, OverrideDict], -) -> list[tuple[str | None, OverrideDict]]: - out: list[tuple[str | None, OverrideDict]] = [] - - override_table = _format_override_table(override_table) - - new_parsed: OverrideDict - for value, parsed in value_parsed_list: - if value and (name := Requirement(value).name) in override_table: - new_parsed = dict(override_table[name], **parsed) # type: ignore - else: - new_parsed = parsed - - out.append((value, new_parsed)) - - return out - - -# * To dependency list -def value_comment_pairs_to_conda( - value_comment_list: list[tuple[OptStr, OptStr]], - sort: bool = True, - deps: Sequence[str] | None = None, - reqs: Sequence[str] | None = None, - override_table: dict[str, Any] | None = None, -) -> dict[str, Any]: - """Convert raw value/comment pairs to install lines""" - - conda_deps: list[str] = [] - pip_deps: list[str] = [] - - def _check_value(value: Any) -> None: - if not value: - raise ValueError("trying to add value that does not exist") - - value_parsed_list = _pyproject_to_value_parsed_pairs(value_comment_list) - - if override_table: - value_parsed_list = _apply_override_table(value_parsed_list, override_table) - - for value, parsed in value_parsed_list: - if parsed: - if parsed["pip"]: - _check_value(value) - pip_deps.append(value) # type: ignore - elif not parsed["skip"]: - _check_value(value) - if parsed["channel"]: - conda_deps.append("{}::{}".format(parsed["channel"], value)) - else: - conda_deps.append(value) # type: ignore - - conda_deps.extend(parsed["packages"]) - elif value: - conda_deps.append(value) - - if deps: - conda_deps.extend(list(deps)) - - if reqs: - pip_deps.extend(list(reqs)) - - if sort: - conda_deps = sorted(conda_deps) - pip_deps = sorted(pip_deps) - - return {"dependencies": conda_deps, "pip": pip_deps} - - -def pyproject_to_conda_lists( - data: tomlkit.toml_document.TOMLDocument, - extras: OptStrSeq = None, - channels: OptStrSeq = None, - python_include: OptStr = None, - python_version: OptStr = None, - include_base_dependencies: bool = True, - sort: bool = True, - deps: Sequence[str] | None = None, - reqs: Sequence[str] | None = None, - remove_whitespace: bool = True, - override_table: dict[str, Any] | None = None, -) -> dict[str, Any]: - if python_include == "infer": - # safer get - if (x := get_in(["project", "requires-python"], data)) is None: - raise ValueError("No value for `requires-python` in pyproject.toml file") - else: - python_include = "python" + x.unwrap() - - if channels is None: - channels_doc = get_in(["tool", "pyproject2conda", "channels"], data, None) - if channels_doc: - channels = channels_doc.unwrap() - - if override_table is None: - override_table_ = get_in( - ["tool", "pyproject2conda", "dependencies"], data, None - ) - if override_table_: - override_table = override_table_.unwrap() - - if isinstance(channels, str): - channels = [channels] - - value_comment_list = _pyproject_to_value_comment_pairs( - data=data, - extras=extras, - include_base_dependencies=include_base_dependencies, - ) - - output = value_comment_pairs_to_conda( - value_comment_list, - sort=sort, - deps=deps, - reqs=reqs, - override_table=override_table, - ) - - # clean up - output["dependencies"] = _clean_conda_strings( - output["dependencies"], python_version=python_version - ) - output["pip"] = _clean_pip_reqs(output["pip"]) - - if python_include: - output["dependencies"].insert(0, python_include) - if channels: - output["channels"] = channels - - # # limit python version/remove python_verions <=> part - # output["dependencies"] = _limit_deps_by_python_version( - # output["dependencies"], python_version - # ) - - if remove_whitespace: - output = {k: _remove_whitespace_list(v) for k, v in output.items()} - - return output - - -def pyproject_to_conda( - data: tomlkit.toml_document.TOMLDocument, - extras: OptStrSeq = None, - channels: OptStrSeq = None, - name: OptStr = None, - python_include: OptStr = None, - stream: str | Path | TextIO | None = None, - python_version: OptStr = None, - include_base_dependencies: bool = True, - header_cmd: OptStr = None, - sort: bool = True, - deps: Sequence[str] | None = None, - reqs: Sequence[str] | None = None, - allow_empty: bool = False, - remove_whitespace: bool = True, -) -> str: - output = pyproject_to_conda_lists( - data=data, - extras=extras, - channels=channels, - python_include=python_include, - python_version=python_version, - include_base_dependencies=include_base_dependencies, - sort=sort, - deps=deps, - reqs=reqs, - remove_whitespace=remove_whitespace, - ) - return _output_to_yaml( - **output, - name=name, - stream=stream, - header_cmd=header_cmd, - allow_empty=allow_empty, - ) - - -def _yaml_to_string( - data: dict[str, Any], - yaml: Any = None, - add_final_eol: bool = False, - header_cmd: str | None = None, -) -> str: - import io - - if yaml is None: - yaml = YAML() - yaml.indent(mapping=2, sequence=4, offset=2) - - buf = io.BytesIO() - yaml.dump(data, buf) - - val = buf.getvalue() - - if not add_final_eol: - val = val[:-1] - return _add_header(val.decode("utf-8"), header_cmd) - - -def _output_to_yaml( - dependencies: list[str] | None, - channels: list[str] | None = None, - pip: list[str] | None = None, - name: OptStr = None, - stream: str | Path | TextIO | None = None, - header_cmd: str | None = None, - allow_empty: bool = False, -) -> str: - data: dict[str, Any] = {} - if name: - data["name"] = name - - if channels: - data["channels"] = channels - - data["dependencies"] = [] - if dependencies: - data["dependencies"].extend(dependencies) - if pip: - data["dependencies"].append("pip") - data["dependencies"].append({"pip": pip}) - - if not data["dependencies"]: - return _check_allow_empty(allow_empty) - else: - # return data - s = _yaml_to_string(data, add_final_eol=True, header_cmd=header_cmd) - _optional_write(s, stream) - return s - - -class PyProject2Conda: - """Wrapper class to transform pyproject.toml -> environment.yaml""" - - def __init__( - self, - data: tomlkit.toml_document.TOMLDocument, - name: OptStr = None, - channels: OptStrSeq = None, - python_include: OptStr = None, - ) -> None: - self.data = data - self.name = name - self.channels = channels - self.python_include = python_include - - def to_conda_yaml( - self, - extras: OptStrSeq = None, - name: OptStr = None, - channels: OptStrSeq = None, - python_include: OptStr = None, - stream: str | Path | TextIO | None = None, - python_version: OptStr = None, - include_base_dependencies: bool = True, - header_cmd: str | None = None, - sort: bool = True, - deps: Sequence[str] | None = None, - reqs: Sequence[str] | None = None, - allow_empty: bool = False, - remove_whitespace: bool = True, - ) -> str: - self._check_extras(extras) - - return pyproject_to_conda( - data=self.data, - extras=extras, - name=name or self.name, - channels=channels or self.channels, - python_include=python_include or self.python_include, - stream=stream, - python_version=python_version, - include_base_dependencies=include_base_dependencies, - header_cmd=header_cmd, - sort=sort, - deps=deps, - reqs=reqs, - allow_empty=allow_empty, - remove_whitespace=remove_whitespace, - ) - - def to_conda_lists( - self, - extras: OptStrSeq = None, - channels: OptStrSeq = None, - python_include: OptStr = None, - python_version: OptStr = None, - include_base_dependencies: bool = True, - sort: bool = True, - deps: Sequence[str] | None = None, - reqs: Sequence[str] | None = None, - remove_whitespace: bool = True, - ) -> dict[str, Any]: - self._check_extras(extras) - - return pyproject_to_conda_lists( - data=self.data, - extras=extras, - channels=channels or self.channels, - python_include=python_include or self.python_include, - python_version=python_version, - include_base_dependencies=include_base_dependencies, - sort=sort, - deps=deps, - reqs=reqs, - remove_whitespace=remove_whitespace, - ) - - def to_requirement_list( - self, - extras: OptStrSeq = None, - include_base_dependencies: bool = True, - sort: bool = True, - reqs: Sequence[str] | None = None, - remove_whitespace: bool = True, - ) -> list[str]: - self._check_extras(extras) - - values = _pyproject_to_value_comment_pairs( - data=self.data, - extras=extras, - include_base_dependencies=include_base_dependencies, - ) - out = [x for x, _ in values if x is not None] - - if reqs: - out.extend(list(reqs)) - - # cleanup reqs: - out = _clean_pip_reqs(out) - - if remove_whitespace: - out = _remove_whitespace_list(out) - - if sort: - return sorted(out) - else: - return out - - def to_requirements( - self, - extras: OptStrSeq = None, - include_base_dependencies: bool = True, - header_cmd: str | None = None, - stream: str | Path | TextIO | None = None, - sort: bool = True, - reqs: Sequence[str] | None = None, - allow_empty: bool = False, - remove_whitespace: bool = True, - ) -> str: - """Create requirements.txt like file with pip dependencies.""" - - self._check_extras(extras) - - reqs = self.to_requirement_list( - extras=extras, - include_base_dependencies=include_base_dependencies, - sort=sort, - reqs=reqs, - remove_whitespace=remove_whitespace, - ) - - if not reqs: - return _check_allow_empty(allow_empty) - else: - s = _add_header(list_to_str(reqs), header_cmd) - _optional_write(s, stream) - return s - - def to_conda_requirements( - self, - extras: OptStrSeq = None, - channels: OptStrSeq = None, - python_include: OptStr = None, - python_version: OptStr = None, - prepend_channel: bool = False, - stream_conda: str | Path | TextIO | None = None, - stream_pip: str | Path | TextIO | None = None, - include_base_dependencies: bool = True, - header_cmd: OptStr = None, - sort: bool = True, - deps: Sequence[str] | None = None, - reqs: Sequence[str] | None = None, - remove_whitespace: bool = True, - ) -> tuple[str, str]: - output = self.to_conda_lists( - extras=extras, - channels=channels, - python_include=python_include, - python_version=python_version, - include_base_dependencies=include_base_dependencies, - sort=sort, - deps=deps, - reqs=reqs, - remove_whitespace=remove_whitespace, - ) - - deps = output.get("dependencies", None) - reqs = output.get("pip", None) - - channels = output.get("channels", None) - if channels and prepend_channel: - assert len(channels) == 1 - channel = channels[0] - # add in channel if none exists - if deps: - deps = [dep if "::" in dep else f"{channel}::{dep}" for dep in deps] - - deps_str = _add_header(list_to_str(deps), header_cmd) - reqs_str = _add_header(list_to_str(reqs), header_cmd) - - if stream_conda and deps_str: - _optional_write(deps_str, stream_conda) - - if stream_pip and reqs_str: - _optional_write(reqs_str, stream_pip) - - return deps_str, reqs_str - - def _check_extras(self, extras: OptStrSeq) -> None: - if extras is None: - return - elif isinstance(extras, str): - sent: Sequence[str] = [extras] - else: - sent = extras - - available = self.list_extras() - - for s in sent: - if s not in available: - raise ValueError(f"{s} not in {available}") - - def _get_opts(self, *keys: str) -> list[str]: - opts = get_in(keys, self.data, None) - if opts: - return list(opts.keys()) - else: - return [] - - def list_extras(self) -> list[str]: - return self._get_opts("project", "optional-dependencies") - - @classmethod - def from_string( - cls, - toml_string: str, - name: OptStr = None, - channels: OptStrSeq = None, - python_include: OptStr = None, - ) -> Self: - data = tomlkit.parse(toml_string) - return cls( - data=data, name=name, channels=channels, python_include=python_include - ) - - @classmethod - def from_path( - cls, - path: str | Path, - name: OptStr = None, - channels: OptStrSeq = None, - python_include: OptStr = None, - ) -> Self: - path = Path(path) - - if not path.exists(): - raise ValueError(f"{path} does not exist") - - with open(path, "rb") as f: - data = tomlkit.load(f) - return cls( - data=data, name=name, channels=channels, python_include=python_include - ) diff --git a/src/pyproject2conda/requirements.py b/src/pyproject2conda/requirements.py index 1ad3ca9..25d61dc 100644 --- a/src/pyproject2conda/requirements.py +++ b/src/pyproject2conda/requirements.py @@ -5,12 +5,9 @@ from __future__ import annotations -import argparse -import re -import shlex -from functools import cached_property, lru_cache +from functools import cached_property from pathlib import Path -from typing import TYPE_CHECKING, TypedDict, cast +from typing import TYPE_CHECKING, cast if TYPE_CHECKING: from typing import ( @@ -29,7 +26,6 @@ from ._typing import ( MISSING_TYPE, OptStr, - OptStrSeq, RequirementCommentPair, RequirementOverridePair, ) @@ -52,6 +48,8 @@ remove_whitespace_list as _remove_whitespace_list, ) +from .overrides import OverrideDeps, OverrideDict + # * Utilities -------------------------------------------------------------------------- def _check_allow_empty(allow_empty: bool) -> str: @@ -62,7 +60,6 @@ def _check_allow_empty(allow_empty: bool) -> str: raise ValueError(msg) -# ** Requirement def _clean_pip_reqs(reqs: list[str]) -> list[str]: return [str(Requirement(r)) for r in reqs] @@ -111,7 +108,7 @@ def _update_requirement( extras: str | Iterable[str] | None | MISSING_TYPE = MISSING, specifier: str | SpecifierSet | None | MISSING_TYPE = MISSING, marker: str | Marker | None | MISSING_TYPE = MISSING, -) -> Requirement: +) -> Requirement: # pragma: no cover if isinstance(requirement, str): requirement = Requirement(requirement) else: @@ -174,7 +171,7 @@ def _iter_value_comment_pairs( def _requirement_comment_pairs( array: tomlkit.items.Array, ) -> list[RequirementCommentPair]: - out = [] + out: list[RequirementCommentPair] = [] for value, comment in _iter_value_comment_pairs(array): if value is None: r = None @@ -192,7 +189,7 @@ def _resolve_extras( if isinstance(extras, str): extras = [extras] - out = [] + out: list[RequirementCommentPair] = [] for extra in extras: for requirement, comment in mapping_requirement_comment_pairs[extra]: if requirement is not None and requirement.name == package_name: @@ -221,140 +218,7 @@ def _factory_empty_tomlkit_Table() -> tomlkit.items.Table: ) -# * Parsing p2c commets ---------------------------------------------------------------- -@lru_cache -def p2c_argparser() -> argparse.ArgumentParser: - parser = argparse.ArgumentParser( - description="Parser searches for comments '# p2c: [OPTIONS] CONDA-PACKAGES" - ) - - parser.add_argument( - "-c", - "--channel", - type=str, - help="Channel to add to the pyproject requirement", - ) - parser.add_argument( - "-p", - "--pip", - action="store_true", - help="If specified, install pyproject dependency with pip", - ) - parser.add_argument( - "-s", - "--skip", - action="store_true", - help="If specified skip pyproject dependency on this line", - ) - - parser.add_argument("packages", nargs="*") - - return parser - - -class OverrideDict(TypedDict, total=False): - """Dict for storing override options.""" - - pip: bool - skip: bool - channel: str | None - packages: str | list[str] - - -def _match_p2c_comment(comment: OptStr) -> OptStr: - if not comment or not (match := re.match(r".*?#\s*p2c:\s*([^\#]*)", comment)): - return None - elif re.match(r".*?##\s*p2c:", comment): - # This checks for double ##. If found, ignore line - return None - else: - return match.group(1).strip() - - -def _parse_p2c(match: OptStr) -> OverrideDict | None: - """Parse match from _match_p2c_comment""" - - if match: - return cast(OverrideDict, vars(p2c_argparser().parse_args(shlex.split(match)))) - else: - return None - - -def parse_p2c_comment(comment: OptStr) -> OverrideDict | None: - if match := _match_p2c_comment(comment): - return _parse_p2c(match) - else: - return None - - -class OverrideDeps: - """Class to work with overrides from comment or table""" - - def __init__( - self, - pip: bool = False, - skip: bool = False, - packages: str | list[str] | None = None, - channel: str | None = None, - ): - if channel is not None and channel.strip() in ["pip", "pypi"]: - channel = None - pip = True - - self.pip = pip - self.skip = skip - self.channel = channel - - if packages is None: - packages = [] - elif isinstance(packages, str): - packages = [packages] - self.packages = packages - - def __repr__(self) -> str: - return repr(self.__dict__) - - @classmethod - def from_comment( - cls, comment: str | None, default: OverrideDict | None = None - ) -> Self | None: - parsed = parse_p2c_comment(comment) - - kws: OverrideDict - if parsed is None: - if default is None: - return None - else: - kws = default - else: - if default: - kws = dict(default, **parsed) # type: ignore - else: - kws = parsed - - return cls(**kws) - - @classmethod - def requirement_comment_to_override_pairs( - cls, - requirement_comment_pairs: list[RequirementCommentPair], - override_table: dict[str, OverrideDict], - ) -> list[RequirementOverridePair]: - out = [] - for requirement, comment in requirement_comment_pairs: - if requirement is not None: - default = override_table.get(requirement.name, None) - else: - default = None - - if ( - override := cls.from_comment(comment=comment, default=default) - ) is not None or requirement is not None: - out.append((requirement, override)) - return out - - -# * Parser ----------------------------------------------------------------------------- +# ** output ---------------------------------------------------------------------------- def _conda_yaml( name: str | None = None, channels: str | Iterable[str] | None = None, @@ -371,7 +235,7 @@ def _as_list(x: str | Iterable[str]) -> Iterable[str]: if not conda_deps and not pip_deps: raise ValueError - out = [] + out: list[str] = [] if name is not None: out.append(f"name: {name}") @@ -387,7 +251,7 @@ def _as_list(x: str | Iterable[str]) -> Iterable[str]: for dep in _as_list(conda_deps): out.append(f" - {dep}") - if pip_deps is not None: + if pip_deps: out.append(" - pip") out.append(" - pip:") @@ -427,7 +291,7 @@ def _create_header(cmd: str | None = None) -> str: ) # prepend '# ' - lines = [] + lines: list[str] = [] for line in header.split("\n"): if len(line.strip()) == 0: lines.append("#") @@ -457,6 +321,7 @@ def _optional_write( stream.write(string) +# * Main class class ParseDepends: """ Parse pyproject.toml file for dependencies @@ -509,7 +374,7 @@ def override_table(self) -> dict[str, OverrideDict]: out = {} else: out = out.unwrap() - return cast(dict[str, OverrideDict], out) + return cast("dict[str, OverrideDict]", out) @cached_property def channels(self) -> list[str]: @@ -527,7 +392,7 @@ def channels(self) -> list[str]: @property def extras(self) -> list[str]: - return list(self.optional_dependencies.keys()) + return list(self.optional_dependencies.keys()) # pyright: ignore @cached_property def _requirement_override_pairs_base( @@ -549,8 +414,8 @@ def _requirement_override_pairs_extras( "package_name[extra,..]" to the actual dependencies. """ unresolved: dict[str, list[RequirementCommentPair]] = { - k: _requirement_comment_pairs(v) - for k, v in self.optional_dependencies.items() + k: _requirement_comment_pairs(v) # pyright: ignore + for k, v in self.optional_dependencies.items() # pyright: ignore } resolved = { @@ -559,7 +424,7 @@ def _requirement_override_pairs_extras( package_name=self.package_name, mapping_requirement_comment_pairs=unresolved, ) - for k, v in unresolved.items() + for k in unresolved } # comments -> overrides @@ -689,19 +554,17 @@ def _init_deps(deps: str | Iterable[str] | None) -> list[str]: assert requirement is not None pip_deps.append(str(requirement)) elif not override.skip: - if requirement is None: - print(requirement, override) assert requirement is not None - conda_deps.append( - str( - _clean_conda_requirement( - requirement, - python_version=python_version, - channel=override.channel, - ) - ) + + r = _clean_conda_requirement( + requirement, + python_version=python_version, + channel=override.channel, ) + if r is not None: + conda_deps.append(str(r)) + conda_deps.extend( _clean_conda_strings( override.packages, python_version=python_version @@ -734,11 +597,11 @@ def _init_deps(deps: str | Iterable[str] | None) -> list[str]: def to_conda_yaml( self, - extras: OptStrSeq = None, + extras: OptStr | Iterable[str] = None, pip_deps: str | Iterable[str] | None = None, conda_deps: str | Iterable[str] | None = None, name: OptStr = None, - channels: OptStrSeq = None, + channels: OptStr | Iterable[str] = None, python_include: OptStr = None, python_version: OptStr = None, include_base: bool = True, @@ -781,7 +644,7 @@ def to_conda_yaml( def to_requirements( self, - extras: OptStrSeq = None, + extras: OptStr | Iterable[str] = None, include_base: bool = True, header_cmd: str | None = None, stream: str | Path | TextIO | None = None, @@ -836,7 +699,7 @@ def to_conda_requirements( ) if channels: - if isinstance(channels, str): + if isinstance(channels, str): # pragma: no cover channels = [channels] else: channels = list(channels) diff --git a/src/pyproject2conda/utils.py b/src/pyproject2conda/utils.py index 60ee6eb..f2ccf3a 100644 --- a/src/pyproject2conda/utils.py +++ b/src/pyproject2conda/utils.py @@ -112,7 +112,7 @@ def update_target( update = any(target_time < dep.stat().st_mtime for dep in deps_filtered) else: - raise ValueError(f"unknown option overwrite={overwrite}") + raise ValueError(f"unknown option overwrite={overwrite}") # pragma: no cover return update diff --git a/tests/test_cli.py b/tests/test_cli.py index 2bb9dde..ba5dd9c 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -70,7 +70,7 @@ def test_create(filename): result = do_run(runner, "yaml", filename="hello/there.toml") - assert isinstance(result.exception, ValueError) + assert isinstance(result.exception, FileNotFoundError) expected = """\ channels: diff --git a/tests/test_parser.py b/tests/test_parser.py index cee6375..3f4f185 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -2,7 +2,9 @@ from __future__ import annotations import tempfile import pytest -from pyproject2conda import parser + +# from pyproject2conda import parser +from pyproject2conda import requirements from pyproject2conda.utils import get_in from textwrap import dedent @@ -25,7 +27,7 @@ def test_list_to_string(): def test_match_p2c_comment(): - from pyproject2conda.requirements import _match_p2c_comment + from pyproject2conda.overrides import _match_p2c_comment expected = "-c -d" for comment in [ @@ -63,7 +65,7 @@ def get_expected(pip=False, skip=False, channel=None, packages=None): "packages": packages, } - from pyproject2conda.requirements import _parse_p2c + from pyproject2conda.overrides import _parse_p2c assert _parse_p2c(None) is None @@ -90,17 +92,17 @@ def get_expected(pip=False, skip=False, channel=None, packages=None): ) -def test_value_comment_pairs(): - d: list[tuple[str | None, str | None]] = [(None, "# p2c: --pip")] +# def test_value_comment_pairs(): +# d: list[tuple[str | None, str | None]] = [(None, "# p2c: --pip")] - with pytest.raises(ValueError): - out = parser.value_comment_pairs_to_conda(d) +# with pytest.raises(ValueError): +# out = parser.value_comment_pairs_to_conda(d) - d = [("hello", "# p2c: there")] +# d = [("hello", "# p2c: there")] - out = parser.value_comment_pairs_to_conda(d) +# out = parser.value_comment_pairs_to_conda(d) - assert out["dependencies"] == ["hello", "there"] +# assert out["dependencies"] == ["hello", "there"] def test_header(): @@ -136,27 +138,27 @@ def test_header(): assert out == header -def test_yaml_to_str(): - d = {"dep": ["a", "b"]} +# def test_yaml_to_str(): +# d = {"dep": ["a", "b"]} - s = parser._yaml_to_string(d, add_final_eol=False) +# s = parser._yaml_to_string(d, add_final_eol=False) - expected = """\ - dep: - - a - - b""" +# expected = """\ +# dep: +# - a +# - b""" - assert s == dedent(expected) +# assert s == dedent(expected) - s = parser._yaml_to_string(d, add_final_eol=True) +# s = parser._yaml_to_string(d, add_final_eol=True) - expected = """\ - dep: - - a - - b - """ +# expected = """\ +# dep: +# - a +# - b +# """ - assert s == dedent(expected) +# assert s == dedent(expected) def test_optional_write(): @@ -178,10 +180,15 @@ def test_optional_write(): def test_output_to_yaml(): - s = parser._output_to_yaml( - dependencies=["a"], + from pyproject2conda.requirements import _conda_yaml + + with pytest.raises(ValueError): + s = _conda_yaml() + + s = _conda_yaml( + conda_deps=["a"], channels=["conda-forge"], - pip=["pip-thing"], + pip_deps=["pip-thing"], name="hello", ) @@ -230,15 +237,69 @@ def test_infer(): [tool.pyproject2conda] - channels = ['conda-forge'] + channels = 'conda-forge' """ ) - d = parser.PyProject2Conda.from_string(toml) + d = requirements.ParseDepends.from_string(toml) with pytest.raises(ValueError): d.to_conda_yaml(python_include="infer") +def test_channels() -> None: + toml = dedent( + """\ + [project] + name = "hello" + """ + ) + + d = requirements.ParseDepends.from_string(toml) + + assert d.channels == [] + + toml = dedent( + """\ + [project] + name = "hello" + + [tool.pyproject2conda] + channels = 'conda-forge' + """ + ) + + d = requirements.ParseDepends.from_string(toml) + + assert d.channels == ["conda-forge"] + + +def test_pip_requirements() -> None: + toml = dedent( + """\ + [project] + requires-python = ">=3.8,<3.11" + dependencies = [ + "athing", # p2c: -p # a comment + "bthing", # p2c: -s bthing-conda + "cthing; python_version<'3.10'", # p2c: -c conda-forge + ] + """ + ) + + expected = dedent( + """\ + athing + bthing + cthing;python_version<"3.10" + hello + """ + ) + + d = requirements.ParseDepends.from_string(toml) + + assert d.to_requirements(pip_deps="hello") == expected + + def test_package_name(): toml = dedent( """\ @@ -273,9 +334,9 @@ def test_package_name(): channels = ['conda-forge'] """ ) - d = parser.PyProject2Conda.from_string(toml) + d = requirements.ParseDepends.from_string(toml) with pytest.raises(ValueError): - d.to_conda_lists() + d.conda_pip_requirements("dev") @pytest.mark.parametrize( @@ -316,7 +377,7 @@ def test_package_name(): channels = ['conda-forge'] """ ), - # table syntax + # Table syntax dedent( """\ [project] @@ -361,16 +422,14 @@ def test_package_name(): ], ) def test_complete(toml): - d = parser.PyProject2Conda.from_string(toml) + d = requirements.ParseDepends.from_string(toml) # test unknown extra with pytest.raises(ValueError): d.to_conda_yaml(extras="a-thing") # test list: - assert d.list_extras() == ["test", "dev-extras", "dev", "dist-pypi"] - - assert d._get_opts("hello", "there") == [] + assert d.extras == ["test", "dev-extras", "dev", "dist-pypi"] expected = """\ channels: @@ -492,7 +551,7 @@ def test_complete(toml): assert dedent(expected) == out - out = d.to_conda_yaml(extras="dist-pypi", include_base_dependencies=False) + out = d.to_conda_yaml(extras="dist-pypi", include_base=False) expected = """\ channels: @@ -555,7 +614,7 @@ def test_complete(toml): """ assert dedent(expected) == d.to_conda_yaml( - deps=["dep;python_version<'3.10'"], reqs=["req"] + conda_deps="dep;python_version<'3.10'", pip_deps=["req"] ) expected = """\ @@ -570,8 +629,8 @@ def test_complete(toml): - req;python_version<"3.10" """ assert dedent(expected) == d.to_conda_yaml( - deps=["dep;python_version<'3.10'"], - reqs=["req;python_version<'3.10'"], + conda_deps=["dep;python_version<'3.10'"], + pip_deps=["req;python_version<'3.10'"], python_include="python=3.10", python_version="3.10", ) @@ -605,7 +664,7 @@ def test_missing_dependencies(): """ ) - d = parser.PyProject2Conda.from_string(toml) + d = requirements.ParseDepends.from_string(toml) expected = """\ channels: @@ -620,7 +679,7 @@ def test_missing_dependencies(): assert dedent(expected) == d.to_conda_yaml( extras="test", - include_base_dependencies=False, + include_base=False, ) toml = dedent( @@ -649,16 +708,16 @@ def test_missing_dependencies(): """ ) - d = parser.PyProject2Conda.from_string(toml) + d = requirements.ParseDepends.from_string(toml) assert dedent(expected) == d.to_conda_yaml( extras="test", - include_base_dependencies=False, + include_base=False, ) assert dedent(expected) == d.to_conda_yaml( extras="test", - include_base_dependencies=True, + include_base=True, ) # check output has no dependencies: @@ -670,6 +729,85 @@ def test_missing_dependencies(): assert f(allow_empty=True) == "No dependencies for this environment\n" +def test_clean_conda() -> None: + toml = dedent( + """\ + [project] + name = "hello" + requires-python = ">=3.8,<3.11" + + [project.optional-dependencies] + test = [ + "pandas", + "pytest", # p2c: -c conda-forge + "pytest-accept", # p2c: -p + ] + dev-extras = [ + # p2c: -s additional-thing # this is an additional conda package + "matplotlib", # p2c: -s conda-forge::conda-matplotlib + ] + dev = [ + "hello[test]", + "hello[dev-extras]", + ] + + [tool.pyproject2conda] + channels = ['conda-forge'] + """ + ) + + expected = """\ +channels: + - conda-forge +dependencies: + - additional-thing + - conda-forge::conda-matplotlib + """ + + d = requirements.ParseDepends.from_string(toml) + + assert dedent(expected) == d.to_conda_yaml( + extras="dev-extras", + include_base=False, + ) + + +def test_no_optional_deps() -> None: + toml = dedent( + """\ + [project] + name = "hello" + requires-python = ">=3.8,<3.11" + dependencies = [ + "athing >0.5", # p2c: -p # a comment + "bthing > 1.0", # p2c: -s 'bthing-conda > 2.0' + "cthing; python_version < '3.10'", # p2c: -c conda-forge + ] + + [tool.pyproject2conda] + channels = ['conda-forge'] + """ + ) + + expected = """\ +channels: + - conda-forge +dependencies: + - python>=3.8,<3.11 + - bthing-conda>2.0 + - conda-forge::cthing + - pip + - pip: + - athing>0.5 + """ + + d = requirements.ParseDepends.from_string(toml) + + assert d.optional_dependencies == {} + + assert dedent(expected) == d.to_conda_yaml(python_include="infer") + + @pytest.mark.parametrize( "toml", [ @@ -695,7 +833,7 @@ def test_missing_dependencies(): name = "hello" requires-python = ">=3.8, <3.11" dependencies = [ - "athing >0.5", + "athing >0.5", # p2c: -p "bthing > 1.0", "cthing; python_version < '3.10'", ] @@ -705,7 +843,8 @@ def test_missing_dependencies(): channels = ['conda-forge'] [tool.pyproject2conda.dependencies] - athing = {channel = "pip"} + # this will be ignored, since have option above + athing = {channel = "conda-forge"} bthing = {skip = true, packages = "bthing-conda > 2.0"} cthing = {channel = "conda-forge"} """ @@ -713,7 +852,7 @@ def test_missing_dependencies(): ], ) def test_spaces(toml): - d = parser.PyProject2Conda.from_string(toml) + d = requirements.ParseDepends.from_string(toml) expected = """\ channels: From 6c7ca0e027948668403cb3741050018163ac9f70 Mon Sep 17 00:00:00 2001 From: wpk Date: Fri, 17 Nov 2023 10:21:20 -0500 Subject: [PATCH 29/36] fix: fixed bug in parser --- src/pyproject2conda/requirements.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/src/pyproject2conda/requirements.py b/src/pyproject2conda/requirements.py index 25d61dc..4112af2 100644 --- a/src/pyproject2conda/requirements.py +++ b/src/pyproject2conda/requirements.py @@ -572,13 +572,10 @@ def _init_deps(deps: str | Iterable[str] | None) -> list[str]: ) elif requirement: - conda_deps.append( - str( - _clean_conda_requirement( - requirement, python_version=python_version - ) - ) - ) + r = _clean_conda_requirement(requirement, python_version=python_version) + + if r is not None: + conda_deps.append(str(r)) pip_deps, conda_deps = ( self._cleanup( From 7c14e79326d58657d62e4451399b1155624d6131 Mon Sep 17 00:00:00 2001 From: wpk Date: Fri, 17 Nov 2023 10:22:14 -0500 Subject: [PATCH 30/36] chore: update requirements to reflect new pyproject2conda --- docs/reference/index.md | 3 ++- pyproject.toml | 16 ++++++++-------- requirements/cog.txt | 3 +-- requirements/dev-base.txt | 5 ++--- requirements/dev-complete.txt | 4 ++-- requirements/dev.txt | 4 ++-- requirements/docs.txt | 3 +-- requirements/py310-dev-base.yaml | 1 - requirements/py310-docs.yaml | 3 +-- requirements/py310-test.yaml | 1 - requirements/py310-typing.yaml | 1 - requirements/py311-test.yaml | 1 - requirements/py311-typing.yaml | 1 - requirements/py312-test.yaml | 1 - requirements/py312-typing.yaml | 1 - requirements/py38-test.yaml | 1 - requirements/py38-typing.yaml | 1 - requirements/py39-test.yaml | 1 - requirements/py39-typing.yaml | 1 - requirements/test.txt | 3 +-- requirements/typing.txt | 5 ++--- 21 files changed, 22 insertions(+), 38 deletions(-) diff --git a/docs/reference/index.md b/docs/reference/index.md index bcc148b..90c7b21 100644 --- a/docs/reference/index.md +++ b/docs/reference/index.md @@ -19,7 +19,8 @@ Modules :toctree: generated/ :template: autodocsumm/module.rst - parser + overrides + requirements config cli diff --git a/pyproject.toml b/pyproject.toml index 1dce79b..26d2f02 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,7 +28,6 @@ dynamic = ["readme", "version"] requires-python = ">=3.8" dependencies = [ "tomli", - "ruamel.yaml", "tomlkit", "typer", "packaging", @@ -52,7 +51,7 @@ test = [ ] dev-extras = [ "setuptools-scm>=8.0", # - "pytest-accept", # p2c: -p + "pytest-accept", # "nbval", "ipython", "ipykernel", @@ -66,11 +65,7 @@ typing = [ "pyproject2conda[typing-extras]", # "pyproject2conda[test]", ] -nox = [ - "nox", - "noxopt", # p2c: -p - "ruamel.yaml", -] +nox = ["nox", "noxopt", "ruamel.yaml"] dev = [ "pyproject2conda[test]", # "pyproject2conda[typing-extras]", @@ -79,7 +74,7 @@ dev = [ tools = [ "pre-commit", # # "cruft", - "scriv", # p2c: -p + "scriv", "nbqa", "pyright", ] @@ -112,6 +107,11 @@ dist-conda = [ ] cog = ["cogapp"] +[tool.pyproject2conda.dependencies] +pytest-accept = { pip = true } +noxopt = { pip = true } +scriv = { pip = true } + [tool.pyproject2conda] user_config = "config/userconfig.toml" template_python = "requirements/py{py}-{env}" diff --git a/requirements/cog.txt b/requirements/cog.txt index afdd21d..d704dc7 100644 --- a/requirements/cog.txt +++ b/requirements/cog.txt @@ -9,8 +9,7 @@ # cogapp packaging -ruamel.yaml tomli tomlkit typer -typing-extensions;python_version<'3.9' +typing-extensions;python_version<"3.9" diff --git a/requirements/dev-base.txt b/requirements/dev-base.txt index c76fb3b..e6cd806 100644 --- a/requirements/dev-base.txt +++ b/requirements/dev-base.txt @@ -16,11 +16,10 @@ pytest-accept pytest-cov pytest-sugar pytest-xdist -pytype;python_version<'3.11' -ruamel.yaml +pytype;python_version<"3.11" setuptools-scm>=8.0 tomli tomlkit typer types-click -typing-extensions;python_version<'3.9' +typing-extensions;python_version<"3.9" diff --git a/requirements/dev-complete.txt b/requirements/dev-complete.txt index 99322cf..da3d63c 100644 --- a/requirements/dev-complete.txt +++ b/requirements/dev-complete.txt @@ -21,7 +21,7 @@ pytest-accept pytest-cov pytest-sugar pytest-xdist -pytype;python_version<'3.11' +pytype;python_version<"3.11" ruamel.yaml scriv setuptools-scm>=8.0 @@ -29,4 +29,4 @@ tomli tomlkit typer types-click -typing-extensions;python_version<'3.9' +typing-extensions;python_version<"3.9" diff --git a/requirements/dev.txt b/requirements/dev.txt index 98070d8..72774af 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -18,11 +18,11 @@ pytest-accept pytest-cov pytest-sugar pytest-xdist -pytype;python_version<'3.11' +pytype;python_version<"3.11" ruamel.yaml setuptools-scm>=8.0 tomli tomlkit typer types-click -typing-extensions;python_version<'3.9' +typing-extensions;python_version<"3.9" diff --git a/requirements/docs.txt b/requirements/docs.txt index e99473a..e3147e5 100644 --- a/requirements/docs.txt +++ b/requirements/docs.txt @@ -13,7 +13,6 @@ ipython myst-nb packaging pyenchant -ruamel.yaml sphinx-autobuild sphinx-book-theme sphinx-click @@ -23,4 +22,4 @@ sphinxcontrib-spelling tomli tomlkit typer -typing-extensions;python_version<'3.9' +typing-extensions;python_version<"3.9" diff --git a/requirements/py310-dev-base.yaml b/requirements/py310-dev-base.yaml index fd92c2a..5b78c3a 100644 --- a/requirements/py310-dev-base.yaml +++ b/requirements/py310-dev-base.yaml @@ -20,7 +20,6 @@ dependencies: - pytest-sugar - pytest-xdist - pytype - - ruamel.yaml - setuptools-scm>=8.0 - tomli - tomlkit diff --git a/requirements/py310-docs.yaml b/requirements/py310-docs.yaml index bda7cfb..591f829 100644 --- a/requirements/py310-docs.yaml +++ b/requirements/py310-docs.yaml @@ -17,12 +17,11 @@ dependencies: - myst-nb - packaging - pyenchant - - ruamel.yaml - - sphinx>=5.3.0 - sphinx-autobuild - sphinx-book-theme - sphinx-click - sphinx-copybutton + - sphinx>=5.3.0 - sphinxcontrib-spelling - tomli - tomlkit diff --git a/requirements/py310-test.yaml b/requirements/py310-test.yaml index a1ba3d7..6b46141 100644 --- a/requirements/py310-test.yaml +++ b/requirements/py310-test.yaml @@ -16,7 +16,6 @@ dependencies: - pytest-cov - pytest-sugar - pytest-xdist - - ruamel.yaml - tomli - tomlkit - typer diff --git a/requirements/py310-typing.yaml b/requirements/py310-typing.yaml index b286b19..b1c3c1c 100644 --- a/requirements/py310-typing.yaml +++ b/requirements/py310-typing.yaml @@ -18,7 +18,6 @@ dependencies: - pytest-sugar - pytest-xdist - pytype - - ruamel.yaml - tomli - tomlkit - typer diff --git a/requirements/py311-test.yaml b/requirements/py311-test.yaml index b35c79c..64d28d4 100644 --- a/requirements/py311-test.yaml +++ b/requirements/py311-test.yaml @@ -16,7 +16,6 @@ dependencies: - pytest-cov - pytest-sugar - pytest-xdist - - ruamel.yaml - tomli - tomlkit - typer diff --git a/requirements/py311-typing.yaml b/requirements/py311-typing.yaml index 6d6d0c1..170ae64 100644 --- a/requirements/py311-typing.yaml +++ b/requirements/py311-typing.yaml @@ -17,7 +17,6 @@ dependencies: - pytest-cov - pytest-sugar - pytest-xdist - - ruamel.yaml - tomli - tomlkit - typer diff --git a/requirements/py312-test.yaml b/requirements/py312-test.yaml index ace8f0c..132b621 100644 --- a/requirements/py312-test.yaml +++ b/requirements/py312-test.yaml @@ -16,7 +16,6 @@ dependencies: - pytest-cov - pytest-sugar - pytest-xdist - - ruamel.yaml - tomli - tomlkit - typer diff --git a/requirements/py312-typing.yaml b/requirements/py312-typing.yaml index c6e3382..2a729d4 100644 --- a/requirements/py312-typing.yaml +++ b/requirements/py312-typing.yaml @@ -17,7 +17,6 @@ dependencies: - pytest-cov - pytest-sugar - pytest-xdist - - ruamel.yaml - tomli - tomlkit - typer diff --git a/requirements/py38-test.yaml b/requirements/py38-test.yaml index e684ad6..af0fbac 100644 --- a/requirements/py38-test.yaml +++ b/requirements/py38-test.yaml @@ -16,7 +16,6 @@ dependencies: - pytest-cov - pytest-sugar - pytest-xdist - - ruamel.yaml - tomli - tomlkit - typer diff --git a/requirements/py38-typing.yaml b/requirements/py38-typing.yaml index 21b7c3c..a7f5cc9 100644 --- a/requirements/py38-typing.yaml +++ b/requirements/py38-typing.yaml @@ -18,7 +18,6 @@ dependencies: - pytest-sugar - pytest-xdist - pytype - - ruamel.yaml - tomli - tomlkit - typer diff --git a/requirements/py39-test.yaml b/requirements/py39-test.yaml index c902163..6fca3b7 100644 --- a/requirements/py39-test.yaml +++ b/requirements/py39-test.yaml @@ -16,7 +16,6 @@ dependencies: - pytest-cov - pytest-sugar - pytest-xdist - - ruamel.yaml - tomli - tomlkit - typer diff --git a/requirements/py39-typing.yaml b/requirements/py39-typing.yaml index 03d436c..4a1c9c3 100644 --- a/requirements/py39-typing.yaml +++ b/requirements/py39-typing.yaml @@ -18,7 +18,6 @@ dependencies: - pytest-sugar - pytest-xdist - pytype - - ruamel.yaml - tomli - tomlkit - typer diff --git a/requirements/test.txt b/requirements/test.txt index 306399d..0acd4c5 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -12,8 +12,7 @@ pytest pytest-cov pytest-sugar pytest-xdist -ruamel.yaml tomli tomlkit typer -typing-extensions;python_version<'3.9' +typing-extensions;python_version<"3.9" diff --git a/requirements/typing.txt b/requirements/typing.txt index 47af58d..a5e2728 100644 --- a/requirements/typing.txt +++ b/requirements/typing.txt @@ -13,10 +13,9 @@ pytest pytest-cov pytest-sugar pytest-xdist -pytype;python_version<'3.11' -ruamel.yaml +pytype;python_version<"3.11" tomli tomlkit typer types-click -typing-extensions;python_version<'3.9' +typing-extensions;python_version<"3.9" From 61a5f815ec2e58c4ae53a27ea7edaf76477913e6 Mon Sep 17 00:00:00 2001 From: wpk Date: Fri, 17 Nov 2023 10:26:59 -0500 Subject: [PATCH 31/36] chore: add changelog snippet --- ...15_154156_william.krekelberg_add_pyproject_table_override.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/changelog.d/20231115_154156_william.krekelberg_add_pyproject_table_override.md b/changelog.d/20231115_154156_william.krekelberg_add_pyproject_table_override.md index 70d5232..7c07b5f 100644 --- a/changelog.d/20231115_154156_william.krekelberg_add_pyproject_table_override.md +++ b/changelog.d/20231115_154156_william.krekelberg_add_pyproject_table_override.md @@ -16,6 +16,8 @@ Uncomment the section that is right (remove the HTML comment wrapper). - Can now specify conda changes using `tool.pyproject2conda.dependencies` table. This is an alternative to using `# p2c:` comments. +- Refactored code. Split `parser` to `requirements` and `overrides`. Also + cleaned up the parsing logic to hopefully make future changes simpler. +## v0.10.0 — 2023-11-17 + +### Added + +- Can now specify conda changes using `tool.pyproject2conda.dependencies` table. + This is an alternative to using `# p2c:` comments. +- Refactored code. Split `parser` to `requirements` and `overrides`. Also + cleaned up the parsing logic to hopefully make future changes simpler. + ## v0.9.0 — 2023-11-14 ### Added diff --git a/changelog.d/20231115_154156_william.krekelberg_add_pyproject_table_override.md b/changelog.d/20231115_154156_william.krekelberg_add_pyproject_table_override.md deleted file mode 100644 index 7c07b5f..0000000 --- a/changelog.d/20231115_154156_william.krekelberg_add_pyproject_table_override.md +++ /dev/null @@ -1,45 +0,0 @@ - - - - - -### Added - -- Can now specify conda changes using `tool.pyproject2conda.dependencies` table. - This is an alternative to using `# p2c:` comments. -- Refactored code. Split `parser` to `requirements` and `overrides`. Also - cleaned up the parsing logic to hopefully make future changes simpler. - - - - - From 8f088793e5202c6f599a0dc506a0dcdf375aed60 Mon Sep 17 00:00:00 2001 From: wpk Date: Mon, 20 Nov 2023 11:25:52 -0500 Subject: [PATCH 33/36] feat: can now access build-system.requires data. This can be useful for creating an environment to build the package without needing `isolated-build` option. This can speed up subsequent builds --- src/pyproject2conda/requirements.py | 28 ++++- tests/test_cli.py | 1 + tests/test_parser.py | 163 ++++++++++++++++------------ 3 files changed, 120 insertions(+), 72 deletions(-) diff --git a/src/pyproject2conda/requirements.py b/src/pyproject2conda/requirements.py index 4112af2..b96d368 100644 --- a/src/pyproject2conda/requirements.py +++ b/src/pyproject2conda/requirements.py @@ -357,6 +357,16 @@ def dependencies(self) -> tomlkit.items.Array: ), ) + @cached_property + def build_system_requires(self) -> tomlkit.items.Array: + """build-system.requires""" + return cast( + "tomlkit.items.Array", + self.get_in( + "build-system", "requires", factory=_factory_empty_tomlkit_Array + ), + ) + @cached_property def optional_dependencies(self) -> tomlkit.items.Table: """project.optional-dependencies""" @@ -392,7 +402,9 @@ def channels(self) -> list[str]: @property def extras(self) -> list[str]: - return list(self.optional_dependencies.keys()) # pyright: ignore + return list(self.optional_dependencies.keys()) + [ # pyright: ignore + "build-system.requires" + ] @cached_property def _requirement_override_pairs_base( @@ -428,13 +440,25 @@ def _requirement_override_pairs_extras( } # comments -> overrides - return { + out = { k: OverrideDeps.requirement_comment_to_override_pairs( requirement_comment_pairs=v, override_table=self.override_table ) for k, v in resolved.items() } + # add in build-system.requires + out[ + "build-system.requires" + ] = OverrideDeps.requirement_comment_to_override_pairs( + requirement_comment_pairs=_requirement_comment_pairs( + self.build_system_requires + ), + override_table=self.override_table, + ) + + return out + @staticmethod def _cleanup( values: list[str], diff --git a/tests/test_cli.py b/tests/test_cli.py index ba5dd9c..a37357f 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -54,6 +54,7 @@ def test_list(filename): * dev-extras * dev * dist-pypi + * build-system.requires """ check_result(result, expected) diff --git a/tests/test_parser.py b/tests/test_parser.py index 3f4f185..1563a92 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -345,79 +345,87 @@ def test_package_name(): # comment syntax dedent( """\ - [project] - name = "hello" - requires-python = ">=3.8, <3.11" - dependencies = [ - "athing", # p2c: -p # a comment - "bthing", # p2c: -s bthing-conda - "cthing; python_version<'3.10'", # p2c: -c conda-forge - ] - - [project.optional-dependencies] - test = [ - "pandas", - "pytest", # p2c: -c conda-forge - ] - dev-extras = [ - # p2c: -s additional-thing # this is an additional conda package - "matplotlib", # p2c: -s conda-matplotlib - ] - dev = [ - "hello[test]", - "hello[dev-extras]", - ] - dist-pypi = [ - "setuptools", - "build", # p2c: -p - ] - - - [tool.pyproject2conda] - channels = ['conda-forge'] - """ + [build-system] + requires = ["setuptools>=61.2", "setuptools_scm[toml]>=8.0"] + build-backend = "setuptools.build_meta" + + [project] + name = "hello" + requires-python = ">=3.8, <3.11" + dependencies = [ + "athing", # p2c: -p # a comment + "bthing", # p2c: -s bthing-conda + "cthing; python_version<'3.10'", # p2c: -c conda-forge + ] + + [project.optional-dependencies] + test = [ + "pandas", + "pytest", # p2c: -c conda-forge + ] + dev-extras = [ + # p2c: -s additional-thing # this is an additional conda package + "matplotlib", # p2c: -s conda-matplotlib + ] + dev = [ + "hello[test]", + "hello[dev-extras]", + ] + dist-pypi = [ + "setuptools", + "build", # p2c: -p + ] + + + [tool.pyproject2conda] + channels = ['conda-forge'] + """ ), # Table syntax dedent( """\ - [project] - name = "hello" - requires-python = ">=3.8, <3.11" - dependencies = [ - "athing", - "bthing", - "cthing; python_version<'3.10'", - ] - - [project.optional-dependencies] - test = [ - "pandas", - "pytest", - ] - dev-extras = [ - "matplotlib", - ] - dev = [ - "hello[test]", - "hello[dev-extras]", - ] - dist-pypi = [ - "setuptools", - "build", - ] - - - [tool.pyproject2conda] - channels = ['conda-forge'] - - [tool.pyproject2conda.dependencies] - athing = {pip = true} - bthing = {skip = true, packages = "bthing-conda"} - cthing = {channel = "conda-forge"} - pytest = {channel = "conda-forge"} - matplotlib = {skip = true, packages = ["additional-thing", "conda-matplotlib"]} - build = { channel = "pip" } - """ + [build-system] + requires = ["setuptools>=61.2", "setuptools_scm[toml]>=8.0"] + build-backend = "setuptools.build_meta" + + [project] + name = "hello" + requires-python = ">=3.8, <3.11" + dependencies = [ + "athing", + "bthing", + "cthing; python_version<'3.10'", + ] + + [project.optional-dependencies] + test = [ + "pandas", + "pytest", + ] + dev-extras = [ + "matplotlib", + ] + dev = [ + "hello[test]", + "hello[dev-extras]", + ] + dist-pypi = [ + "setuptools", + "build", + ] + + + [tool.pyproject2conda] + channels = ['conda-forge'] + + [tool.pyproject2conda.dependencies] + athing = {pip = true} + bthing = {skip = true, packages = "bthing-conda"} + cthing = {channel = "conda-forge"} + pytest = {channel = "conda-forge"} + matplotlib = {skip = true, packages = ["additional-thing", "conda-matplotlib"]} + build = { channel = "pip" } + """ ), ], ) @@ -429,7 +437,22 @@ def test_complete(toml): d.to_conda_yaml(extras="a-thing") # test list: - assert d.extras == ["test", "dev-extras", "dev", "dist-pypi"] + assert d.extras == [ + "test", + "dev-extras", + "dev", + "dist-pypi", + "build-system.requires", + ] + + # test build-system.requires + expected = """\ + setuptools>=61.2 + setuptools_scm[toml]>=8.0 + """ + assert dedent(expected) == d.to_requirements( + extras="build-system.requires", include_base=False + ) expected = """\ channels: From 870039707ce5df70e9c0a5a649097d57ad341160 Mon Sep 17 00:00:00 2001 From: wpk Date: Mon, 20 Nov 2023 11:27:56 -0500 Subject: [PATCH 34/36] chore: add changelog snippet --- ...607_william.krekelberg_get_build_system.md | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 changelog.d/20231120_112607_william.krekelberg_get_build_system.md diff --git a/changelog.d/20231120_112607_william.krekelberg_get_build_system.md b/changelog.d/20231120_112607_william.krekelberg_get_build_system.md new file mode 100644 index 0000000..6588472 --- /dev/null +++ b/changelog.d/20231120_112607_william.krekelberg_get_build_system.md @@ -0,0 +1,43 @@ + + + + + +### Added + +- Can now access "build-system.requires" as an extra. This can be useful for + creating isolated environments to build a package. + + + + + From 492ff68db92134126a0d074b4349860ea1eb2753 Mon Sep 17 00:00:00 2001 From: wpk Date: Tue, 28 Nov 2023 13:52:43 -0500 Subject: [PATCH 35/36] feat: can now add pip as conda dependency --- ...8_134544_william.krekelberg_include_pip.md | 46 +++++++++++++++++++ pyproject.toml | 6 +++ requirements/py310-docs.yaml | 1 + requirements/py310-test.yaml | 1 + requirements/py311-test.yaml | 1 + requirements/py312-test.yaml | 1 + requirements/py38-test.yaml | 1 + requirements/py39-test.yaml | 1 + src/pyproject2conda/cli.py | 2 +- src/pyproject2conda/requirements.py | 29 +++++++----- tests/test_cli.py | 9 +++- tests/test_parser.py | 37 ++++++++++++++- 12 files changed, 119 insertions(+), 16 deletions(-) create mode 100644 changelog.d/20231128_134544_william.krekelberg_include_pip.md diff --git a/changelog.d/20231128_134544_william.krekelberg_include_pip.md b/changelog.d/20231128_134544_william.krekelberg_include_pip.md new file mode 100644 index 0000000..db6a6c7 --- /dev/null +++ b/changelog.d/20231128_134544_william.krekelberg_include_pip.md @@ -0,0 +1,46 @@ + + + + + + +### Changed + +- Can now specify `pip` as a conda dependency. This is needed for cases that + there are no pip dependencies in the environment, but you want it there for + installing local packages. This may be the case if using `conda-lock` on an + environment. Note that, much like python is always first in the dependency + list, pip is always last. + + + + diff --git a/pyproject.toml b/pyproject.toml index 26d2f02..919b8e0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -139,6 +139,12 @@ extras = ["dev"] [tool.pyproject2conda.envs.cog] style = "requirements" +# make sure pip is in all conda environments +# that we'll install package into +[[tool.pyproject2conda.overrides]] +envs = ["dev", "dev-complete", "dev-base", "test", "docs"] +deps = ["pip"] + [[tool.pyproject2conda.overrides]] envs = ["test-extras", "dist-conda", "dist-pypi"] base = false diff --git a/requirements/py310-docs.yaml b/requirements/py310-docs.yaml index 591f829..84c052e 100644 --- a/requirements/py310-docs.yaml +++ b/requirements/py310-docs.yaml @@ -26,3 +26,4 @@ dependencies: - tomli - tomlkit - typer + - pip diff --git a/requirements/py310-test.yaml b/requirements/py310-test.yaml index 6b46141..f3669b9 100644 --- a/requirements/py310-test.yaml +++ b/requirements/py310-test.yaml @@ -19,3 +19,4 @@ dependencies: - tomli - tomlkit - typer + - pip diff --git a/requirements/py311-test.yaml b/requirements/py311-test.yaml index 64d28d4..3994e3c 100644 --- a/requirements/py311-test.yaml +++ b/requirements/py311-test.yaml @@ -19,3 +19,4 @@ dependencies: - tomli - tomlkit - typer + - pip diff --git a/requirements/py312-test.yaml b/requirements/py312-test.yaml index 132b621..b95054b 100644 --- a/requirements/py312-test.yaml +++ b/requirements/py312-test.yaml @@ -19,3 +19,4 @@ dependencies: - tomli - tomlkit - typer + - pip diff --git a/requirements/py38-test.yaml b/requirements/py38-test.yaml index af0fbac..6948131 100644 --- a/requirements/py38-test.yaml +++ b/requirements/py38-test.yaml @@ -20,3 +20,4 @@ dependencies: - tomlkit - typer - typing-extensions + - pip diff --git a/requirements/py39-test.yaml b/requirements/py39-test.yaml index 6fca3b7..7395bf2 100644 --- a/requirements/py39-test.yaml +++ b/requirements/py39-test.yaml @@ -19,3 +19,4 @@ dependencies: - tomli - tomlkit - typer + - pip diff --git a/src/pyproject2conda/cli.py b/src/pyproject2conda/cli.py index c74bc97..9457c4b 100644 --- a/src/pyproject2conda/cli.py +++ b/src/pyproject2conda/cli.py @@ -739,7 +739,7 @@ def to_json( python=python, ) - conda_deps, pip_deps = d.conda_pip_requirements( + conda_deps, pip_deps = d.conda_and_pip_requirements( extras=extras, python_include=python_include, python_version=python_version, diff --git a/src/pyproject2conda/requirements.py b/src/pyproject2conda/requirements.py index b96d368..33b3695 100644 --- a/src/pyproject2conda/requirements.py +++ b/src/pyproject2conda/requirements.py @@ -232,27 +232,26 @@ def _as_list(x: str | Iterable[str]) -> Iterable[str]: else: return x - if not conda_deps and not pip_deps: - raise ValueError + if not conda_deps: + raise ValueError("Must have at least one conda dependency (i.e., pip)") out: list[str] = [] if name is not None: out.append(f"name: {name}") - if channels is not None: + if channels: out.append("channels:") for channel in _as_list(channels): out.append(f" - {channel}") out.append("dependencies:") - - if conda_deps is not None: - for dep in _as_list(conda_deps): - out.append(f" - {dep}") + for dep in _as_list(conda_deps): + out.append(f" - {dep}") if pip_deps: - out.append(" - pip") + if "pip" not in conda_deps: + raise ValueError("Must have pip in conda_deps") out.append(" - pip:") for dep in _as_list(pip_deps): @@ -534,7 +533,7 @@ def pip_requirements( out, remove_whitespace=remove_whitespace, unique=unique, sort=sort ) - def conda_pip_requirements( + def conda_and_pip_requirements( self, extras: str | Iterable[str] | None = None, include_base: bool = True, @@ -614,6 +613,14 @@ def _init_deps(deps: str | Iterable[str] | None) -> list[str]: + conda_deps ) + # special if have pip requirements or just pip in conda_deps + # in this case, make sure pip is last + if "pip" in conda_deps: + conda_deps.remove("pip") + conda_deps.append("pip") + elif pip_deps: + conda_deps.append("pip") + return conda_deps, pip_deps def to_conda_yaml( @@ -635,7 +642,7 @@ def to_conda_yaml( ) -> str: self._check_extras(extras) - conda_deps, pip_deps = self.conda_pip_requirements( + conda_deps, pip_deps = self.conda_and_pip_requirements( extras=extras, include_base=include_base, pip_deps=pip_deps, @@ -707,7 +714,7 @@ def to_conda_requirements( pip_deps: str | Iterable[str] | None = None, remove_whitespace: bool = True, ) -> tuple[str, str]: - conda_deps, pip_deps = self.conda_pip_requirements( + conda_deps, pip_deps = self.conda_and_pip_requirements( extras=extras, include_base=include_base, pip_deps=pip_deps, diff --git a/tests/test_cli.py b/tests/test_cli.py index a37357f..ff3f618 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -546,6 +546,7 @@ def test_conda_requirements(filename): #conda requirements bthing-conda conda-forge::cthing +pip #pip requirements athing @@ -559,6 +560,7 @@ def test_conda_requirements(filename): expected_conda = """\ bthing-conda conda-forge::cthing +pip """ expected_pip = """\ athing @@ -604,6 +606,7 @@ def test_conda_requirements(filename): expected_conda = """\ achannel::bthing-conda conda-forge::cthing +achannel::pip """ check_results_conda_req(d / "hello-conda.txt", expected_conda) @@ -617,7 +620,7 @@ def test_json(filename): result = do_run(runner, "j", filename=filename) expected = """\ -{"dependencies": ["bthing-conda", "conda-forge::cthing"], "pip": ["athing"], "channels": ["conda-forge"]} +{"dependencies": ["bthing-conda", "conda-forge::cthing", "pip"], "pip": ["athing"], "channels": ["conda-forge"]} """ assert result.output == dedent(expected) @@ -632,7 +635,7 @@ def check_results(path, expected): d = Path(d_tmp) expected = { # type: ignore - "dependencies": ["bthing-conda", "conda-forge::cthing"], + "dependencies": ["bthing-conda", "conda-forge::cthing", "pip"], "pip": ["athing"], "channels": ["conda-forge"], } @@ -649,6 +652,7 @@ def check_results(path, expected): "conda-forge::pytest", "additional-thing", "conda-matplotlib", + "pip", ], "pip": ["athing"], "channels": ["conda-forge"], @@ -675,6 +679,7 @@ def check_results(path, expected): "conda-forge::pytest", "conda-matplotlib", "pandas", + "pip", ], "pip": ["athing"], "channels": ["conda-forge"], diff --git a/tests/test_parser.py b/tests/test_parser.py index 1563a92..eaf8634 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -186,7 +186,7 @@ def test_output_to_yaml(): s = _conda_yaml() s = _conda_yaml( - conda_deps=["a"], + conda_deps=["a", "pip"], channels=["conda-forge"], pip_deps=["pip-thing"], name="hello", @@ -336,7 +336,7 @@ def test_package_name(): ) d = requirements.ParseDepends.from_string(toml) with pytest.raises(ValueError): - d.conda_pip_requirements("dev") + d.conda_and_pip_requirements("dev") @pytest.mark.parametrize( @@ -922,3 +922,36 @@ def test_spaces(toml): """ assert dedent(expected) == d.to_requirements(remove_whitespace=True) + + +def test_include_pip(): + toml = dedent( + """\ + [build-system] + requires = ["setuptools>=61.2", "setuptools_scm[toml]>=8.0"] + build-backend = "setuptools.build_meta" + + [project] + name = "hello" + requires-python = ">=3.8, <3.11" + dependencies = [ + "athing", + ] + + [project.optional-dependencies] + test = [ + "xthing", + ] + """ + ) + + expected = """\ + dependencies: + - athing + - xthing + - pip + """ + + d = requirements.ParseDepends.from_string(toml) + + assert dedent(expected) == d.to_conda_yaml(extras="test", conda_deps="pip") From ec45e67c2b1a18317422b109cbd22697e87b4b33 Mon Sep 17 00:00:00 2001 From: wpk Date: Tue, 28 Nov 2023 14:20:14 -0500 Subject: [PATCH 36/36] chore: update changelog --- CHANGELOG.md | 15 ++++++ ...607_william.krekelberg_get_build_system.md | 43 ----------------- ...8_134544_william.krekelberg_include_pip.md | 46 ------------------- 3 files changed, 15 insertions(+), 89 deletions(-) delete mode 100644 changelog.d/20231120_112607_william.krekelberg_get_build_system.md delete mode 100644 changelog.d/20231128_134544_william.krekelberg_include_pip.md diff --git a/CHANGELOG.md b/CHANGELOG.md index d15f92c..aa70c21 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,21 @@ See the fragment files in [changelog.d] +## v0.11.0 — 2023-11-28 + +### Added + +- Can now access "build-system.requires" as an extra. This can be useful for + creating isolated environments to build a package. + +### Changed + +- Can now specify `pip` as a conda dependency. This is needed for cases that + there are no pip dependencies in the environment, but you want it there for + installing local packages. This may be the case if using `conda-lock` on an + environment. Note that, much like python is always first in the dependency + list, pip is always last. + ## v0.10.0 — 2023-11-17 ### Added diff --git a/changelog.d/20231120_112607_william.krekelberg_get_build_system.md b/changelog.d/20231120_112607_william.krekelberg_get_build_system.md deleted file mode 100644 index 6588472..0000000 --- a/changelog.d/20231120_112607_william.krekelberg_get_build_system.md +++ /dev/null @@ -1,43 +0,0 @@ - - - - - -### Added - -- Can now access "build-system.requires" as an extra. This can be useful for - creating isolated environments to build a package. - - - - - diff --git a/changelog.d/20231128_134544_william.krekelberg_include_pip.md b/changelog.d/20231128_134544_william.krekelberg_include_pip.md deleted file mode 100644 index db6a6c7..0000000 --- a/changelog.d/20231128_134544_william.krekelberg_include_pip.md +++ /dev/null @@ -1,46 +0,0 @@ - - - - - - -### Changed - -- Can now specify `pip` as a conda dependency. This is needed for cases that - there are no pip dependencies in the environment, but you want it there for - installing local packages. This may be the case if using `conda-lock` on an - environment. Note that, much like python is always first in the dependency - list, pip is always last. - - - -