diff --git a/.github/ISSUE_TEMPLATE/bug-report.md b/.github/ISSUE_TEMPLATE/bug-report.md index 674d24844..2a1b732f7 100644 --- a/.github/ISSUE_TEMPLATE/bug-report.md +++ b/.github/ISSUE_TEMPLATE/bug-report.md @@ -1,12 +1,22 @@ +--- +name: Bug report +about: Create a report to help us improve +title: "" +labels: bug +assignees: "" +--- + **Issue** -Describe what's the expected behaviour and what you're observing. +Describe what's the expected behavior and what you're observing. **Environment** Provide at least: - OS: +- Shell: +- Python version and path: - `pip list` of the host python where `virtualenv` is installed: ```console diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md deleted file mode 100644 index 10d03bf5f..000000000 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ /dev/null @@ -1,30 +0,0 @@ ---- -name: Bug report -about: Create a report to help us improve -title: "" -labels: bug -assignees: "" ---- - -**Issue** - -Describe what's the expected behaviour and what you're observing. - -**Environment** - -Provide at least: - -- OS: -- `pip list` of the host python where `virtualenv` is installed: - - ```console - - ``` - -**Output of the virtual environment creation** - -Make sure to run the creation with `-vvv --with-traceback`: - -```console - -``` diff --git a/.github/workflows/check.yaml b/.github/workflows/check.yaml new file mode 100644 index 000000000..3f943853f --- /dev/null +++ b/.github/workflows/check.yaml @@ -0,0 +1,146 @@ +name: ๐Ÿงช check +on: + workflow_dispatch: + push: + branches: ["main"] + pull_request: + schedule: + - cron: "0 8 * * *" + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + test: + name: ๐Ÿงช test ${{ matrix.py }} - ${{ matrix.os }} + if: github.event_name != 'schedule' || github.repository_owner == 'pypa' + runs-on: ${{ matrix.os }} + timeout-minutes: 40 + strategy: + fail-fast: false + matrix: + py: + - "3.13t" + - "3.13" + - "3.12" + - "3.11" + - "3.10" + - "3.9" + - "3.8" + - pypy-3.11 + - pypy-3.10 + - pypy-3.9 + - pypy-3.8 + - graalpy-24.1 + os: + - ubuntu-24.04 + - macos-15 + - windows-2025 + include: + - { os: macos-15, py: "brew@3.11" } + - { os: macos-15, py: "brew@3.10" } + - { os: macos-15, py: "brew@3.9" } + exclude: + - { os: windows-2025, py: "graalpy-24.1" } + - { os: windows-2025, py: "pypy-3.10" } + - { os: windows-2025, py: "pypy-3.9" } + - { os: windows-2025, py: "pypy-3.8" } + steps: + - name: ๐Ÿš€ Install uv + uses: astral-sh/setup-uv@v4 + - name: ๐Ÿ“ฅ Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: ๐Ÿ Setup Python for tox + uses: actions/setup-python@v5 + with: + python-version: "3.13" + - name: ๐Ÿ“ฆ Install tox with this virtualenv + shell: bash + run: | + if [[ "${{ matrix.py }}" == "3.13t" ]]; then + uv tool install --no-managed-python --python 3.13 tox --with . + else + uv tool install --no-managed-python --python 3.13 tox --with tox-uv --with . + fi + - name: ๐Ÿ Setup Python for test ${{ matrix.py }} + uses: actions/setup-python@v5 + if: ${{ !startsWith(matrix.py, 'brew@') }} + with: + python-version: ${{ matrix.py }} + - name: ๐Ÿ› ๏ธ Install OS dependencies + shell: bash + run: | + if [ "${{ runner.os }}" = "Linux" ]; then + sudo apt-get install -y software-properties-common + sudo apt-add-repository ppa:fish-shell/release-4 -y + curl -fsSL https://apt.fury.io/nushell/gpg.key | sudo gpg --dearmor -o /etc/apt/trusted.gpg.d/fury-nushell.gpg + echo "deb https://apt.fury.io/nushell/ /" | sudo tee /etc/apt/sources.list.d/fury.list + sudo apt-get update -y + sudo apt-get install snapd fish csh nushell -y + elif [ "${{ runner.os }}" = "macOS" ]; then + brew update + if [[ "${{ matrix.py }}" == brew@* ]]; then + PY=$(echo '${{ matrix.py }}' | cut -c 6-) + brew install python@$PY || brew upgrade python@$PY + echo "/usr/local/opt/python@$PY/libexec/bin" >>"${GITHUB_PATH}" + fi + brew install fish tcsh nushell || brew upgrade fish tcsh nushell + elif [ "${{ runner.os }}" = "Windows" ]; then + choco install nushell + fi + - name: ๐Ÿงฌ Pick environment to run + shell: bash + run: | + py="${{ matrix.py }}" + if [[ "$py" == brew@* ]]; then + brew_version="${py#brew@}" + echo "TOX_DISCOVER=/opt/homebrew/bin/python${brew_version}" >> "$GITHUB_ENV" + py="$brew_version" + fi + [[ "$py" == graalpy-* ]] && py="graalpy" + echo "TOXENV=$py" >> "$GITHUB_ENV" + echo "Set TOXENV=$py" + - name: ๐Ÿ—๏ธ Setup test suite + run: tox run -vvvv --notest --skip-missing-interpreters false + - name: ๐Ÿƒ Run test suite + run: tox run --skip-pkg-install + timeout-minutes: 20 + env: + PYTEST_ADDOPTS: "-vv --durations=20" + CI_RUN: "yes" + DIFF_AGAINST: HEAD + check: + name: ๐Ÿ”Ž check ${{ matrix.tox_env }} - ${{ matrix.os }} + if: github.event_name != 'schedule' || github.repository_owner == 'pypa' + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: + - ubuntu-24.04 + - windows-2025 + tox_env: + - dev + - docs + - readme + - upgrade + - zipapp + exclude: + - { os: windows-2025, tox_env: readme } + - { os: windows-2025, tox_env: docs } + steps: + - name: ๐Ÿš€ Install uv + uses: astral-sh/setup-uv@v4 + - name: ๐Ÿ“ฆ Install tox + run: uv tool install --python-preference only-managed --python 3.13 tox --with tox-uv + - name: ๐Ÿ“ฅ Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: ๐Ÿ—๏ธ Setup check suite + run: tox run -vv --notest --skip-missing-interpreters false -e ${{ matrix.tox_env }} + - name: ๐Ÿƒ Run check for ${{ matrix.tox_env }} + run: tox run --skip-pkg-install -e ${{ matrix.tox_env }} diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml deleted file mode 100644 index 7479c470a..000000000 --- a/.github/workflows/check.yml +++ /dev/null @@ -1,127 +0,0 @@ -name: check -on: - push: - tags-ignore: ["**"] - pull_request: - schedule: - - cron: "0 8 * * *" - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - test: - name: test ${{ matrix.py }} - ${{ matrix.os }} - if: github.event_name != 'schedule' || github.repository_owner == 'pypa' - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - py: - - "3.12.0-beta.2" - - "3.11" - - "3.10" - - "3.9" - - "3.8" - - "3.7" - - pypy-3.9 - - pypy-3.8 - - pypy-3.7 - os: - - ubuntu-latest - - macos-latest - - windows-latest - include: - - { os: macos-latest, py: "brew@3.9" } - - { os: macos-latest, py: "brew@3.8" } - steps: - - uses: taiki-e/install-action@cargo-binstall - - name: Install OS dependencies - run: | - set -x - for i in 1 2 3; do - echo "try $i" && \ - ${{ runner.os == 'Linux' && 'sudo apt-get update -y && sudo apt-get install snapd fish csh -y' || true }} && \ - ${{ runner.os == 'Linux' && 'cargo binstall -y nu' || true }} && \ - ${{ runner.os == 'macOS' && 'brew install fish tcsh nushell' || true }} && \ - ${{ runner.os == 'Windows' && 'choco install nushell' || true }} && \ - exit 0 || true; - sleep 1 - done - exit 1 - shell: bash - - name: Setup python for tox - uses: actions/setup-python@v4 - with: - python-version: "3.11" - - name: Install tox - run: python -m pip install tox pip -U - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - name: Use local virtualenv for tox - run: python -m pip install . - - name: Setup brew python for test ${{ matrix.py }} - if: startsWith(matrix.py,'brew@') - run: | - set -e - PY=$(echo '${{ matrix.py }}' | cut -c 6-) - brew upgrade python@$PY || brew install python@$PY - echo "/usr/local/opt/python@$PY/libexec/bin" >>"${GITHUB_PATH}" - shell: bash - - name: Setup python for test ${{ matrix.py }} - if: "!( startsWith(matrix.py,'brew@') || endsWith(matrix.py, '-dev') )" - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.py }} - - name: Pick environment to run - run: | - import os; import platform; import sys; from pathlib import Path - env = f'TOXENV=py{"" if platform.python_implementation() == "CPython" else "py"}3{sys.version_info.minor}' - print(f"Picked: {env} for {sys.version} based of {sys.executable}") - with Path(os.environ["GITHUB_ENV"]).open("ta") as file_handler: - file_handler.write(env) - shell: python - - name: Setup test suite - run: tox -vv --notest - - name: Run test suite - run: tox --skip-pkg-install - env: - PYTEST_ADDOPTS: "-vv --durations=20" - CI_RUN: "yes" - DIFF_AGAINST: HEAD - - check: - name: ${{ matrix.tox_env }} - ${{ matrix.os }} - if: github.event_name != 'schedule' || github.repository_owner == 'pypa' - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - os: - - ubuntu-latest - - windows-latest - tox_env: - - dev - - docs - - readme - - upgrade - - zipapp - exclude: - - { os: windows-latest, tox_env: readme } - - { os: windows-latest, tox_env: docs } - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - name: Setup Python "3.11" - uses: actions/setup-python@v4 - with: - python-version: "3.11" - - name: Install tox - run: python -m pip install tox - - name: Run check for ${{ matrix.tox_env }} - run: python -m tox -e ${{ matrix.tox_env }} - env: - UPGRADE_ADVISORY: "yes" diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml new file mode 100644 index 000000000..820c63f65 --- /dev/null +++ b/.github/workflows/release.yaml @@ -0,0 +1,49 @@ +name: Release to PyPI +on: + push: + tags: ["*"] + +env: + dists-artifact-name: python-package-distributions + +jobs: + build: + runs-on: ubuntu-24.04 + steps: + - name: ๐Ÿ“ฅ Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: ๐Ÿš€ Install the latest version of uv + uses: astral-sh/setup-uv@v4 + with: + enable-cache: true + cache-dependency-glob: "pyproject.toml" + github-token: ${{ secrets.GITHUB_TOKEN }} + - name: ๐Ÿ“ฆ Build package + run: uv build --python 3.13 --python-preference only-managed --sdist --wheel . --out-dir dist + - name: ๐Ÿ“ฆ Store the distribution packages + uses: actions/upload-artifact@v4 + with: + name: ${{ env.dists-artifact-name }} + path: dist/* + + release: + needs: + - build + runs-on: ubuntu-24.04 + environment: + name: release + url: https://pypi.org/project/virtualenv/${{ github.ref_name }} + permissions: + id-token: write + steps: + - name: ๐Ÿ“ฅ Download all the dists + uses: actions/download-artifact@v4 + with: + name: ${{ env.dists-artifact-name }} + path: dist/ + - name: ๐Ÿš€ Publish to PyPI + uses: pypa/gh-action-pypi-publish@v1.12.3 + with: + attestations: true diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml deleted file mode 100644 index b48393ebc..000000000 --- a/.github/workflows/release.yml +++ /dev/null @@ -1,27 +0,0 @@ -name: Release to PyPI -on: - push: - tags: ["*"] - -jobs: - release: - runs-on: ubuntu-latest - environment: - name: release - url: https://pypi.org/p/virtualenv - permissions: - id-token: write - steps: - - name: Setup python to build package - uses: actions/setup-python@v4 - with: - python-version: "3.11" - - name: Install build - run: python -m pip install build - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - name: Build package - run: pyproject-build -s -w . -o dist - - name: Publish to PyPI - uses: pypa/gh-action-pypi-publish@v1.8.10 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4bfaf2a3d..17611776d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,33 +1,41 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.4.0 + rev: v6.0.0 hooks: - id: end-of-file-fixer - id: trailing-whitespace - - repo: https://github.com/psf/black - rev: 23.7.0 + - repo: https://github.com/python-jsonschema/check-jsonschema + rev: 0.33.2 hooks: - - id: black + - id: check-github-workflows + args: ["--verbose"] + - repo: https://github.com/codespell-project/codespell + rev: v2.4.1 + hooks: + - id: codespell + args: ["--write-changes"] - repo: https://github.com/tox-dev/tox-ini-fmt - rev: "1.3.1" + rev: "1.6.0" hooks: - id: tox-ini-fmt args: ["-p", "fix"] - repo: https://github.com/tox-dev/pyproject-fmt - rev: "1.1.0" + rev: "v2.6.0" hooks: - id: pyproject-fmt - additional_dependencies: ["tox>=4.8"] - - repo: https://github.com/pre-commit/mirrors-prettier - rev: "v3.0.3" - hooks: - - id: prettier - args: ["--print-width=120", "--prose-wrap=always"] - repo: https://github.com/astral-sh/ruff-pre-commit - rev: "v0.0.287" + rev: "v0.12.8" hooks: + - id: ruff-format - id: ruff - args: [--fix, --exit-non-zero-on-fix] + args: ["--fix", "--unsafe-fixes", "--exit-non-zero-on-fix"] + - repo: https://github.com/rbubley/mirrors-prettier + rev: "v3.6.2" + hooks: + - id: prettier + additional_dependencies: + - prettier@3.3.3 + - "@prettier/plugin-xml@3.4.1" - repo: meta hooks: - id: check-hooks-apply diff --git a/README.md b/README.md index 7c98b12b5..87f45e077 100644 --- a/README.md +++ b/README.md @@ -7,9 +7,7 @@ [![Discord](https://img.shields.io/discord/803025117553754132)](https://discord.gg/pypa) [![Downloads](https://static.pepy.tech/badge/virtualenv/month)](https://pepy.tech/project/virtualenv) [![PyPI - License](https://img.shields.io/pypi/l/virtualenv?style=flat-square)](https://opensource.org/licenses/MIT) -[![Build Status](https://github.com/pypa/virtualenv/workflows/check/badge.svg?branch=main&event=push)](https://github.com/pypa/virtualenv/actions?query=workflow%3Acheck) -[![Code style: -black](https://img.shields.io/badge/code%20style-black-000000.svg?style=flat-square)](https://github.com/psf/black) +[![check](https://github.com/pypa/virtualenv/actions/workflows/check.yaml/badge.svg)](https://github.com/pypa/virtualenv/actions/workflows/check.yaml) A tool for creating isolated `virtual` python environments. diff --git a/docs/changelog.rst b/docs/changelog.rst index 6a193cba3..e333f091f 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,6 +5,354 @@ Release History .. towncrier release notes start +v20.34.0 (2025-08-13) +--------------------- + +Features - 20.34.0 +~~~~~~~~~~~~~~~~~~ +- Abstract out caching in discovery - by :user:`esafak`. + Decouple `FileCache` from `py_info` (discovery) - by :user:`esafak`. + Remove references to py_info in FileCache - by :user:`esafak`. + Decouple discovery from creator plugins - by :user:`esafak`. + Decouple discovery by duplicating info utils - by :user:`esafak`. (:issue:`2074`) +- Add PyPy 3.11 support. Contributed by :user:`esafak`. (:issue:`2932`) + +Bugfixes - 20.34.0 +~~~~~~~~~~~~~~~~~~ +- Upgrade embedded wheel pip to ``25.2`` from ``25.1.1`` - by :user:`gaborbernat`. (:issue:`2333`) +- Accept RuntimeError in `test_too_many_open_files`, by :user:`esafak` (:issue:`2935`) +- Python in PATH takes precedence over uv-managed python. Contributed by :user:`edgarrmondragon`. (:issue:`2952`) + +v20.33.1 (2025-08-05) +--------------------- + +Bugfixes - 20.33.1 +~~~~~~~~~~~~~~~~~~ +- Correctly unpack _get_tcl_tk_libs() response in PythonInfo. + Contributed by :user:`esafak`. (:issue:`2930`) +- Restore `py_info.py` timestamp in `test_py_info_cache_invalidation_on_py_info_change` + Contributed by :user:`esafak`. (:issue:`2933`) + +v20.33.0 (2025-08-03) +--------------------- + +Features - 20.33.0 +~~~~~~~~~~~~~~~~~~ +- Added support for Tcl and Tkinter. You're welcome. + Contributed by :user:`esafak`. (:issue:`425`) + +Bugfixes - 20.33.0 +~~~~~~~~~~~~~~~~~~ +- Prevent logging setup when --help is passed, fixing a flaky test. + Contributed by :user:`esafak`. (:issue:`u`) +- Fix cache invalidation for PythonInfo by hashing `py_info.py`. + Contributed by :user:`esafak`. (:issue:`2467`) +- When no discovery plugins are found, the application would crash with a StopIteration. + This change catches the StopIteration and raises a RuntimeError with a more informative message. + Contributed by :user:`esafak`. (:issue:`2493`) +- Stop `--try-first-with` overriding absolute `--python` paths. + Contributed by :user:`esafak`. (:issue:`2659`) +- Force UTF-8 encoding for pip download + Contributed by :user:`esafak`. (:issue:`2780`) +- Creating a virtual environment on a filesystem without symlink-support would fail even with `--copies` + Make `fs_supports_symlink` perform a real symlink creation check on all platforms. + Contributed by :user:`esafak`. (:issue:`2786`) +- Add a note to the user guide recommending the use of a specific Python version when creating virtual environments. + Contributed by :user:`esafak`. (:issue:`2808`) +- Fix 'Too many open files' error due to a file descriptor leak in virtualenv's locking mechanism. + Contributed by :user:`esafak`. (:issue:`2834`) +- Support renamed Windows venv redirector (`venvlauncher.exe` and `venvwlauncher.exe`) on Python 3.13 + Contributed by :user:`esafak`. (:issue:`2851`) +- Resolve Nushell activation script deprecation warnings by dynamically selecting the ``--optional`` flag for Nushell + ``get`` command on version 0.106.0 and newer, while retaining the deprecated ``-i`` flag for older versions to maintain + compatibility. Contributed by :user:`gaborbernat`. (:issue:`2910`) + +v20.32.0 (2025-07-20) +--------------------- + +Features - 20.32.0 +~~~~~~~~~~~~~~~~~~ +- Warn on incorrect invocation of Nushell activation script - by :user:`esafak`. (:issue:`nushell_activation`) +- Discover uv-managed Python installations (:issue:`2901`) + +Bugfixes - 20.32.0 +~~~~~~~~~~~~~~~~~~ +- Ignore missing absolute paths for python discovery - by :user:`esafak` (:issue:`2870`) +- Upgrade embedded setuptools to ``80.9.0`` from ``80.3.1`` - by :user:`gaborbernat`. (:issue:`2900`) + +v20.31.2 (2025-05-08) +--------------------- + +No significant changes. + + +v20.31.1 (2025-05-05) +--------------------- + +Bugfixes - 20.31.1 +~~~~~~~~~~~~~~~~~~ +- Upgrade embedded wheels: + + * pip to ``25.1.1`` from ``25.1`` + * setuptools to ``80.3.1`` from ``78.1.0`` (:issue:`2880`) + +v20.31.0 (2025-05-05) +--------------------- + +Features - 20.31.0 +~~~~~~~~~~~~~~~~~~ +- No longer bundle ``wheel`` wheels (except on Python 3.8), ``setuptools`` includes native ``bdist_wheel`` support. Update ``pip`` to ``25.1``. (:issue:`2868`) + +Bugfixes - 20.31.0 +~~~~~~~~~~~~~~~~~~ +- ``get_embed_wheel()`` no longer fails with a :exc:`TypeError` when it is + called with an unknown *distribution*. (:issue:`2877`) +- Fix ``HelpFormatter`` error with Python 3.14.0b1. (:issue:`2878`) + +v20.30.0 (2025-03-31) +--------------------- + +Features - 20.30.0 +~~~~~~~~~~~~~~~~~~ +- Add support for `GraalPy `_. (:issue:`2832`) + +Bugfixes - 20.30.0 +~~~~~~~~~~~~~~~~~~ +- Upgrade embedded wheels: + + * setuptools to ``78.1.0`` from ``75.3.2`` (:issue:`2863`) + +v20.29.3 (2025-03-06) +--------------------- + +Bugfixes - 20.29.3 +~~~~~~~~~~~~~~~~~~ +- Ignore unreadable directories in ``PATH``. (:issue:`2794`) + +v20.29.2 (2025-02-10) +--------------------- + +Bugfixes - 20.29.2 +~~~~~~~~~~~~~~~~~~ +- Remove old virtualenv wheel from the source distribution - by :user:`gaborbernat`. (:issue:`2841`) +- Upgrade embedded wheel pip to ``25.0.1`` from ``24.3.1`` - by :user:`gaborbernat`. (:issue:`2843`) + +v20.29.1 (2025-01-17) +--------------------- + +Bugfixes - 20.29.1 +~~~~~~~~~~~~~~~~~~ +- Fix PyInfo cache incompatibility warnings - by :user:`robsdedude`. (:issue:`2827`) + +v20.29.0 (2025-01-15) +--------------------- + +Features - 20.29.0 +~~~~~~~~~~~~~~~~~~ +- Add support for selecting free-threaded Python interpreters, e.g., `python3.13t`. (:issue:`2809`) + +Bugfixes - 20.29.0 +~~~~~~~~~~~~~~~~~~ +- Upgrade embedded wheels: + + * setuptools to ``75.8.0`` from ``75.6.0`` (:issue:`2823`) + +v20.28.1 (2025-01-02) +--------------------- + +Bugfixes - 20.28.1 +~~~~~~~~~~~~~~~~~~ +- Skip tcsh tests on broken tcsh versions - by :user:`gaborbernat`. (:issue:`2814`) + +v20.28.0 (2024-11-25) +--------------------- + +Features - 20.28.0 +~~~~~~~~~~~~~~~~~~ +- Write CACHEDIR.TAG file on creation - by "user:`neilramsay`. (:issue:`2803`) + +v20.27.2 (2024-11-25) +--------------------- + +Bugfixes - 20.27.2 +~~~~~~~~~~~~~~~~~~ +- Upgrade embedded wheels: + + * setuptools to ``75.3.0`` from ``75.2.0`` (:issue:`2798`) +- Upgrade embedded wheels: + + * wheel to ``0.45.0`` from ``0.44.0`` + * setuptools to ``75.5.0`` (:issue:`2800`) +- no longer forcibly echo off during windows batch activation (:issue:`2801`) +- Upgrade embedded wheels: + + * setuptools to ``75.6.0`` from ``75.5.0`` + * wheel to ``0.45.1`` from ``0.45.0`` (:issue:`2804`) + +v20.27.1 (2024-10-28) +--------------------- + +Bugfixes - 20.27.1 +~~~~~~~~~~~~~~~~~~ +- Upgrade embedded wheels: + + * pip to ``24.3.1`` from ``24.2`` (:issue:`2789`) + +v20.27.0 (2024-10-17) +--------------------- + +Features - 20.27.0 +~~~~~~~~~~~~~~~~~~ +- Drop 3.7 support as the CI environments no longer allow it running - by :user:`gaborbernat`. (:issue:`2758`) + +Bugfixes - 20.27.0 +~~~~~~~~~~~~~~~~~~ +- When a ``$PATH`` entry cannot be checked for existence, skip it instead of terminating - by :user:`hroncok`. (:issue:`2782`) +- Upgrade embedded wheels: + + * setuptools to ``75.2.0`` from ``75.1.0`` + * Removed pip of ``24.0`` + * Removed setuptools of ``68.0.0`` + * Removed wheel of ``0.42.0`` + + - by :user:`gaborbernat`. (:issue:`2783`) +- Fix zipapp is broken on Windows post distlib ``0.3.9`` - by :user:`gaborbernat`. (:issue:`2784`) + +v20.26.6 (2024-09-27) +--------------------- + +Bugfixes - 20.26.6 +~~~~~~~~~~~~~~~~~~ +- Properly quote string placeholders in activation script templates to mitigate + potential command injection - by :user:`y5c4l3`. (:issue:`2768`) + +v20.26.5 (2024-09-17) +--------------------- + +Bugfixes - 20.26.5 +~~~~~~~~~~~~~~~~~~ +- Upgrade embedded wheels: setuptools to ``75.1.0`` from ``74.1.2`` - by :user:`gaborbernat`. (:issue:`2765`) + +v20.26.4 (2024-09-07) +--------------------- + +Bugfixes - 20.26.4 +~~~~~~~~~~~~~~~~~~ +- no longer create `()` output in console during activation of a virtualenv by .bat file. (:issue:`2728`) +- Upgrade embedded wheels: + + * wheel to ``0.44.0`` from ``0.43.0`` + * pip to ``24.2`` from ``24.1`` + * setuptools to ``74.1.2`` from ``70.1.0`` (:issue:`2760`) + +v20.26.3 (2024-06-21) +--------------------- + +Bugfixes - 20.26.3 +~~~~~~~~~~~~~~~~~~ +- Upgrade embedded wheels: + + * setuptools to ``70.1.0`` from ``69.5.1`` + * pip to ``24.1`` from ``24.0`` (:issue:`2741`) + +v20.26.2 (2024-05-13) +--------------------- + +Bugfixes - 20.26.2 +~~~~~~~~~~~~~~~~~~ +- ``virtualenv.pyz`` no longer fails when zipapp path contains a symlink - by :user:`HandSonic` and :user:`petamas`. (:issue:`1949`) +- Fix bad return code from activate.sh if hashing is disabled - by :user:'fenkes-ibm'. (:issue:`2717`) + +v20.26.1 (2024-04-29) +--------------------- + +Bugfixes - 20.26.1 +~~~~~~~~~~~~~~~~~~ +- fix PATH-based Python discovery on Windows - by :user:`ofek`. (:issue:`2712`) + +v20.26.0 (2024-04-23) +--------------------- + +Bugfixes - 20.26.0 +~~~~~~~~~~~~~~~~~~ +- allow builtin discovery to discover specific interpreters (e.g. ``python3.12``) given an unspecific spec (e.g. ``python3``) - by :user:`flying-sheep`. (:issue:`2709`) + +v20.25.3 (2024-04-17) +--------------------- + +Bugfixes - 20.25.3 +~~~~~~~~~~~~~~~~~~ +- Python 3.13.0a6 renamed pathmod to parser. (:issue:`2702`) + +v20.25.2 (2024-04-16) +--------------------- + +Bugfixes - 20.25.2 +~~~~~~~~~~~~~~~~~~ +- Upgrade embedded wheels: + + - setuptools of ``69.1.0`` to ``69.5.1`` + - wheel of ``0.42.0`` to ``0.43.0`` (:issue:`2699`) + +v20.25.1 (2024-02-21) +--------------------- + +Bugfixes - 20.25.1 +~~~~~~~~~~~~~~~~~~ +- Upgrade embedded wheels: + + * setuptools to ``69.0.3`` from ``69.0.2`` + * pip to ``23.3.2`` from ``23.3.1`` (:issue:`2681`) +- Upgrade embedded wheels: + + - pip ``23.3.2`` to ``24.0``, + - setuptools ``69.0.3`` to ``69.1.0``. (:issue:`2691`) + +Misc - 20.25.1 +~~~~~~~~~~~~~~ +- :issue:`2688` + +v20.25.0 (2023-12-01) +--------------------- + +Features - 20.25.0 +~~~~~~~~~~~~~~~~~~ +- The tests now pass on the CI with Python 3.13.0a2 - by :user:`hroncok`. (:issue:`2673`) + +Bugfixes - 20.25.0 +~~~~~~~~~~~~~~~~~~ +- Upgrade embedded wheels: + + * wheel to ``0.41.3`` from ``0.41.2`` (:issue:`2665`) +- Upgrade embedded wheels: + + * wheel to ``0.42.0`` from ``0.41.3`` + * setuptools to ``69.0.2`` from ``68.2.2`` (:issue:`2669`) + +v20.24.6 (2023-10-23) +--------------------- + +Bugfixes - 20.24.6 +~~~~~~~~~~~~~~~~~~ +- Use get_hookimpls method instead of the private attribute in tests. (:issue:`2649`) +- Upgrade embedded wheels: + + * setuptools to ``68.2.2`` from ``68.2.0`` + * pip to ``23.3.1`` from ``23.2.1`` (:issue:`2656`) + + +v20.24.5 (2023-09-08) +--------------------- + +Bugfixes - 20.24.5 +~~~~~~~~~~~~~~~~~~ +- Declare PyPy 3.10 support - by :user:`cclauss`. (:issue:`2638`) +- Brew on macOS no longer allows copy builds - disallow choosing this by :user:`gaborbernat`. (:issue:`2640`) +- Upgrade embedded wheels: + + * setuptools to ``68.2.0`` from ``68.1.2`` (:issue:`2642`) + + v20.24.4 (2023-08-30) --------------------- @@ -84,7 +432,7 @@ v20.23.0 (2023-04-27) Features - 20.23.0 ~~~~~~~~~~~~~~~~~~ -- Do not install ``wheel`` and ``setuptools`` seed packages for Python 3.12+. To restore the old behaviour use: +- Do not install ``wheel`` and ``setuptools`` seed packages for Python 3.12+. To restore the old behavior use: - for ``wheel`` use ``VIRTUALENV_WHEEL=bundle`` environment variable or ``--wheel=bundle`` CLI flag, - for ``setuptools`` use ``VIRTUALENV_SETUPTOOLS=bundle`` environment variable or ``--setuptools=bundle`` CLI flag. @@ -181,7 +529,7 @@ v20.17.1 (2022-12-05) Bugfixes - 20.17.1 ~~~~~~~~~~~~~~~~~~ - A ``py`` or ``python`` spec means any Python rather than ``CPython`` - by :user:`gaborbernat`. (`#2460 `_) -- Make ``activate.nu`` respect ``VIRTUAL_ENV_DISABLE_PROMPT`` and not set the prompt if reqeusted - by :user:`m-lima`. (`#2461 `_) +- Make ``activate.nu`` respect ``VIRTUAL_ENV_DISABLE_PROMPT`` and not set the prompt if requested - by :user:`m-lima`. (`#2461 `_) v20.17.0 (2022-11-27) @@ -218,7 +566,7 @@ Features - 20.16.6 Bugfixes - 20.16.6 ~~~~~~~~~~~~~~~~~~ -- Fix selected scheme on debian derivatives for python 3.10 when ``python3-distutils`` is not installed or the ``venv`` scheme is not avaiable - by :user:`asottile`. (`#2350 `_) +- Fix selected scheme on debian derivatives for python 3.10 when ``python3-distutils`` is not installed or the ``venv`` scheme is not available - by :user:`asottile`. (`#2350 `_) - Allow the test suite to pass even with the original C shell (rather than ``tcsh``) - by :user:`kulikjak`. (`#2418 `_) - Fix fallback handling of downloading wheels for bundled packages - by :user:`schaap`. (`#2429 `_) - Upgrade embedded setuptools to ``65.5.0`` from ``65.3.0`` and pip to ``22.3`` from ``22.2.2`` - by :user:`gaborbernat`. (`#2434 `_) @@ -254,7 +602,7 @@ v20.16.2 (2022-07-27) Bugfixes - 20.16.2 ~~~~~~~~~~~~~~~~~~ -- Bump embeded pip from ``22.2`` to ``22.2.1`` - by :user:`gaborbernat`. (`#2391 `_) +- Bump embedded pip from ``22.2`` to ``22.2.1`` - by :user:`gaborbernat`. (`#2391 `_) v20.16.1 (2022-07-26) @@ -354,7 +702,7 @@ v20.13.1 (2022-02-05) Bugfixes - 20.13.1 ~~~~~~~~~~~~~~~~~~ - fix "execv() arg 2 must contain only strings" error on M1 MacOS (`#2282 `_) -- Ugrade embedded setuptools to ``60.5.0`` from ``60.2.0`` - by :user:`asottile`. (`#2289 `_) +- Upgrade embedded setuptools to ``60.5.0`` from ``60.2.0`` - by :user:`asottile`. (`#2289 `_) - Upgrade embedded pip to ``22.0.3`` and setuptools to ``60.6.0`` - by :user:`gaborbernat` and :user:`asottile`. (`#2294 `_) @@ -398,7 +746,7 @@ Features - 20.12.0 Bugfixes - 20.12.0 ~~~~~~~~~~~~~~~~~~ - Fix ``--download`` option - by :user:`mayeut`. (`#2120 `_) -- Ugrade embedded setuptools to ``60.2.0`` from ``60.1.1`` - by :user:`gaborbernat`. (`#2263 `_) +- Upgrade embedded setuptools to ``60.2.0`` from ``60.1.1`` - by :user:`gaborbernat`. (`#2263 `_) v20.11.2 (2021-12-29) @@ -439,7 +787,7 @@ Features - 20.10.0 A similar technique `was proposed to Python, for the venv module `_ - by ``hroncok`` (`#2208 `_) - The activated virtualenv prompt is now always wrapped in parentheses. This affects venvs created with the ``--prompt`` attribute, and matches virtualenv's - behaviour on par with venv. (`#2224 `_) + behavior on par with venv. (`#2224 `_) Bugfixes - 20.10.0 ~~~~~~~~~~~~~~~~~~ @@ -791,7 +1139,7 @@ Bugfixes - 20.0.26 - Improve periodic update handling: - better logging output while running and enable logging on background process call ( - ``_VIRTUALENV_PERIODIC_UPDATE_INLINE`` may be used to debug behaviour inline) + ``_VIRTUALENV_PERIODIC_UPDATE_INLINE`` may be used to debug behavior inline) - fallback to unverified context when querying the PyPi for release date, - stop downloading wheels once we reach the embedded version, @@ -1151,7 +1499,7 @@ v20.0.2 (2020-02-11) Features - 20.0.2 ~~~~~~~~~~~~~~~~~ - Print out a one line message about the created virtual environment when no :option:`verbose` is set, this can now be - silenced to get back the original behaviour via the :option:`quiet` flag - by :user:`pradyunsg`. (`#1557 `_) + silenced to get back the original behavior via the :option:`quiet` flag - by :user:`pradyunsg`. (`#1557 `_) - Allow virtualenv's app data cache to be overridden by ``VIRTUALENV_OVERRIDE_APP_DATA`` - by :user:`asottile`. (`#1559 `_) - Passing in the virtual environment name/path is now required (no longer defaults to ``venv``) - by :user:`gaborbernat`. (`#1568 `_) - Add a CLI flag :option:`with-traceback` that allows displaying the stacktrace of the virtualenv when a failure occurs diff --git a/docs/changelog/2074.feature.rst b/docs/changelog/2074.feature.rst new file mode 100644 index 000000000..61b1c38b5 --- /dev/null +++ b/docs/changelog/2074.feature.rst @@ -0,0 +1 @@ +Add AppData and Cache protocols to discovery for decoupling - by :user:`esafak`. diff --git a/docs/changelog/2774.feature.rst b/docs/changelog/2774.feature.rst new file mode 100644 index 000000000..7821b80cf --- /dev/null +++ b/docs/changelog/2774.feature.rst @@ -0,0 +1 @@ +Ensure python3.exe and python3 on Windows for Python 3. - by :user:`esafak`. diff --git a/docs/changelog/2944.bugfix.rst b/docs/changelog/2944.bugfix.rst new file mode 100644 index 000000000..21ba0d22e --- /dev/null +++ b/docs/changelog/2944.bugfix.rst @@ -0,0 +1 @@ +Replaced direct references to tcl/tk library paths with getattr. By :user:`esafak` diff --git a/docs/cli_interface.rst b/docs/cli_interface.rst index 7f5798f52..27ea231ef 100644 --- a/docs/cli_interface.rst +++ b/docs/cli_interface.rst @@ -10,7 +10,7 @@ CLI flags It modifies the environment variables in a shell to create an isolated Python environment, so you'll need to have a shell to run it. You can type in ``virtualenv`` (name of the application) followed by flags that control its -behaviour. All options have sensible defaults, and there's one required argument: the name/path of the virtual +behavior. All options have sensible defaults, and there's one required argument: the name/path of the virtual environment to create. The default values for the command line options can be overridden via the :ref:`conf_file` or :ref:`env_vars`. Environment variables takes priority over the configuration file values (``--help`` will show if a default comes from the environment variable as the help message will end in this case @@ -24,6 +24,51 @@ The options that can be passed to virtualenv, along with their default values an :module: virtualenv.run :func: build_parser_only +Discovery options +~~~~~~~~~~~~~~~~~ + +Understanding Interpreter Discovery: ``--python`` vs. ``--try-first-with`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +You can control which Python interpreter ``virtualenv`` selects using the ``--python`` and ``--try-first-with`` flags. +To avoid confusion, it's best to think of them as the "rule" and the "hint". + +**``--python ``: The Rule** + +This flag sets the mandatory requirements for the interpreter. The ```` can be: + +- **A version string** (e.g., ``python3.8``, ``pypy3``). ``virtualenv`` will search for any interpreter that matches this version. +- **An absolute path** (e.g., ``/usr/bin/python3.8``). This is a *strict* requirement. Only the interpreter at this exact path will be used. If it does not exist or is not a valid interpreter, creation will fail. + +**``--try-first-with ``: The Hint** + +This flag provides a path to a Python executable to check *before* ``virtualenv`` performs its standard search. This can speed up discovery or help select a specific interpreter when multiple versions exist on your system. + +**How They Work Together** + +``virtualenv`` will only use an interpreter from ``--try-first-with`` if it **satisfies the rule** from the ``--python`` flag. The ``--python`` rule always wins. + +**Examples:** + +1. **Hint does not match the rule:** + + .. code-block:: bash + + virtualenv --python python3.8 --try-first-with /usr/bin/python3.10 my-env + + - **Result:** ``virtualenv`` first inspects ``/usr/bin/python3.10``. It sees this does not match the ``python3.8`` rule and **rejects it**. It then proceeds with its normal search to find a ``python3.8`` interpreter elsewhere. + +2. **Hint does not match a strict path rule:** + + .. code-block:: bash + + virtualenv --python /usr/bin/python3.8 --try-first-with /usr/bin/python3.10 my-env + + - **Result:** The rule is strictly ``/usr/bin/python3.8``. ``virtualenv`` checks the ``/usr/bin/python3.10`` hint, sees the path doesn't match, and **rejects it**. It then moves on to test ``/usr/bin/python3.8`` and successfully creates the environment. + +This approach ensures that the behavior is predictable and that ``--python`` remains the definitive source of truth for the user's intent. + + Defaults ~~~~~~~~ diff --git a/docs/conf.py b/docs/conf.py index cd4f9bfc8..f02c8a7cb 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -52,7 +52,7 @@ def setup(app): root, exe = here.parent, Path(sys.executable) towncrier = exe.with_name(f"towncrier{exe.suffix}") cmd = [str(towncrier), "build", "--draft", "--version", "NEXT"] - new = subprocess.check_output(cmd, cwd=root, text=True, stderr=subprocess.DEVNULL, encoding="UTF-8") # noqa: S603 + new = subprocess.check_output(cmd, cwd=root, text=True, stderr=subprocess.DEVNULL, encoding="UTF-8") (root / "docs" / "_draft.rst").write_text("" if "No significant changes" in new else new, encoding="UTF-8") # the CLI arguments are dynamically generated @@ -66,7 +66,7 @@ def setup(app): sys.path.append(str(here)) # noinspection PyUnresolvedReferences - from render_cli import CliTable, literal_data + from render_cli import CliTable, literal_data # noqa: PLC0415 app.add_css_file("custom.css") app.add_directive(CliTable.name, CliTable) diff --git a/docs/development.rst b/docs/development.rst index ef5b75e61..941a8b5c9 100644 --- a/docs/development.rst +++ b/docs/development.rst @@ -106,7 +106,7 @@ that folder. Release ~~~~~~~ -virtualenv's release schedule is tied to ``pip``, ``setuptools`` and ``wheel``. We bundle the latest version of these +virtualenv's release schedule is tied to ``pip`` and ``setuptools``. We bundle the latest version of these libraries so each time there's a new version of any of these, there will be a new virtualenv release shortly afterwards (we usually wait just a few days to avoid pulling in any broken releases). diff --git a/docs/extend.rst b/docs/extend.rst index 940e783e1..6dc107e3b 100644 --- a/docs/extend.rst +++ b/docs/extend.rst @@ -10,7 +10,7 @@ Extend functionality Python discovery ---------------- -The python discovery mechanism is a component that needs to answer the following answer: based on some type of user +The python discovery mechanism is a component that needs to answer the following question: based on some type of user input give me a Python interpreter on the machine that matches that. The builtin interpreter tries to discover an installed Python interpreter (based on PEP-515 and ``PATH`` discovery) on the users machine where the user input is a python specification. An alternative such discovery mechanism for example would be to use the popular diff --git a/docs/index.rst b/docs/index.rst index 424a8ae27..52484a9ae 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,4 +1,4 @@ -Virtualenv +virtualenv ========== .. image:: https://img.shields.io/pypi/v/virtualenv?style=flat-square @@ -30,7 +30,12 @@ Virtualenv :target: https://pypistats.org/packages/virtualenv :alt: Package popularity -``virtualenv`` is a tool to create isolated Python environments. Since Python ``3.3``, a subset of it has been +``virtualenv`` is a tool to create isolated Python environments. + +virtualenv vs venv +------------------ + +Since Python ``3.3``, a subset of it has been integrated into the standard library under the `venv module `_. The ``venv`` module does not offer all features of this library, to name just a few more prominent: @@ -40,6 +45,9 @@ integrated into the standard library under the `venv module `_, - does not have as rich programmatic API (describe virtual environments without creating them). +Concept and purpose of virtualenv +--------------------------------- + The basic problem being addressed is one of dependencies and versions, and indirectly permissions. Imagine you have an application that needs version ``1`` of ``LibFoo``, but another application requires version ``2``. How can you use both these libraries? If you install everything into your host python (e.g. ``python3.8``) @@ -53,6 +61,31 @@ In all these cases, ``virtualenv`` can help you. It creates an environment that that doesn't share libraries with other virtualenv environments (and optionally doesn't access the globally installed libraries either). + +Compatibility +------------- + +With the release of virtualenv 20.22, April 2023, (`release note `__) target interpreters are now limited to Python v. 3.7+. + +Trying to use an earlier version will normally result in the target interpreter raising a syntax error. This virtualenv tool will then print some details about the exception and abort, ie no explicit warning about trying to use an outdated/incompatible version. It may look like this: + +.. code-block:: console + + $ virtualenv --discovery pyenv -p python3.6 foo + RuntimeError: failed to query /home/velle/.pyenv/versions/3.6.15/bin/python3.6 with code 1 err: ' File "/home/velle/.virtualenvs/toxrunner/lib/python3.12/site-packages/virtualenv/discovery/py_info.py", line 7 + from __future__ import annotations + ^ + SyntaxError: future feature annotations is not defined + + +In tox, even if the interpreter is installed and available, the message is (somewhat misleading): + +.. code-block:: console + + py36: skipped because could not find python interpreter with spec(s): py36 + + + Useful links ------------ diff --git a/docs/installation.rst b/docs/installation.rst index ab5820345..b82d1dd19 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -4,7 +4,7 @@ Installation via pipx -------- -:pypi:`virtualenv` is a CLI tool that needs a Python interpreter to run. If you already have a ``Python 3.5+`` +:pypi:`virtualenv` is a CLI tool that needs a Python interpreter to run. If you already have a ``Python 3.7+`` interpreter the best is to use :pypi:`pipx` to install virtualenv into an isolated environment. This has the added benefit that later you'll be able to upgrade virtualenv without affecting other parts of the system. @@ -58,11 +58,14 @@ with a python interpreter: The root level zipapp is always the current latest release. To get the last supported zipapp against a given python minor release use the link ``https://bootstrap.pypa.io/virtualenv/x.y/virtualenv.pyz``, e.g. for the last virtualenv -supporting Python 2.7 use -`https://bootstrap.pypa.io/virtualenv/2.7/virtualenv.pyz `_. +supporting Python 3.11 use +`https://bootstrap.pypa.io/virtualenv/3.11/virtualenv.pyz `_. If you are looking for past version of virtualenv.pyz they are available here: -https://github.com/pypa/get-virtualenv/blob//public//virtualenv.pyz?raw=true + +.. code-block:: console + + https://github.com/pypa/get-virtualenv/blob//public//virtualenv.pyz?raw=true latest unreleased ----------------- @@ -81,8 +84,8 @@ Python and OS Compatibility virtualenv works with the following Python interpreter implementations: -- `CPython `_ versions 3.7, 3.8, 3.9, 3.10, 3.11, 3.12 -- `PyPy `_ 3.7, 3.8, 3.9 +- `CPython `_: ``3.13 >= python_version >= 3.7`` +- `PyPy `_: ``3.10 >= python_version >= 3.7`` This means virtualenv works on the latest patch version of each of these minor versions. Previous patch versions are supported on a best effort approach. @@ -90,9 +93,13 @@ supported on a best effort approach. CPython is shipped in multiple forms, and each OS repackages it, often applying some customization along the way. Therefore we cannot say universally that we support all platforms, but rather specify some we test against. In case of ones not specified here the support is unknown, though likely will work. If you find some cases please open a feature -request on our issue tracker. Note, as of ``20.18.0`` we no longer support running under Python less than 3.7. Also, as -of ``20.22.0`` we no longer support creating environments for Python 2.7, 3.5 and 3.6. +request on our issue tracker. +Note: + +- as of ``20.27.0`` -- ``2024-10-17`` -- we no longer support running under Python ``<=3.7``, +- as of ``20.18.0`` -- ``2023-02-06`` -- we no longer support running under Python ``<=3.6``, +- as of ``20.22.0`` -- ``2023-04-19`` -- we no longer support creating environments for Python ``<=3.6``. Linux ~~~~~ @@ -107,19 +114,11 @@ macOS ~~~~~ In case of macOS we support: -- installations from `python.org `_ -- python versions installed via `brew `_ (both older python2.7 and python3) -- Python 3 part of XCode (Python framework - ``/Library/Frameworks/Python3.framework/``) -- Python 2 part of the OS (``/System/Library/Frameworks/Python.framework/Versions/``) +- installations from `python.org `_, +- python versions installed via `brew `_, +- Python 3 part of XCode (Python framework - ``/Library/Frameworks/Python3.framework/``). Windows ~~~~~~~ - Installations from `python.org `_ -- Windows Store Python - note only `version 3.7+ `_ - -Packaging variants -~~~~~~~~~~~~~~~~~~ -- Normal variant (file structure as comes from `python.org `_). -- We support CPython 2 system installations that do not contain the python files for the standard library if the - respective compiled files are present (e.g. only ``os.pyc``, not ``os.py``). This can be used by custom systems may - want to maximize available storage or obfuscate source code by removing ``.py`` files. +- Windows Store Python - note only `version 3.8+ `_ diff --git a/docs/render_cli.py b/docs/render_cli.py index af74b8b02..ad77fc6a2 100644 --- a/docs/render_cli.py +++ b/docs/render_cli.py @@ -16,7 +16,7 @@ class TableRow(NamedTuple): names: list[str] default: str choices: set[str] - help: str # noqa: A003 + help: str class TextAsDefault(NamedTuple): @@ -84,12 +84,14 @@ def a(*args, **kwargs): return True if key == "creator": if name == "venv": - from virtualenv.create.via_global_ref.venv import ViaGlobalRefMeta + from virtualenv.create.via_global_ref.venv import ViaGlobalRefMeta # noqa: PLC0415 meta = ViaGlobalRefMeta() meta.symlink_error = None return meta - from virtualenv.create.via_global_ref.builtin.via_global_self_do import BuiltinViaGlobalRefMeta + from virtualenv.create.via_global_ref.builtin.via_global_self_do import ( # noqa: PLC0415 + BuiltinViaGlobalRefMeta, + ) meta = BuiltinViaGlobalRefMeta() meta.symlink_error = None @@ -183,7 +185,7 @@ def _get_targeted_names(self, row): @staticmethod def _get_help_text(row): name = row.names[0] - content = row.help[: row.help.index("(") - 1] if name in ("--creator",) else row.help + content = row.help[: row.help.index("(") - 1] if name == "--creator" else row.help help_body = n.paragraph("", "", n.Text(content)) if row.choices is not None: help_body += n.Text("; choice of: ") diff --git a/docs/user_guide.rst b/docs/user_guide.rst index b5534c860..82cc235b1 100644 --- a/docs/user_guide.rst +++ b/docs/user_guide.rst @@ -30,8 +30,18 @@ Virtualenv has one basic command: virtualenv venv +.. note:: + + When creating a virtual environment, it's recommended to use a specific Python version, for example, by invoking + virtualenv with ``python3.10 -m virtualenv venv``. If you use a generic command like ``python3 -m virtualenv venv``, + the created environment will be linked to ``/usr/bin/python3``. This can be problematic because when a new Python + version is installed on the system, the ``/usr/bin/python3`` symlink will likely be updated to point to the new + version. This will cause the virtual environment to inadvertently use the new Python version, which is often not the + desired behavior. Using a specific version ensures that the virtual environment is tied to that exact version, + providing stability and predictability. + This will create a python virtual environment of the same version as virtualenv, installed into the subdirectory -``venv``. The command line tool has quite a few of flags that modify the tool's behaviour, for a +``venv``. The command line tool has quite a few of flags that modify the tool's behavior, for a full list make sure to check out :ref:`cli_flags`. The tool works in two phases: @@ -86,13 +96,14 @@ format is either: - the python implementation is all alphabetic characters (``python`` means any implementation, and if is missing it defaults to ``python``), - - the version is a dot separated version number, + - the version is a dot separated version number optionally followed by ``t`` for free-threading, - the architecture is either ``-64`` or ``-32`` (missing means ``any``). For example: - ``python3.8.1`` means any python implementation having the version ``3.8.1``, - ``3`` means any python implementation having the major version ``3``, + - ``3.13t`` means any python implementation having the version ``3.13`` with free threading, - ``cpython3`` means a ``CPython`` implementation having the version ``3``, - ``pypy2`` means a python interpreter with the ``PyPy`` implementation and major version ``2``. @@ -101,6 +112,8 @@ format is either: - If we're on Windows look into the Windows registry, and check if we see any registered Python implementations that match the specification. This is in line with expectation laid out inside `PEP-514 `_ + - If `uv-managed `_ Python installations are available, use the + first one that matches the specification. - Try to discover a matching python executable within the folders enumerated on the ``PATH`` environment variable. In this case we'll try to find an executable that has a name roughly similar to the specification (for exact logic, please see the implementation code). @@ -139,8 +152,7 @@ Seeders ------- These will install for you some seed packages (one or more of: :pypi:`pip`, :pypi:`setuptools`, :pypi:`wheel`) that enables you to install additional python packages into the created virtual environment (by invoking pip). Installing -:pypi:`setuptools` and :pypi:`wheel` is disabled by default on Python 3.12+ environments. There are two -main seed mechanisms available: +:pypi:`setuptools` is disabled by default on Python 3.12+ environments. :pypi:`wheel` is only installed on Python 3.8, by default. There are two main seed mechanisms available: - ``pip`` - this method uses the bundled pip with virtualenv to install the seed packages (note, a new child process needs to be created to do this, which can be expensive especially on Windows). @@ -182,7 +194,7 @@ package. These wheels may be acquired from multiple locations as follows: not start using a new embedded versions half way through. - The automatic behaviour might be disabled via the :option:`no-periodic-update` configuration flag/option. To acquire + The automatic behavior might be disabled via the :option:`no-periodic-update` configuration flag/option. To acquire the release date of a package virtualenv will perform the following: - lookup ``https://pypi.org/pypi//json`` (primary truth source), @@ -243,7 +255,7 @@ executables, rather than relying on your shell to resolve them to your virtual e Activator scripts also modify your shell prompt to indicate which environment is currently active, by prepending the environment name (or the name specified by ``--prompt`` when initially creating the environment) in brackets, like -``(venv)``. You can disable this behaviour by setting the environment variable ``VIRTUAL_ENV_DISABLE_PROMPT`` to any +``(venv)``. You can disable this behavior by setting the environment variable ``VIRTUAL_ENV_DISABLE_PROMPT`` to any value. You can also get the environment name via the environment variable ``VIRTUAL_ENV_PROMPT`` if you want to customize your prompt, for example. diff --git a/pyproject.toml b/pyproject.toml index 8df35ef57..f879b4034 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,8 +15,10 @@ keywords = [ "virtual", ] license = "MIT" -maintainers = [{ name = "Bernat Gabor", email = "gaborjbernat@gmail.com" }] -requires-python = ">=3.7" +maintainers = [ + { name = "Bernat Gabor", email = "gaborjbernat@gmail.com" }, +] +requires-python = ">=3.8" classifiers = [ "Development Status :: 5 - Production/Stable", "Intended Audience :: Developers", @@ -25,12 +27,12 @@ classifiers = [ "Operating System :: Microsoft :: Windows", "Operating System :: POSIX", "Programming Language :: Python :: 3 :: Only", - "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", "Topic :: Software Development :: Libraries", @@ -41,15 +43,16 @@ dynamic = [ "version", ] dependencies = [ - "distlib<1,>=0.3.7", - "filelock<4,>=3.12.2", - 'importlib-metadata>=6.6; python_version < "3.8"', - "platformdirs<4,>=3.9.1", + "distlib>=0.3.7,<1", + "filelock>=3.12.2,<4", + "importlib-metadata>=6.6; python_version<'3.8'", + "platformdirs>=3.9.1,<5", + "typing-extensions>=4.13.2; python_version<'3.11'", ] optional-dependencies.docs = [ "furo>=2023.7.26", "proselint>=0.13", - "sphinx>=7.1.2", + "sphinx>=7.1.2,!=7.3", "sphinx-argparse>=0.4", "sphinxcontrib-towncrier>=0.2.1a0", "towncrier>=23.6", @@ -62,94 +65,136 @@ optional-dependencies.test = [ "packaging>=23.1", "pytest>=7.4", "pytest-env>=0.8.2", - 'pytest-freezer>=0.4.8; platform_python_implementation == "PyPy"', + "pytest-freezer>=0.4.8; platform_python_implementation=='PyPy' or platform_python_implementation=='GraalVM' or (platform_python_implementation=='CPython' and sys_platform=='win32' and python_version>='3.13')", "pytest-mock>=3.11.1", "pytest-randomly>=3.12", "pytest-timeout>=2.1", "setuptools>=68", - 'time-machine>=2.10; platform_python_implementation == "CPython"', + "time-machine>=2.10; platform_python_implementation=='CPython'", ] urls.Documentation = "https://virtualenv.pypa.io" urls.Homepage = "https://github.com/pypa/virtualenv" urls.Source = "https://github.com/pypa/virtualenv" urls.Tracker = "https://github.com/pypa/virtualenv/issues" scripts.virtualenv = "virtualenv.__main__:run_with_catch" -[project.entry-points."virtualenv.activate"] -bash = "virtualenv.activation.bash:BashActivator" -batch = "virtualenv.activation.batch:BatchActivator" -cshell = "virtualenv.activation.cshell:CShellActivator" -fish = "virtualenv.activation.fish:FishActivator" -nushell = "virtualenv.activation.nushell:NushellActivator" -powershell = "virtualenv.activation.powershell:PowerShellActivator" -python = "virtualenv.activation.python:PythonActivator" -[project.entry-points."virtualenv.create"] -cpython3-mac-framework = "virtualenv.create.via_global_ref.builtin.cpython.mac_os:CPython3macOsFramework" -cpython3-posix = "virtualenv.create.via_global_ref.builtin.cpython.cpython3:CPython3Posix" -cpython3-win = "virtualenv.create.via_global_ref.builtin.cpython.cpython3:CPython3Windows" -pypy3-posix = "virtualenv.create.via_global_ref.builtin.pypy.pypy3:PyPy3Posix" -pypy3-win = "virtualenv.create.via_global_ref.builtin.pypy.pypy3:Pypy3Windows" -venv = "virtualenv.create.via_global_ref.venv:Venv" -[project.entry-points."virtualenv.discovery"] -builtin = "virtualenv.discovery.builtin:Builtin" -[project.entry-points."virtualenv.seed"] -app-data = "virtualenv.seed.embed.via_app_data.via_app_data:FromAppData" -pip = "virtualenv.seed.embed.pip_invoke:PipInvoke" +entry-points."virtualenv.activate".bash = "virtualenv.activation.bash:BashActivator" +entry-points."virtualenv.activate".batch = "virtualenv.activation.batch:BatchActivator" +entry-points."virtualenv.activate".cshell = "virtualenv.activation.cshell:CShellActivator" +entry-points."virtualenv.activate".fish = "virtualenv.activation.fish:FishActivator" +entry-points."virtualenv.activate".nushell = "virtualenv.activation.nushell:NushellActivator" +entry-points."virtualenv.activate".powershell = "virtualenv.activation.powershell:PowerShellActivator" +entry-points."virtualenv.activate".python = "virtualenv.activation.python:PythonActivator" +entry-points."virtualenv.create".cpython3-mac-brew = "virtualenv.create.via_global_ref.builtin.cpython.mac_os:CPython3macOsBrew" +entry-points."virtualenv.create".cpython3-mac-framework = "virtualenv.create.via_global_ref.builtin.cpython.mac_os:CPython3macOsFramework" +entry-points."virtualenv.create".cpython3-posix = "virtualenv.create.via_global_ref.builtin.cpython.cpython3:CPython3Posix" +entry-points."virtualenv.create".cpython3-win = "virtualenv.create.via_global_ref.builtin.cpython.cpython3:CPython3Windows" +entry-points."virtualenv.create".graalpy-posix = "virtualenv.create.via_global_ref.builtin.graalpy:GraalPyPosix" +entry-points."virtualenv.create".graalpy-win = "virtualenv.create.via_global_ref.builtin.graalpy:GraalPyWindows" +entry-points."virtualenv.create".pypy3-posix = "virtualenv.create.via_global_ref.builtin.pypy.pypy3:PyPy3Posix" +entry-points."virtualenv.create".pypy3-win = "virtualenv.create.via_global_ref.builtin.pypy.pypy3:Pypy3Windows" +entry-points."virtualenv.create".venv = "virtualenv.create.via_global_ref.venv:Venv" +entry-points."virtualenv.discovery".builtin = "virtualenv.discovery.builtin:Builtin" +entry-points."virtualenv.seed".app-data = "virtualenv.seed.embed.via_app_data.via_app_data:FromAppData" +entry-points."virtualenv.seed".pip = "virtualenv.seed.embed.pip_invoke:PipInvoke" [tool.hatch] build.hooks.vcs.version-file = "src/virtualenv/version.py" -build.targets.sdist.include = ["/src", "/tests", "/tasks", "/tox.ini"] +build.targets.sdist.include = [ + "/src", + "/tests", + "/tasks", + "/tox.ini", +] version.source = "vcs" -[tool.black] -line-length = 120 - [tool.ruff] -select = ["ALL"] line-length = 120 -target-version = "py37" -isort = {known-first-party = ["virtualenv"], required-imports = ["from __future__ import annotations"]} -ignore = [ - "ANN", # no type checking added yet - "D10", # no docstrings - "D40", # no imperative mode for docstrings - "PTH", # no pathlib, <=39 has problems on Windows with absolute/resolve, can revisit once we no longer need 39 - "INP001", # ignore implicit namespace packages - "D203", # `one-blank-line-before-class` (D203) and `no-blank-line-before-class` (D211) are incompatible - "D212", # `multi-line-summary-first-line` (D212) and `multi-line-summary-second-line` (D213) are incompatible - "S104", # Possible binding to all interface -] -[tool.ruff.per-file-ignores] -"tests/**/*.py" = [ - "S101", # asserts allowed in tests... - "FBT", # don"t care about booleans as positional arguments in tests - "INP001", # no implicit namespace - "D", # don"t care about documentation in tests - "S603", # `subprocess` call: check for execution of untrusted input - "PLR2004", # Magic value used in comparison, consider replacing with a constant variable +format.preview = true +format.docstring-code-line-length = 100 +format.docstring-code-format = true +lint.select = [ + "ALL", +] +lint.ignore = [ + "ANN", # no type checking added yet + "COM812", # conflict with formatter + "CPY", # No copyright header + "D10", # no docstrings + "D40", # no imperative mode for docstrings + "D203", # `one-blank-line-before-class` (D203) and `no-blank-line-before-class` (D211) are incompatible + "D212", # `multi-line-summary-first-line` (D212) and `multi-line-summary-second-line` (D213) are incompatible + "DOC", # no restructuredtext support + "INP001", # ignore implicit namespace packages + "ISC001", # conflict with formatter + "PLR0914", # Too many local variables + "PLR0917", # Too many positional arguments + "PLR6301", # Method could be a function, class method, or static method + "PLW1510", # no need for check for subprocess + "PTH", # no pathlib, <=39 has problems on Windows with absolute/resolve, can revisit once we no longer need 39 + "S104", # Possible binding to all interfaces + "S404", # Using subprocess is alright + "S603", # subprocess calls are fine +] +lint.per-file-ignores."src/virtualenv/activation/python/activate_this.py" = [ + "F821", # ignore undefined template string placeholders +] +lint.per-file-ignores."tests/**/*.py" = [ + "D", # don't care about documentation in tests + "FBT", # don't care about booleans as positional arguments in tests + "INP001", # no implicit namespace + "PLC2701", # Private import + "PLR2004", # Magic value used in comparison, consider replacing with a constant variable + "S101", # asserts allowed in tests + "S603", # `subprocess` call: check for execution of untrusted input ] +lint.isort = { known-first-party = [ + "virtualenv", +], required-imports = [ + "from __future__ import annotations", +] } +lint.preview = true + +[tool.codespell] +builtin = "clear,usage,en-GB_to_en-US" +count = true + +[tool.pyproject-fmt] +max_supported_python = "3.13" [tool.pytest.ini_options] -markers = ["slow"] +markers = [ + "slow", +] timeout = 600 addopts = "--showlocals --no-success-flaky-report" -env = ["PYTHONIOENCODING=utf-8"] +env = [ + "PYTHONIOENCODING=utf-8", +] [tool.coverage] html.show_contexts = true html.skip_covered = false report.omit = [ - # site.py is ran before the coverage can be enabled, no way to measure coverage on this + # site.py is run before the coverage can be enabled, no way to measure coverage on this "**/src/virtualenv/create/via_global_ref/builtin/python2/site.py", "**/src/virtualenv/create/via_global_ref/_virtualenv.py", "**/src/virtualenv/activation/python/activate_this.py", "**/src/virtualenv/seed/wheels/embed/pip-*.whl/pip/**", ] -paths.source = ["src", "**/site-packages"] +paths.source = [ + "src", + "**/site-packages", +] report.fail_under = 76 -run.source = ["${_COVERAGE_SRC}", "tests"] +run.source = [ + "${_COVERAGE_SRC}", + "tests", +] run.dynamic_context = "test_function" run.parallel = true -run.plugins = ["covdefaults"] +run.plugins = [ + "covdefaults", +] run.relative_files = true [tool.towncrier] diff --git a/src/virtualenv/__main__.py b/src/virtualenv/__main__.py index ee7341b6e..49f59da38 100644 --- a/src/virtualenv/__main__.py +++ b/src/virtualenv/__main__.py @@ -1,27 +1,39 @@ from __future__ import annotations +import errno import logging import os import sys from timeit import default_timer +LOGGER = logging.getLogger(__name__) + def run(args=None, options=None, env=None): env = os.environ if env is None else env start = default_timer() - from virtualenv.run import cli_run - from virtualenv.util.error import ProcessCallFailedError + from virtualenv.run import cli_run # noqa: PLC0415 + from virtualenv.util.error import ProcessCallFailedError # noqa: PLC0415 if args is None: args = sys.argv[1:] try: session = cli_run(args, options, env) - logging.warning(LogSession(session, start)) + LOGGER.warning(LogSession(session, start)) except ProcessCallFailedError as exception: print(f"subprocess call failed for {exception.cmd} with code {exception.code}") # noqa: T201 print(exception.out, file=sys.stdout, end="") # noqa: T201 print(exception.err, file=sys.stderr, end="") # noqa: T201 - raise SystemExit(exception.code) # noqa: TRY200, B904 + raise SystemExit(exception.code) # noqa: B904 + except OSError as exception: + if exception.errno == errno.EMFILE: + print( # noqa: T201 + "OSError: [Errno 24] Too many open files. You may need to increase your OS open files limit.\n" + " On macOS/Linux, try 'ulimit -n 2048'.\n" + " For Windows, this is not a common issue, but you can try to close some applications.", + file=sys.stderr, + ) + raise class LogSession: @@ -48,22 +60,23 @@ def __str__(self) -> str: def run_with_catch(args=None, env=None): - from virtualenv.config.cli.parser import VirtualEnvOptions + from virtualenv.config.cli.parser import VirtualEnvOptions # noqa: PLC0415 env = os.environ if env is None else env options = VirtualEnvOptions() try: run(args, options, env) - except (KeyboardInterrupt, SystemExit, Exception) as exception: + except (KeyboardInterrupt, SystemExit, Exception) as exception: # noqa: BLE001 try: if getattr(options, "with_traceback", False): raise if not (isinstance(exception, SystemExit) and exception.code == 0): - logging.error("%s: %s", type(exception).__name__, exception) # noqa: TRY400 + LOGGER.error("%s: %s", type(exception).__name__, exception) # noqa: TRY400 code = exception.code if isinstance(exception, SystemExit) else 1 sys.exit(code) finally: - logging.shutdown() # force flush of log messages before the trace is printed + for handler in LOGGER.handlers: # force flush of log messages before the trace is printed + handler.flush() if __name__ == "__main__": # pragma: no cov diff --git a/src/virtualenv/activation/__init__.py b/src/virtualenv/activation/__init__.py index f6cf75694..5aa4a9d76 100644 --- a/src/virtualenv/activation/__init__.py +++ b/src/virtualenv/activation/__init__.py @@ -10,10 +10,10 @@ __all__ = [ "BashActivator", - "PowerShellActivator", - "CShellActivator", - "PythonActivator", "BatchActivator", + "CShellActivator", "FishActivator", "NushellActivator", + "PowerShellActivator", + "PythonActivator", ] diff --git a/src/virtualenv/activation/activator.py b/src/virtualenv/activation/activator.py index 7d41e8ae5..dd404b47c 100644 --- a/src/virtualenv/activation/activator.py +++ b/src/virtualenv/activation/activator.py @@ -1,10 +1,10 @@ from __future__ import annotations import os -from abc import ABCMeta, abstractmethod +from abc import ABC, abstractmethod -class Activator(metaclass=ABCMeta): +class Activator(ABC): """Generates activate script for the virtual environment.""" def __init__(self, options) -> None: diff --git a/src/virtualenv/activation/bash/__init__.py b/src/virtualenv/activation/bash/__init__.py index 5e095ddf0..4f160744f 100644 --- a/src/virtualenv/activation/bash/__init__.py +++ b/src/virtualenv/activation/bash/__init__.py @@ -12,6 +12,14 @@ def templates(self): def as_name(self, template): return Path(template).stem + def replacements(self, creator, dest): + data = super().replacements(creator, dest) + data.update({ + "__TCL_LIBRARY__": getattr(creator.interpreter, "tcl_lib", None) or "", + "__TK_LIBRARY__": getattr(creator.interpreter, "tk_lib", None) or "", + }) + return data + __all__ = [ "BashActivator", diff --git a/src/virtualenv/activation/bash/activate.sh b/src/virtualenv/activation/bash/activate.sh index b06e3fd33..b54358e9e 100644 --- a/src/virtualenv/activation/bash/activate.sh +++ b/src/virtualenv/activation/bash/activate.sh @@ -23,6 +23,17 @@ deactivate () { unset _OLD_VIRTUAL_PYTHONHOME fi + if ! [ -z "${_OLD_VIRTUAL_TCL_LIBRARY+_}" ]; then + TCL_LIBRARY="$_OLD_VIRTUAL_TCL_LIBRARY" + export TCL_LIBRARY + unset _OLD_VIRTUAL_TCL_LIBRARY + fi + if ! [ -z "${_OLD_VIRTUAL_TK_LIBRARY+_}" ]; then + TK_LIBRARY="$_OLD_VIRTUAL_TK_LIBRARY" + export TK_LIBRARY + unset _OLD_VIRTUAL_TK_LIBRARY + fi + # The hash command must be called to get it to forget past # commands. Without forgetting past commands the $PATH changes # we made may not be respected @@ -45,18 +56,18 @@ deactivate () { # unset irrelevant variables deactivate nondestructive -VIRTUAL_ENV='__VIRTUAL_ENV__' +VIRTUAL_ENV=__VIRTUAL_ENV__ if ([ "$OSTYPE" = "cygwin" ] || [ "$OSTYPE" = "msys" ]) && $(command -v cygpath &> /dev/null) ; then VIRTUAL_ENV=$(cygpath -u "$VIRTUAL_ENV") fi export VIRTUAL_ENV _OLD_VIRTUAL_PATH="$PATH" -PATH="$VIRTUAL_ENV/__BIN_NAME__:$PATH" +PATH="$VIRTUAL_ENV/"__BIN_NAME__":$PATH" export PATH -if [ "x__VIRTUAL_PROMPT__" != x ] ; then - VIRTUAL_ENV_PROMPT="__VIRTUAL_PROMPT__" +if [ "x"__VIRTUAL_PROMPT__ != x ] ; then + VIRTUAL_ENV_PROMPT=__VIRTUAL_PROMPT__ else VIRTUAL_ENV_PROMPT=$(basename "$VIRTUAL_ENV") fi @@ -68,6 +79,22 @@ if ! [ -z "${PYTHONHOME+_}" ] ; then unset PYTHONHOME fi +if [ __TCL_LIBRARY__ != "" ]; then + if ! [ -z "${TCL_LIBRARY+_}" ] ; then + _OLD_VIRTUAL_TCL_LIBRARY="$TCL_LIBRARY" + fi + TCL_LIBRARY=__TCL_LIBRARY__ + export TCL_LIBRARY +fi + +if [ __TK_LIBRARY__ != "" ]; then + if ! [ -z "${TK_LIBRARY+_}" ] ; then + _OLD_VIRTUAL_TK_LIBRARY="$TK_LIBRARY" + fi + TK_LIBRARY=__TK_LIBRARY__ + export TK_LIBRARY +fi + if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then _OLD_VIRTUAL_PS1="${PS1-}" PS1="(${VIRTUAL_ENV_PROMPT}) ${PS1-}" @@ -84,4 +111,4 @@ pydoc () { # The hash command must be called to get it to forget past # commands. Without forgetting past commands the $PATH changes # we made may not be respected -hash -r 2>/dev/null +hash -r 2>/dev/null || true diff --git a/src/virtualenv/activation/batch/__init__.py b/src/virtualenv/activation/batch/__init__.py index a6d58ebb4..3d74ba835 100644 --- a/src/virtualenv/activation/batch/__init__.py +++ b/src/virtualenv/activation/batch/__init__.py @@ -15,6 +15,10 @@ def templates(self): yield "deactivate.bat" yield "pydoc.bat" + @staticmethod + def quote(string): + return string + def instantiate_template(self, replacements, template, creator): # ensure the text has all newlines as \r\n - required by batch base = super().instantiate_template(replacements, template, creator) diff --git a/src/virtualenv/activation/batch/activate.bat b/src/virtualenv/activation/batch/activate.bat index 0bad4c1e6..62f393c80 100644 --- a/src/virtualenv/activation/batch/activate.bat +++ b/src/virtualenv/activation/batch/activate.bat @@ -1,3 +1,10 @@ +@REM This file is UTF-8 encoded, so we need to update the current code page while executing it +@for /f "tokens=2 delims=:." %%a in ('"%SystemRoot%\System32\chcp.com"') do @set _OLD_CODEPAGE=%%a + +@if defined _OLD_CODEPAGE ( + "%SystemRoot%\System32\chcp.com" 65001 > nul +) + @set "VIRTUAL_ENV=__VIRTUAL_ENV__" @set "VIRTUAL_ENV_PROMPT=__VIRTUAL_PROMPT__" @@ -26,6 +33,12 @@ @set PYTHONHOME= +@if defined TCL_LIBRARY @set "_OLD_VIRTUAL_TCL_LIBRARY=%TCL_LIBRARY%" +@if NOT "__TCL_LIBRARY__"=="" @set "TCL_LIBRARY=__TCL_LIBRARY__" + +@if defined TK_LIBRARY @set "_OLD_VIRTUAL_TK_LIBRARY=%TK_LIBRARY%" +@if NOT "__TK_LIBRARY__"=="" @set "TK_LIBRARY=__TK_LIBRARY__" + @REM if defined _OLD_VIRTUAL_PATH ( @if not defined _OLD_VIRTUAL_PATH @goto ENDIFVPATH1 @set "PATH=%_OLD_VIRTUAL_PATH%" @@ -36,3 +49,8 @@ :ENDIFVPATH2 @set "PATH=%VIRTUAL_ENV%\__BIN_NAME__;%PATH%" + +@if defined _OLD_CODEPAGE ( + "%SystemRoot%\System32\chcp.com" %_OLD_CODEPAGE% > nul + @set _OLD_CODEPAGE= +) diff --git a/src/virtualenv/activation/batch/deactivate.bat b/src/virtualenv/activation/batch/deactivate.bat index 8939c6c0d..7a12d47ed 100644 --- a/src/virtualenv/activation/batch/deactivate.bat +++ b/src/virtualenv/activation/batch/deactivate.bat @@ -12,6 +12,14 @@ @set _OLD_VIRTUAL_PYTHONHOME= :ENDIFVHOME +@if defined _OLD_VIRTUAL_TCL_LIBRARY @set "TCL_LIBRARY=%_OLD_VIRTUAL_TCL_LIBRARY%" +@if not defined _OLD_VIRTUAL_TCL_LIBRARY @set TCL_LIBRARY= +@set _OLD_VIRTUAL_TCL_LIBRARY= + +@if defined _OLD_VIRTUAL_TK_LIBRARY @set "TK_LIBRARY=%_OLD_VIRTUAL_TK_LIBRARY%" +@if not defined _OLD_VIRTUAL_TK_LIBRARY @set TK_LIBRARY= +@set _OLD_VIRTUAL_TK_LIBRARY= + @if not defined _OLD_VIRTUAL_PATH @goto ENDIFVPATH @set "PATH=%_OLD_VIRTUAL_PATH%" @set _OLD_VIRTUAL_PATH= diff --git a/src/virtualenv/activation/cshell/activate.csh b/src/virtualenv/activation/cshell/activate.csh index f0c9cca96..5c02616d7 100644 --- a/src/virtualenv/activation/cshell/activate.csh +++ b/src/virtualenv/activation/cshell/activate.csh @@ -5,20 +5,32 @@ set newline='\ ' -alias deactivate 'test $?_OLD_VIRTUAL_PATH != 0 && setenv PATH "$_OLD_VIRTUAL_PATH:q" && unset _OLD_VIRTUAL_PATH; rehash; test $?_OLD_VIRTUAL_PROMPT != 0 && set prompt="$_OLD_VIRTUAL_PROMPT:q" && unset _OLD_VIRTUAL_PROMPT; unsetenv VIRTUAL_ENV; unsetenv VIRTUAL_ENV_PROMPT; test "\!:*" != "nondestructive" && unalias deactivate && unalias pydoc' +alias deactivate 'test $?_OLD_VIRTUAL_PATH != 0 && setenv PATH "$_OLD_VIRTUAL_PATH:q" && unset _OLD_VIRTUAL_PATH; rehash; test $?_OLD_VIRTUAL_TCL_LIBRARY != 0 && setenv TCL_LIBRARY "$_OLD_VIRTUAL_TCL_LIBRARY:q" && unset _OLD_VIRTUAL_TCL_LIBRARY || unsetenv TCL_LIBRARY; test $?_OLD_VIRTUAL_TK_LIBRARY != 0 && setenv TK_LIBRARY "$_OLD_VIRTUAL_TK_LIBRARY:q" && unset _OLD_VIRTUAL_TK_LIBRARY || unsetenv TK_LIBRARY; test $?_OLD_VIRTUAL_PROMPT != 0 && set prompt="$_OLD_VIRTUAL_PROMPT:q" && unset _OLD_VIRTUAL_PROMPT; unsetenv VIRTUAL_ENV; unsetenv VIRTUAL_ENV_PROMPT; test "\!:*" != "nondestructive" && unalias deactivate && unalias pydoc' # Unset irrelevant variables. deactivate nondestructive -setenv VIRTUAL_ENV '__VIRTUAL_ENV__' +setenv VIRTUAL_ENV __VIRTUAL_ENV__ set _OLD_VIRTUAL_PATH="$PATH:q" -setenv PATH "$VIRTUAL_ENV:q/__BIN_NAME__:$PATH:q" +setenv PATH "$VIRTUAL_ENV:q/"__BIN_NAME__":$PATH:q" +if (__TCL_LIBRARY__ != "") then + if ($?TCL_LIBRARY) then + set _OLD_VIRTUAL_TCL_LIBRARY="$TCL_LIBRARY" + endif + setenv TCL_LIBRARY __TCL_LIBRARY__ +endif +if (__TK_LIBRARY__ != "") then + if ($?TK_LIBRARY) then + set _OLD_VIRTUAL_TK_LIBRARY="$TK_LIBRARY" + endif + setenv TK_LIBRARY __TK_LIBRARY__ +endif -if ('__VIRTUAL_PROMPT__' != "") then - setenv VIRTUAL_ENV_PROMPT '__VIRTUAL_PROMPT__' +if (__VIRTUAL_PROMPT__ != "") then + setenv VIRTUAL_ENV_PROMPT __VIRTUAL_PROMPT__ else setenv VIRTUAL_ENV_PROMPT "$VIRTUAL_ENV:t:q" endif diff --git a/src/virtualenv/activation/fish/__init__.py b/src/virtualenv/activation/fish/__init__.py index 57f790f47..26263566e 100644 --- a/src/virtualenv/activation/fish/__init__.py +++ b/src/virtualenv/activation/fish/__init__.py @@ -7,6 +7,14 @@ class FishActivator(ViaTemplateActivator): def templates(self): yield "activate.fish" + def replacements(self, creator, dest): + data = super().replacements(creator, dest) + data.update({ + "__TCL_LIBRARY__": getattr(creator.interpreter, "tcl_lib", None) or "", + "__TK_LIBRARY__": getattr(creator.interpreter, "tk_lib", None) or "", + }) + return data + __all__ = [ "FishActivator", diff --git a/src/virtualenv/activation/fish/activate.fish b/src/virtualenv/activation/fish/activate.fish index fcedde465..c9d174997 100644 --- a/src/virtualenv/activation/fish/activate.fish +++ b/src/virtualenv/activation/fish/activate.fish @@ -18,7 +18,7 @@ function deactivate -d 'Exit virtualenv mode and return to the normal environmen # reset old environment variables if test -n "$_OLD_VIRTUAL_PATH" # https://github.com/fish-shell/fish-shell/issues/436 altered PATH handling - if test (echo $FISH_VERSION | head -c 1) -lt 3 + if test (string sub -s 1 -l 1 $FISH_VERSION) -lt 3 set -gx PATH (_fishify_path "$_OLD_VIRTUAL_PATH") else set -gx PATH $_OLD_VIRTUAL_PATH @@ -26,6 +26,23 @@ function deactivate -d 'Exit virtualenv mode and return to the normal environmen set -e _OLD_VIRTUAL_PATH end + if test -n __TCL_LIBRARY__ + if test -n "$_OLD_VIRTUAL_TCL_LIBRARY"; + set -gx TCL_LIBRARY "$_OLD_VIRTUAL_TCL_LIBRARY"; + set -e _OLD_VIRTUAL_TCL_LIBRARY; + else; + set -e TCL_LIBRARY; + end + end + if test -n __TK_LIBRARY__ + if test -n "$_OLD_VIRTUAL_TK_LIBRARY"; + set -gx TK_LIBRARY "$_OLD_VIRTUAL_TK_LIBRARY"; + set -e _OLD_VIRTUAL_TK_LIBRARY; + else; + set -e TK_LIBRARY; + end + end + if test -n "$_OLD_VIRTUAL_PYTHONHOME" set -gx PYTHONHOME "$_OLD_VIRTUAL_PYTHONHOME" set -e _OLD_VIRTUAL_PYTHONHOME @@ -58,20 +75,33 @@ end # Unset irrelevant variables. deactivate nondestructive -set -gx VIRTUAL_ENV '__VIRTUAL_ENV__' +set -gx VIRTUAL_ENV __VIRTUAL_ENV__ # https://github.com/fish-shell/fish-shell/issues/436 altered PATH handling -if test (echo $FISH_VERSION | head -c 1) -lt 3 - set -gx _OLD_VIRTUAL_PATH (_bashify_path $PATH) +if test (string sub -s 1 -l 1 $FISH_VERSION) -lt 3 + set -gx _OLD_VIRTUAL_PATH (_bashify_path $PATH) else set -gx _OLD_VIRTUAL_PATH $PATH end -set -gx PATH "$VIRTUAL_ENV"'/__BIN_NAME__' $PATH +set -gx PATH "$VIRTUAL_ENV"'/'__BIN_NAME__ $PATH + +if test -n __TCL_LIBRARY__ + if set -q TCL_LIBRARY; + set -gx _OLD_VIRTUAL_TCL_LIBRARY $TCL_LIBRARY; + end + set -gx TCL_LIBRARY '__TCL_LIBRARY__' +end +if test -n __TK_LIBRARY__ + if set -q TK_LIBRARY; + set -gx _OLD_VIRTUAL_TK_LIBRARY $TK_LIBRARY; + end + set -gx TK_LIBRARY '__TK_LIBRARY__' +end # Prompt override provided? # If not, just use the environment name. -if test -n '__VIRTUAL_PROMPT__' - set -gx VIRTUAL_ENV_PROMPT '__VIRTUAL_PROMPT__' +if test -n __VIRTUAL_PROMPT__ + set -gx VIRTUAL_ENV_PROMPT __VIRTUAL_PROMPT__ else set -gx VIRTUAL_ENV_PROMPT (basename "$VIRTUAL_ENV") end diff --git a/src/virtualenv/activation/nushell/__init__.py b/src/virtualenv/activation/nushell/__init__.py index 68cd4a3b4..d3b312497 100644 --- a/src/virtualenv/activation/nushell/__init__.py +++ b/src/virtualenv/activation/nushell/__init__.py @@ -7,12 +7,35 @@ class NushellActivator(ViaTemplateActivator): def templates(self): yield "activate.nu" + @staticmethod + def quote(string): + """ + Nushell supports raw strings like: r###'this is a string'###. + + https://github.com/nushell/nushell.github.io/blob/main/book/working_with_strings.md + + This method finds the maximum continuous sharps in the string and then + quote it with an extra sharp. + """ + max_sharps = 0 + current_sharps = 0 + for char in string: + if char == "#": + current_sharps += 1 + max_sharps = max(current_sharps, max_sharps) + else: + current_sharps = 0 + wrapping = "#" * (max_sharps + 1) + return f"r{wrapping}'{string}'{wrapping}" + def replacements(self, creator, dest_folder): # noqa: ARG002 return { "__VIRTUAL_PROMPT__": "" if self.flag_prompt is None else self.flag_prompt, "__VIRTUAL_ENV__": str(creator.dest), "__VIRTUAL_NAME__": creator.env_name, "__BIN_NAME__": str(creator.bin_dir.relative_to(creator.dest)), + "__TCL_LIBRARY__": getattr(creator.interpreter, "tcl_lib", None) or "", + "__TK_LIBRARY__": getattr(creator.interpreter, "tk_lib", None) or "", } diff --git a/src/virtualenv/activation/nushell/activate.nu b/src/virtualenv/activation/nushell/activate.nu index 19d4fa1d7..b48fdd03f 100644 --- a/src/virtualenv/activation/nushell/activate.nu +++ b/src/virtualenv/activation/nushell/activate.nu @@ -1,94 +1,85 @@ -# virtualenv activation module -# Activate with `overlay use activate.nu` -# Deactivate with `deactivate`, as usual +# virtualenv activation module: +# - Activate with `overlay use activate.nu` +# - Deactivate with `deactivate`, as usual # -# To customize the overlay name, you can call `overlay use activate.nu as foo`, -# but then simply `deactivate` won't work because it is just an alias to hide -# the "activate" overlay. You'd need to call `overlay hide foo` manually. +# To customize the overlay name, you can call `overlay use activate.nu as foo`, but then simply `deactivate` won't work +# because it is just an alias to hide the "activate" overlay. You'd need to call `overlay hide foo` manually. + +module warning { + export-env { + const file = path self + error make -u { + msg: $"`($file | path basename)` is meant to be used with `overlay use`, not `source`" + } + } + +} + +use warning export-env { + + let nu_ver = (version | get version | split row '.' | take 2 | each { into int }) + if $nu_ver.0 == 0 and $nu_ver.1 < 106 { + error make { + msg: 'virtualenv Nushell activation requires Nushell 0.106 or greater.' + } + } + def is-string [x] { ($x | describe) == 'string' } def has-env [...names] { - $names | each {|n| - $n in $env - } | all {|i| $i == true} + $names | each {|n| $n in $env } | all {|i| $i } } - # Emulates a `test -z`, but btter as it handles e.g 'false' def is-env-true [name: string] { - if (has-env $name) { - # Try to parse 'true', '0', '1', and fail if not convertible - let parsed = (do -i { $env | get $name | into bool }) - if ($parsed | describe) == 'bool' { - $parsed + if (has-env $name) { + let val = ($env | get --optional $name) + if ($val | describe) == 'bool' { + $val + } else { + not ($val | is-empty) + } } else { - not ($env | get -i $name | is-empty) + false } - } else { - false - } } - let virtual_env = '__VIRTUAL_ENV__' - let bin = '__BIN_NAME__' - - let is_windows = ($nu.os-info.family) == 'windows' - let path_name = (if (has-env 'Path') { - 'Path' - } else { - 'PATH' - } - ) - + let virtual_env = __VIRTUAL_ENV__ + let bin = __BIN_NAME__ + let path_name = if (has-env 'Path') { 'Path' } else { 'PATH' } let venv_path = ([$virtual_env $bin] | path join) let new_path = ($env | get $path_name | prepend $venv_path) - - # If there is no default prompt, then use the env name instead - let virtual_env_prompt = (if ('__VIRTUAL_PROMPT__' | is-empty) { + let virtual_env_prompt = if (__VIRTUAL_PROMPT__ | is-empty) { ($virtual_env | path basename) } else { - '__VIRTUAL_PROMPT__' - }) - - let new_env = { - $path_name : $new_path - VIRTUAL_ENV : $virtual_env - VIRTUAL_ENV_PROMPT : $virtual_env_prompt + __VIRTUAL_PROMPT__ } - - let new_env = (if (is-env-true 'VIRTUAL_ENV_DISABLE_PROMPT') { - $new_env + let new_env = { $path_name: $new_path VIRTUAL_ENV: $virtual_env VIRTUAL_ENV_PROMPT: $virtual_env_prompt } + if (has-env 'TCL_LIBRARY') { + let $new_env = $new_env | insert TCL_LIBRARY __TCL_LIBRARY__ + } + if (has-env 'TK_LIBRARY') { + let $new_env = $new_env | insert TK_LIBRARY __TK_LIBRARY__ + } + let old_prompt_command = if (has-env 'PROMPT_COMMAND') { $env.PROMPT_COMMAND } else { '' } + let new_env = if (is-env-true 'VIRTUAL_ENV_DISABLE_PROMPT') { + $new_env } else { - # Creating the new prompt for the session - let virtual_prefix = $'(char lparen)($virtual_env_prompt)(char rparen) ' - - # Back up the old prompt builder - let old_prompt_command = (if (has-env 'PROMPT_COMMAND') { - $env.PROMPT_COMMAND - } else { - '' - }) - - let new_prompt = (if (has-env 'PROMPT_COMMAND') { - if 'closure' in ($old_prompt_command | describe) { - {|| $'($virtual_prefix)(do $old_prompt_command)' } - } else { - {|| $'($virtual_prefix)($old_prompt_command)' } - } - } else { - {|| $'($virtual_prefix)' } - }) - - $new_env | merge { - PROMPT_COMMAND : $new_prompt - VIRTUAL_PREFIX : $virtual_prefix - } - }) - - # Environment variables that will be loaded as the virtual env + let virtual_prefix = $'(char lparen)($virtual_env_prompt)(char rparen) ' + let new_prompt = if (has-env 'PROMPT_COMMAND') { + if ('closure' in ($old_prompt_command | describe)) { + {|| $'($virtual_prefix)(do $old_prompt_command)' } + } else { + {|| $'($virtual_prefix)($old_prompt_command)' } + } + } else { + {|| $'($virtual_prefix)' } + } + $new_env | merge { PROMPT_COMMAND: $new_prompt VIRTUAL_PREFIX: $virtual_prefix } + } load-env $new_env } diff --git a/src/virtualenv/activation/powershell/__init__.py b/src/virtualenv/activation/powershell/__init__.py index 1f6d0f4e4..8489656cc 100644 --- a/src/virtualenv/activation/powershell/__init__.py +++ b/src/virtualenv/activation/powershell/__init__.py @@ -7,6 +7,18 @@ class PowerShellActivator(ViaTemplateActivator): def templates(self): yield "activate.ps1" + @staticmethod + def quote(string): + """ + This should satisfy PowerShell quoting rules [1], unless the quoted + string is passed directly to Windows native commands [2]. + + [1]: https://learn.microsoft.com/en-us/powershell/module/microsoft.powershell.core/about/about_quoting_rules + [2]: https://learn.microsoft.com/en-us/powershell/module/microsoft.powershell.core/about/about_parsing#passing-arguments-that-contain-quote-characters + """ # noqa: D205 + string = string.replace("'", "''") + return f"'{string}'" + __all__ = [ "PowerShellActivator", diff --git a/src/virtualenv/activation/powershell/activate.ps1 b/src/virtualenv/activation/powershell/activate.ps1 index 5ccfe120c..9f95e4370 100644 --- a/src/virtualenv/activation/powershell/activate.ps1 +++ b/src/virtualenv/activation/powershell/activate.ps1 @@ -7,6 +7,24 @@ function global:deactivate([switch] $NonDestructive) { Remove-Variable "_OLD_VIRTUAL_PATH" -Scope global } + if (Test-Path variable:_OLD_VIRTUAL_TCL_LIBRARY) { + $env:TCL_LIBRARY = $variable:_OLD_VIRTUAL_TCL_LIBRARY + Remove-Variable "_OLD_VIRTUAL_TCL_LIBRARY" -Scope global + } else { + if (Test-Path env:TCL_LIBRARY) { + Remove-Item env:TCL_LIBRARY -ErrorAction SilentlyContinue + } + } + + if (Test-Path variable:_OLD_VIRTUAL_TK_LIBRARY) { + $env:TK_LIBRARY = $variable:_OLD_VIRTUAL_TK_LIBRARY + Remove-Variable "_OLD_VIRTUAL_TK_LIBRARY" -Scope global + } else { + if (Test-Path env:TK_LIBRARY) { + Remove-Item env:TK_LIBRARY -ErrorAction SilentlyContinue + } + } + if (Test-Path function:_old_virtual_prompt) { $function:prompt = $function:_old_virtual_prompt Remove-Item function:\_old_virtual_prompt @@ -37,16 +55,30 @@ deactivate -nondestructive $VIRTUAL_ENV = $BASE_DIR $env:VIRTUAL_ENV = $VIRTUAL_ENV -if ("__VIRTUAL_PROMPT__" -ne "") { - $env:VIRTUAL_ENV_PROMPT = "__VIRTUAL_PROMPT__" +if (__VIRTUAL_PROMPT__ -ne "") { + $env:VIRTUAL_ENV_PROMPT = __VIRTUAL_PROMPT__ } else { $env:VIRTUAL_ENV_PROMPT = $( Split-Path $env:VIRTUAL_ENV -Leaf ) } +if (__TCL_LIBRARY__ -ne "") { + if (Test-Path env:TCL_LIBRARY) { + New-Variable -Scope global -Name _OLD_VIRTUAL_TCL_LIBRARY -Value $env:TCL_LIBRARY + } + $env:TCL_LIBRARY = __TCL_LIBRARY__ +} + +if (__TK_LIBRARY__ -ne "") { + if (Test-Path env:TK_LIBRARY) { + New-Variable -Scope global -Name _OLD_VIRTUAL_TK_LIBRARY -Value $env:TK_LIBRARY + } + $env:TK_LIBRARY = __TK_LIBRARY__ +} + New-Variable -Scope global -Name _OLD_VIRTUAL_PATH -Value $env:PATH -$env:PATH = "$env:VIRTUAL_ENV/__BIN_NAME____PATH_SEP__" + $env:PATH +$env:PATH = "$env:VIRTUAL_ENV/" + __BIN_NAME__ + __PATH_SEP__ + $env:PATH if (!$env:VIRTUAL_ENV_DISABLE_PROMPT) { function global:_old_virtual_prompt { "" diff --git a/src/virtualenv/activation/python/__init__.py b/src/virtualenv/activation/python/__init__.py index 3126a39f2..e900f7ec9 100644 --- a/src/virtualenv/activation/python/__init__.py +++ b/src/virtualenv/activation/python/__init__.py @@ -10,10 +10,14 @@ class PythonActivator(ViaTemplateActivator): def templates(self): yield "activate_this.py" + @staticmethod + def quote(string): + return repr(string) + def replacements(self, creator, dest_folder): replacements = super().replacements(creator, dest_folder) lib_folders = OrderedDict((os.path.relpath(str(i), str(dest_folder)), None) for i in creator.libs) - lib_folders = os.pathsep.join(lib_folders.keys()).replace("\\", "\\\\") # escape Windows path characters + lib_folders = os.pathsep.join(lib_folders.keys()) replacements.update( { "__LIB_FOLDERS__": lib_folders, diff --git a/src/virtualenv/activation/python/activate_this.py b/src/virtualenv/activation/python/activate_this.py index 33a3eaee8..9cc816fab 100644 --- a/src/virtualenv/activation/python/activate_this.py +++ b/src/virtualenv/activation/python/activate_this.py @@ -1,10 +1,12 @@ """ Activate virtualenv for current interpreter: -Use exec(open(this_file).read(), {'__file__': this_file}). +import runpy +runpy.run_path(this_file) This can be used when you must use an existing Python interpreter, not the virtualenv bin/python. """ # noqa: D415 + from __future__ import annotations import os @@ -14,22 +16,22 @@ try: abs_file = os.path.abspath(__file__) except NameError as exc: - msg = "You must use exec(open(this_file).read(), {'__file__': this_file}))" + msg = "You must use import runpy; runpy.run_path(this_file)" raise AssertionError(msg) from exc bin_dir = os.path.dirname(abs_file) -base = bin_dir[: -len("__BIN_NAME__") - 1] # strip away the bin part from the __file__, plus the path separator +base = bin_dir[: -len(__BIN_NAME__) - 1] # strip away the bin part from the __file__, plus the path separator # prepend bin to PATH (this file is inside the bin directory) os.environ["PATH"] = os.pathsep.join([bin_dir, *os.environ.get("PATH", "").split(os.pathsep)]) os.environ["VIRTUAL_ENV"] = base # virtual env is right above bin directory -os.environ["VIRTUAL_ENV_PROMPT"] = "__VIRTUAL_PROMPT__" or os.path.basename(base) # noqa: SIM222 +os.environ["VIRTUAL_ENV_PROMPT"] = __VIRTUAL_PROMPT__ or os.path.basename(base) # add the virtual environments libraries to the host python import mechanism prev_length = len(sys.path) -for lib in "__LIB_FOLDERS__".split(os.pathsep): +for lib in __LIB_FOLDERS__.split(os.pathsep): path = os.path.realpath(os.path.join(bin_dir, lib)) - site.addsitedir(path.decode("utf-8") if "__DECODE_PATH__" else path) + site.addsitedir(path.decode("utf-8") if __DECODE_PATH__ else path) sys.path[:] = sys.path[prev_length:] + sys.path[0:prev_length] sys.real_prefix = sys.prefix diff --git a/src/virtualenv/activation/via_template.py b/src/virtualenv/activation/via_template.py index 239318cde..85f932605 100644 --- a/src/virtualenv/activation/via_template.py +++ b/src/virtualenv/activation/via_template.py @@ -1,8 +1,9 @@ from __future__ import annotations import os +import shlex import sys -from abc import ABCMeta, abstractmethod +from abc import ABC, abstractmethod from .activator import Activator @@ -16,11 +17,21 @@ def read_binary(module_name: str, filename: str) -> bytes: from importlib.resources import read_binary -class ViaTemplateActivator(Activator, metaclass=ABCMeta): +class ViaTemplateActivator(Activator, ABC): @abstractmethod def templates(self): raise NotImplementedError + @staticmethod + def quote(string): + """ + Quote strings in the activation script. + + :param string: the string to quote + :return: quoted string that works in the activation script + """ + return shlex.quote(string) + def generate(self, creator): dest_folder = creator.bin_dir replacements = self.replacements(creator, dest_folder) @@ -36,6 +47,8 @@ def replacements(self, creator, dest_folder): # noqa: ARG002 "__VIRTUAL_NAME__": creator.env_name, "__BIN_NAME__": str(creator.bin_dir.relative_to(creator.dest)), "__PATH_SEP__": os.pathsep, + "__TCL_LIBRARY__": getattr(creator.interpreter, "tcl_lib", None) or "", + "__TK_LIBRARY__": getattr(creator.interpreter, "tk_lib", None) or "", } def _generate(self, replacements, templates, to_folder, creator): @@ -47,8 +60,10 @@ def _generate(self, replacements, templates, to_folder, creator): # errors when the dest is not writable if dest.exists(): dest.unlink() + # Powershell assumes Windows 1252 encoding when reading files without BOM + encoding = "utf-8-sig" if str(template).endswith(".ps1") else "utf-8" # use write_bytes to avoid platform specific line normalization (\n -> \r\n) - dest.write_bytes(text.encode("utf-8")) + dest.write_bytes(text.encode(encoding)) generated.append(dest) return generated @@ -61,7 +76,7 @@ def instantiate_template(self, replacements, template, creator): text = binary.decode("utf-8", errors="strict") for key, value in replacements.items(): value_uni = self._repr_unicode(creator, value) - text = text.replace(key, value_uni) + text = text.replace(key, self.quote(value_uni)) return text @staticmethod diff --git a/src/virtualenv/app_data/__init__.py b/src/virtualenv/app_data/__init__.py index 148c94183..d7f148023 100644 --- a/src/virtualenv/app_data/__init__.py +++ b/src/virtualenv/app_data/__init__.py @@ -12,6 +12,8 @@ from .via_disk_folder import AppDataDiskFolder from .via_tempdir import TempAppData +LOGGER = logging.getLogger(__name__) + def _default_app_data_dir(env): key = "VIRTUALENV_OVERRIDE_APP_DATA" @@ -37,13 +39,13 @@ def make_app_data(folder, **kwargs): if not os.path.isdir(folder): try: os.makedirs(folder) - logging.debug("created app data folder %s", folder) + LOGGER.debug("created app data folder %s", folder) except OSError as exception: - logging.info("could not create app data folder %s due to %r", folder, exception) + LOGGER.info("could not create app data folder %s due to %r", folder, exception) if os.access(folder, os.W_OK): return AppDataDiskFolder(folder) - logging.debug("app data folder %s has no write access", folder) + LOGGER.debug("app data folder %s has no write access", folder) return TempAppData() diff --git a/src/virtualenv/app_data/base.py b/src/virtualenv/app_data/base.py index 4d82e2138..2077deebd 100644 --- a/src/virtualenv/app_data/base.py +++ b/src/virtualenv/app_data/base.py @@ -2,13 +2,13 @@ from __future__ import annotations -from abc import ABCMeta, abstractmethod +from abc import ABC, abstractmethod from contextlib import contextmanager from virtualenv.info import IS_ZIPAPP -class AppData(metaclass=ABCMeta): +class AppData(ABC): """Abstract storage interface for the virtualenv application.""" @abstractmethod @@ -67,7 +67,7 @@ def locked(self, path): raise NotImplementedError -class ContentStore(metaclass=ABCMeta): +class ContentStore(ABC): @abstractmethod def exists(self): raise NotImplementedError @@ -91,6 +91,6 @@ def locked(self): __all__ = [ - "ContentStore", "AppData", + "ContentStore", ] diff --git a/src/virtualenv/app_data/via_disk_folder.py b/src/virtualenv/app_data/via_disk_folder.py index fa87149e2..9ebe91c2e 100644 --- a/src/virtualenv/app_data/via_disk_folder.py +++ b/src/virtualenv/app_data/via_disk_folder.py @@ -26,7 +26,7 @@ import json import logging -from abc import ABCMeta +from abc import ABC from contextlib import contextmanager, suppress from hashlib import sha256 @@ -37,6 +37,8 @@ from .base import AppData, ContentStore +LOGGER = logging.getLogger(__name__) + class AppDataDiskFolder(AppData): """Store the application data on the disk within a folder layout.""" @@ -54,7 +56,7 @@ def __str__(self) -> str: return str(self.lock.path) def reset(self): - logging.debug("reset app data folder %s", self.lock.path) + LOGGER.debug("reset app data folder %s", self.lock.path) safe_delete(self.lock.path) def close(self): @@ -77,7 +79,7 @@ def extract(self, path, to_folder): @property def py_info_at(self): - return self.lock / "py_info" / "1" + return self.lock / "py_info" / "2" def py_info(self, path): return PyInfoStoreDisk(self.py_info_at, path) @@ -105,11 +107,10 @@ def wheel_image(self, for_py_version, name): return self.lock.path / "wheel" / for_py_version / "image" / "1" / name -class JSONStoreDisk(ContentStore, metaclass=ABCMeta): - def __init__(self, in_folder, key, msg, msg_args) -> None: +class JSONStoreDisk(ContentStore, ABC): + def __init__(self, in_folder, key, msg_args) -> None: self.in_folder = in_folder self.key = key - self.msg = msg self.msg_args = (*msg_args, self.file) @property @@ -128,7 +129,7 @@ def read(self): except Exception: # noqa: BLE001, S110 pass else: - logging.debug("got %s from %s", self.msg, self.msg_args) + LOGGER.debug("got %s %s from %s", *self.msg_args) return data if bad_format: with suppress(OSError): # reading and writing on the same file may cause race on multiple processes @@ -137,7 +138,7 @@ def read(self): def remove(self): self.file.unlink() - logging.debug("removed %s at %s", self.msg, self.msg_args) + LOGGER.debug("removed %s %s at %s", *self.msg_args) @contextmanager def locked(self): @@ -148,13 +149,13 @@ def write(self, content): folder = self.file.parent folder.mkdir(parents=True, exist_ok=True) self.file.write_text(json.dumps(content, sort_keys=True, indent=2), encoding="utf-8") - logging.debug("wrote %s at %s", self.msg, self.msg_args) + LOGGER.debug("wrote %s %s at %s", *self.msg_args) class PyInfoStoreDisk(JSONStoreDisk): def __init__(self, in_folder, path) -> None: key = sha256(str(path).encode("utf-8")).hexdigest() - super().__init__(in_folder, key, "python info of %s", (path,)) + super().__init__(in_folder, key, ("python info of", path)) class EmbedDistributionUpdateStoreDisk(JSONStoreDisk): @@ -162,8 +163,7 @@ def __init__(self, in_folder, distribution) -> None: super().__init__( in_folder, distribution, - "embed update of distribution %s", - (distribution,), + ("embed update of distribution", distribution), ) diff --git a/src/virtualenv/app_data/via_tempdir.py b/src/virtualenv/app_data/via_tempdir.py index 0a30dfe1c..884a570ce 100644 --- a/src/virtualenv/app_data/via_tempdir.py +++ b/src/virtualenv/app_data/via_tempdir.py @@ -7,6 +7,8 @@ from .via_disk_folder import AppDataDiskFolder +LOGGER = logging.getLogger(__name__) + class TempAppData(AppDataDiskFolder): transient = True @@ -14,13 +16,13 @@ class TempAppData(AppDataDiskFolder): def __init__(self) -> None: super().__init__(folder=mkdtemp()) - logging.debug("created temporary app data folder %s", self.lock.path) + LOGGER.debug("created temporary app data folder %s", self.lock.path) def reset(self): """This is a temporary folder, is already empty to start with.""" def close(self): - logging.debug("remove temporary app data folder %s", self.lock.path) + LOGGER.debug("remove temporary app data folder %s", self.lock.path) safe_delete(self.lock.path) def embed_update_log(self, distribution, for_py_version): diff --git a/src/virtualenv/cache/__init__.py b/src/virtualenv/cache/__init__.py new file mode 100644 index 000000000..8f929c60c --- /dev/null +++ b/src/virtualenv/cache/__init__.py @@ -0,0 +1,9 @@ +from __future__ import annotations + +from .cache import Cache +from .file_cache import FileCache + +__all__ = [ + "Cache", + "FileCache", +] diff --git a/src/virtualenv/cache/cache.py b/src/virtualenv/cache/cache.py new file mode 100644 index 000000000..a13df5aaa --- /dev/null +++ b/src/virtualenv/cache/cache.py @@ -0,0 +1,62 @@ +from __future__ import annotations + +from abc import ABC, abstractmethod +from typing import Any, Generic, Hashable, TypeVar + +try: + from typing import Self # pragma: โ‰ฅ 3.11 cover +except ImportError: + from typing_extensions import Self # pragma: < 3.11 cover + +K = TypeVar("K", bound=Hashable) + + +class Cache(ABC, Generic[K]): + """ + A generic cache interface. + + Add a close() method if the cache needs to perform any cleanup actions, + and an __exit__ method to allow it to be used in a context manager. + """ + + @abstractmethod + def get(self, key: K) -> Any | None: + """ + Get a value from the cache. + + :param key: the key to retrieve + :return: the cached value, or None if not found + """ + raise NotImplementedError + + @abstractmethod + def set(self, key: K, value: Any) -> None: + """ + Set a value in the cache. + + :param key: the key to set + :param value: the value to cache + """ + raise NotImplementedError + + @abstractmethod + def remove(self, key: K) -> None: + """ + Remove a value from the cache. + + :param key: the key to remove + """ + raise NotImplementedError + + @abstractmethod + def clear(self) -> None: + """Clear the entire cache.""" + raise NotImplementedError + + def __enter__(self) -> Self: + return self + + +__all__ = [ + "Cache", +] diff --git a/src/virtualenv/cache/file_cache.py b/src/virtualenv/cache/file_cache.py new file mode 100644 index 000000000..2a5605aea --- /dev/null +++ b/src/virtualenv/cache/file_cache.py @@ -0,0 +1,47 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Callable + +from virtualenv.cache import Cache + +if TYPE_CHECKING: + from pathlib import Path + + from virtualenv.app_data.base import ContentStore + + +class FileCache(Cache): + def __init__(self, store_factory: Callable[[Path], ContentStore], clearer: Callable[[], None] | None) -> None: + self.store_factory = store_factory + self.clearer = clearer + + def get(self, key: Path) -> dict | None: + """Get a value from the file cache.""" + result, store = None, self.store_factory(key) + with store.locked(): + if store.exists(): + result = store.read() + return result + + def set(self, key: Path, value: dict) -> None: + """Set a value in the file cache.""" + store = self.store_factory(key) + with store.locked(): + store.write(value) + + def remove(self, key: Path) -> None: + """Remove a value from the file cache.""" + store = self.store_factory(key) + with store.locked(): + if store.exists(): + store.remove() + + def clear(self) -> None: + """Clear the entire file cache.""" + if self.clearer is not None: + self.clearer() + + +__all__ = [ + "FileCache", +] diff --git a/src/virtualenv/config/cli/parser.py b/src/virtualenv/config/cli/parser.py index 9323d4e81..fd1dc0956 100644 --- a/src/virtualenv/config/cli/parser.py +++ b/src/virtualenv/config/cli/parser.py @@ -107,8 +107,8 @@ def parse_known_args(self, args=None, namespace=None): class HelpFormatter(ArgumentDefaultsHelpFormatter): - def __init__(self, prog) -> None: - super().__init__(prog, max_help_position=32, width=240) + def __init__(self, prog, **kwargs) -> None: + super().__init__(prog, max_help_position=32, width=240, **kwargs) def _get_help_string(self, action): text = super()._get_help_string(action) diff --git a/src/virtualenv/config/convert.py b/src/virtualenv/config/convert.py index 7f1e9975a..ef7581dbd 100644 --- a/src/virtualenv/config/convert.py +++ b/src/virtualenv/config/convert.py @@ -4,6 +4,8 @@ import os from typing import ClassVar +LOGGER = logging.getLogger(__name__) + class TypeData: def __init__(self, default_type, as_type) -> None: @@ -50,8 +52,8 @@ def _validate(self): def convert(self, value, flatten=True): # noqa: ARG002, FBT002 values = self.split_values(value) result = [] - for value in values: - sub_values = value.split(os.pathsep) + for a_value in values: + sub_values = a_value.split(os.pathsep) result.extend(sub_values) return [self.as_type(i) for i in result] @@ -81,7 +83,7 @@ def convert(value, as_type, source): try: return as_type.convert(value) except Exception as exception: - logging.warning("%s failed to convert %r as %r because %r", source, value, as_type, exception) + LOGGER.warning("%s failed to convert %r as %r because %r", source, value, as_type, exception) raise diff --git a/src/virtualenv/config/ini.py b/src/virtualenv/config/ini.py index cd6ecf504..ed0a1b930 100644 --- a/src/virtualenv/config/ini.py +++ b/src/virtualenv/config/ini.py @@ -10,6 +10,8 @@ from .convert import convert +LOGGER = logging.getLogger(__name__) + class IniConfig: VIRTUALENV_CONFIG_FILE_ENV_VAR: ClassVar[str] = "VIRTUALENV_CONFIG_FILE" @@ -44,7 +46,7 @@ def __init__(self, env=None) -> None: except Exception as exc: # noqa: BLE001 exception = exc if exception is not None: - logging.error("failed to read config file %s because %r", config_file, exception) + LOGGER.error("failed to read config file %s because %r", config_file, exception) def _load(self): with self.config_file.open("rt", encoding="utf-8") as file_handler: diff --git a/src/virtualenv/create/creator.py b/src/virtualenv/create/creator.py index 8ff54166e..1e577ada5 100644 --- a/src/virtualenv/create/creator.py +++ b/src/virtualenv/create/creator.py @@ -4,7 +4,8 @@ import logging import os import sys -from abc import ABCMeta, abstractmethod +import textwrap +from abc import ABC, abstractmethod from argparse import ArgumentTypeError from ast import literal_eval from collections import OrderedDict @@ -19,6 +20,7 @@ HERE = Path(os.path.abspath(__file__)).parent DEBUG_SCRIPT = HERE / "debug.py" +LOGGER = logging.getLogger(__name__) class CreatorMeta: @@ -26,7 +28,7 @@ def __init__(self) -> None: self.error = None -class Creator(metaclass=ABCMeta): +class Creator(ABC): """A class that given a python Interpreter creates a virtual environment.""" def __init__(self, options, interpreter) -> None: @@ -153,13 +155,26 @@ def non_write_able(dest, value): def run(self): if self.dest.exists() and self.clear: - logging.debug("delete %s", self.dest) + LOGGER.debug("delete %s", self.dest) safe_delete(self.dest) self.create() + self.add_cachedir_tag() self.set_pyenv_cfg() if not self.no_vcs_ignore: self.setup_ignore_vcs() + def add_cachedir_tag(self): + """Generate a file indicating that this is not meant to be backed up.""" + cachedir_tag_file = self.dest / "CACHEDIR.TAG" + if not cachedir_tag_file.exists(): + cachedir_tag_text = textwrap.dedent(""" + Signature: 8a477f597d28d172789f06886806bc55 + # This file is a cache directory tag created by Python virtualenv. + # For information about cache directory tags, see: + # https://bford.info/cachedir/ + """).strip() + cachedir_tag_file.write_text(cachedir_tag_text, encoding="utf-8") + def set_pyenv_cfg(self): self.pyenv_cfg.content = OrderedDict() self.pyenv_cfg["home"] = os.path.dirname(os.path.abspath(self.interpreter.system_executable)) @@ -195,9 +210,9 @@ def get_env_debug_info(env_exe, debug_script, app_data, env): env = env.copy() env.pop("PYTHONPATH", None) - with app_data.ensure_extracted(debug_script) as debug_script: - cmd = [str(env_exe), str(debug_script)] - logging.debug("debug via %r", LogCmd(cmd)) + with app_data.ensure_extracted(debug_script) as debug_script_extracted: + cmd = [str(env_exe), str(debug_script_extracted)] + LOGGER.debug("debug via %r", LogCmd(cmd)) code, out, err = run_cmd(cmd) try: diff --git a/src/virtualenv/create/debug.py b/src/virtualenv/create/debug.py index ee7fc900a..8a4845e98 100644 --- a/src/virtualenv/create/debug.py +++ b/src/virtualenv/create/debug.py @@ -1,4 +1,5 @@ """Inspect a target Python interpreter virtual environment wise.""" + from __future__ import annotations import sys # built-in @@ -18,10 +19,10 @@ def encode_list_path(value): return [encode_path(i) for i in value] -def run(): # noqa: PLR0912 +def run(): """Print debug data about the virtual environment.""" try: - from collections import OrderedDict + from collections import OrderedDict # noqa: PLC0415 except ImportError: # pragma: no cover # this is possible if the standard library cannot be accessed @@ -47,7 +48,7 @@ def run(): # noqa: PLR0912 result["version"] = sys.version try: - import sysconfig + import sysconfig # noqa: PLC0415 # https://bugs.python.org/issue22199 makefile = getattr(sysconfig, "get_makefile_filename", getattr(sysconfig, "_get_makefile_filename", None)) @@ -55,26 +56,26 @@ def run(): # noqa: PLR0912 except ImportError: pass - import os # landmark + import os # landmark # noqa: PLC0415 result["os"] = repr(os) try: - import site # site + import site # site # noqa: PLC0415 result["site"] = repr(site) except ImportError as exception: # pragma: no cover result["site"] = repr(exception) # pragma: no cover try: - import datetime # site + import datetime # site # noqa: PLC0415 result["datetime"] = repr(datetime) except ImportError as exception: # pragma: no cover result["datetime"] = repr(exception) # pragma: no cover try: - import math # site + import math # site # noqa: PLC0415 result["math"] = repr(math) except ImportError as exception: # pragma: no cover @@ -82,7 +83,7 @@ def run(): # noqa: PLR0912 # try to print out, this will validate if other core modules are available (json in this case) try: - import json + import json # noqa: PLC0415 result["json"] = repr(json) except ImportError as exception: @@ -94,7 +95,7 @@ def run(): # noqa: PLR0912 except (ValueError, TypeError) as exception: # pragma: no cover sys.stderr.write(repr(exception)) sys.stdout.write(repr(result)) # pragma: no cover - raise SystemExit(1) # noqa: TRY200, B904 # pragma: no cover + raise SystemExit(1) # noqa: B904 # pragma: no cover if __name__ == "__main__": diff --git a/src/virtualenv/create/describe.py b/src/virtualenv/create/describe.py index 7167e56d7..1ee250cbc 100644 --- a/src/virtualenv/create/describe.py +++ b/src/virtualenv/create/describe.py @@ -1,13 +1,13 @@ from __future__ import annotations -from abc import ABCMeta +from abc import ABC from collections import OrderedDict from pathlib import Path from virtualenv.info import IS_WIN -class Describe(metaclass=ABCMeta): +class Describe: """Given a host interpreter tell us information about what the created interpreter might look like.""" suffix = ".exe" if IS_WIN else "" @@ -84,19 +84,19 @@ def script(self, name): return self.script_dir / f"{name}{self.suffix}" -class Python3Supports(Describe, metaclass=ABCMeta): +class Python3Supports(Describe, ABC): @classmethod def can_describe(cls, interpreter): return interpreter.version_info.major == 3 and super().can_describe(interpreter) # noqa: PLR2004 -class PosixSupports(Describe, metaclass=ABCMeta): +class PosixSupports(Describe, ABC): @classmethod def can_describe(cls, interpreter): return interpreter.os == "posix" and super().can_describe(interpreter) -class WindowsSupports(Describe, metaclass=ABCMeta): +class WindowsSupports(Describe, ABC): @classmethod def can_describe(cls, interpreter): return interpreter.os == "nt" and super().can_describe(interpreter) @@ -104,7 +104,7 @@ def can_describe(cls, interpreter): __all__ = [ "Describe", - "Python3Supports", "PosixSupports", + "Python3Supports", "WindowsSupports", ] diff --git a/src/virtualenv/create/pyenv_cfg.py b/src/virtualenv/create/pyenv_cfg.py index 03ebfbbda..1d1beacff 100644 --- a/src/virtualenv/create/pyenv_cfg.py +++ b/src/virtualenv/create/pyenv_cfg.py @@ -1,8 +1,11 @@ from __future__ import annotations import logging +import os from collections import OrderedDict +LOGGER = logging.getLogger(__name__) + class PyEnvCfg: def __init__(self, content, path) -> None: @@ -29,11 +32,12 @@ def _read_values(path): return content def write(self): - logging.debug("write %s", self.path) + LOGGER.debug("write %s", self.path) text = "" for key, value in self.content.items(): - line = f"{key} = {value}" - logging.debug("\t%s", line) + normalized_value = os.path.realpath(value) if value and os.path.exists(value) else value + line = f"{key} = {normalized_value}" + LOGGER.debug("\t%s", line) text += line text += "\n" self.path.write_text(text, encoding="utf-8") diff --git a/src/virtualenv/create/via_global_ref/_virtualenv.py b/src/virtualenv/create/via_global_ref/_virtualenv.py index 17f73b1d4..b61db3079 100644 --- a/src/virtualenv/create/via_global_ref/_virtualenv.py +++ b/src/virtualenv/create/via_global_ref/_virtualenv.py @@ -4,7 +4,6 @@ import os import sys -from contextlib import suppress VIRTUALENV_PATCH_FILE = os.path.join(__file__) @@ -51,10 +50,10 @@ class _Finder: lock = [] # noqa: RUF012 def find_spec(self, fullname, path, target=None): # noqa: ARG002 - if fullname in _DISTUTILS_PATCH and self.fullname is None: + if fullname in _DISTUTILS_PATCH and self.fullname is None: # noqa: PLR1702 # initialize lock[0] lazily if len(self.lock) == 0: - import threading + import threading # noqa: PLC0415 lock = threading.Lock() # there is possibility that two threads T1 and T2 are simultaneously running into find_spec, @@ -64,8 +63,8 @@ def find_spec(self, fullname, path, target=None): # noqa: ARG002 # https://docs.python.org/3/faq/library.html#what-kinds-of-global-value-mutation-are-thread-safe self.lock.append(lock) - from functools import partial - from importlib.util import find_spec + from functools import partial # noqa: PLC0415 + from importlib.util import find_spec # noqa: PLC0415 with self.lock[0]: self.fullname = fullname @@ -78,8 +77,10 @@ def find_spec(self, fullname, path, target=None): # noqa: ARG002 old = getattr(spec.loader, func_name) func = self.exec_module if is_new_api else self.load_module if old is not func: - with suppress(AttributeError): # C-Extension loaders are r/o such as zipimporter with <3.7 + try: # noqa: SIM105 setattr(spec.loader, func_name, partial(func, old)) + except AttributeError: + pass # C-Extension loaders are r/o such as zipimporter with <3.7 return spec finally: self.fullname = None diff --git a/src/virtualenv/create/via_global_ref/api.py b/src/virtualenv/create/via_global_ref/api.py index 0d362652e..3f04f4653 100644 --- a/src/virtualenv/create/via_global_ref/api.py +++ b/src/virtualenv/create/via_global_ref/api.py @@ -2,12 +2,14 @@ import logging import os -from abc import ABCMeta +from abc import ABC from pathlib import Path from virtualenv.create.creator import Creator, CreatorMeta from virtualenv.info import fs_supports_symlink +LOGGER = logging.getLogger(__name__) + class ViaGlobalRefMeta(CreatorMeta): def __init__(self) -> None: @@ -26,7 +28,7 @@ def can_symlink(self): return not self.symlink_error -class ViaGlobalRefApi(Creator, metaclass=ABCMeta): +class ViaGlobalRefApi(Creator, ABC): def __init__(self, options, interpreter) -> None: super().__init__(options, interpreter) self.symlinks = self._should_symlink(options) @@ -59,10 +61,15 @@ def add_parser_arguments(cls, parser, interpreter, meta, app_data): dest="system_site", help="give the virtual environment access to the system site-packages dir", ) - group = parser.add_mutually_exclusive_group() if not meta.can_symlink and not meta.can_copy: - msg = "neither symlink or copy method supported" + errors = [] + if meta.symlink_error: + errors.append(f"symlink: {meta.symlink_error}") + if meta.copy_error: + errors.append(f"copy: {meta.copy_error}") + msg = f"neither symlink or copy method supported: {', '.join(errors)}" raise RuntimeError(msg) + group = parser.add_mutually_exclusive_group() if meta.can_symlink: group.add_argument( "--symlinks", @@ -88,10 +95,10 @@ def install_patch(self): text = self.env_patch_text() if text: pth = self.purelib / "_virtualenv.pth" - logging.debug("create virtualenv import hook file %s", pth) + LOGGER.debug("create virtualenv import hook file %s", pth) pth.write_text("import _virtualenv", encoding="utf-8") dest_path = self.purelib / "_virtualenv.py" - logging.debug("create %s", dest_path) + LOGGER.debug("create %s", dest_path) dest_path.write_text(text, encoding="utf-8") def env_patch_text(self): @@ -109,6 +116,6 @@ def set_pyenv_cfg(self): __all__ = [ - "ViaGlobalRefMeta", "ViaGlobalRefApi", + "ViaGlobalRefMeta", ] diff --git a/src/virtualenv/create/via_global_ref/builtin/builtin_way.py b/src/virtualenv/create/via_global_ref/builtin/builtin_way.py index bb520a354..791b1d93d 100644 --- a/src/virtualenv/create/via_global_ref/builtin/builtin_way.py +++ b/src/virtualenv/create/via_global_ref/builtin/builtin_way.py @@ -1,12 +1,12 @@ from __future__ import annotations -from abc import ABCMeta +from abc import ABC from virtualenv.create.creator import Creator from virtualenv.create.describe import Describe -class VirtualenvBuiltin(Creator, Describe, metaclass=ABCMeta): +class VirtualenvBuiltin(Creator, Describe, ABC): """A creator that does operations itself without delegation, if we can create it we can also describe it.""" def __init__(self, options, interpreter) -> None: diff --git a/src/virtualenv/create/via_global_ref/builtin/cpython/common.py b/src/virtualenv/create/via_global_ref/builtin/cpython/common.py index 7c7abd5b4..5cc993bda 100644 --- a/src/virtualenv/create/via_global_ref/builtin/cpython/common.py +++ b/src/virtualenv/create/via_global_ref/builtin/cpython/common.py @@ -1,6 +1,7 @@ from __future__ import annotations -from abc import ABCMeta +import re +from abc import ABC from collections import OrderedDict from pathlib import Path @@ -9,7 +10,7 @@ from virtualenv.create.via_global_ref.builtin.via_global_self_do import ViaGlobalRefVirtualenvBuiltin -class CPython(ViaGlobalRefVirtualenvBuiltin, metaclass=ABCMeta): +class CPython(ViaGlobalRefVirtualenvBuiltin, ABC): @classmethod def can_describe(cls, interpreter): return interpreter.implementation == "CPython" and super().can_describe(interpreter) @@ -19,7 +20,7 @@ def exe_stem(cls): return "python" -class CPythonPosix(CPython, PosixSupports, metaclass=ABCMeta): +class CPythonPosix(CPython, PosixSupports, ABC): """Create a CPython virtual environment on POSIX platforms.""" @classmethod @@ -30,14 +31,17 @@ def _executables(cls, interpreter): yield host_exe, list(targets.keys()), RefMust.NA, RefWhen.ANY -class CPythonWindows(CPython, WindowsSupports, metaclass=ABCMeta): +class CPythonWindows(CPython, WindowsSupports, ABC): @classmethod def _executables(cls, interpreter): # symlink of the python executables does not work reliably, copy always instead # - https://bugs.python.org/issue42013 # - venv host = cls.host_python(interpreter) - for path in (host.parent / n for n in {"python.exe", host.name}): # noqa: PLC0208 + names = {"python.exe", host.name} + if interpreter.version_info.major == 3: # noqa: PLR2004 + names.update({"python3.exe", "python3"}) + for path in (host.parent / n for n in names): yield host, [path.name], RefMust.COPY, RefWhen.ANY # for more info on pythonw.exe see https://stackoverflow.com/a/30313091 python_w = host.parent / "pythonw.exe" @@ -54,9 +58,19 @@ def is_mac_os_framework(interpreter): return False +def is_macos_brew(interpreter): + return interpreter.platform == "darwin" and _BREW.fullmatch(interpreter.system_prefix) is not None + + +_BREW = re.compile( + r"/(usr/local|opt/homebrew)/(opt/python@3\.\d{1,2}|Cellar/python@3\.\d{1,2}/3\.\d{1,2}\.\d{1,2})/Frameworks/" + r"Python\.framework/Versions/3\.\d{1,2}", +) + __all__ = [ "CPython", "CPythonPosix", "CPythonWindows", "is_mac_os_framework", + "is_macos_brew", ] diff --git a/src/virtualenv/create/via_global_ref/builtin/cpython/cpython3.py b/src/virtualenv/create/via_global_ref/builtin/cpython/cpython3.py index 0b7b023f6..a46da45dd 100644 --- a/src/virtualenv/create/via_global_ref/builtin/cpython/cpython3.py +++ b/src/virtualenv/create/via_global_ref/builtin/cpython/cpython3.py @@ -8,20 +8,24 @@ from textwrap import dedent from virtualenv.create.describe import Python3Supports -from virtualenv.create.via_global_ref.builtin.ref import PathRefToDest +from virtualenv.create.via_global_ref.builtin.ref import ExePathRefToDest, PathRefToDest from virtualenv.create.via_global_ref.store import is_store_python -from .common import CPython, CPythonPosix, CPythonWindows, is_mac_os_framework +from .common import CPython, CPythonPosix, CPythonWindows, is_mac_os_framework, is_macos_brew -class CPython3(CPython, Python3Supports, metaclass=abc.ABCMeta): +class CPython3(CPython, Python3Supports, abc.ABC): """CPython 3 or later.""" class CPython3Posix(CPythonPosix, CPython3): @classmethod def can_describe(cls, interpreter): - return is_mac_os_framework(interpreter) is False and super().can_describe(interpreter) + return ( + is_mac_os_framework(interpreter) is False + and is_macos_brew(interpreter) is False + and super().can_describe(interpreter) + ) def env_patch_text(self): text = super().env_patch_text() @@ -65,7 +69,35 @@ def sources(cls, interpreter): @classmethod def executables(cls, interpreter): - return super().sources(interpreter) + sources = super().sources(interpreter) + if interpreter.version_info >= (3, 13): + # Create new refs with corrected launcher paths + updated_sources = [] + for ref in sources: + if ref.src.name == "python.exe": + launcher_path = ref.src.with_name("venvlauncher.exe") + if launcher_path.exists(): + new_ref = ExePathRefToDest( + launcher_path, dest=ref.dest, targets=[ref.base, *ref.aliases], must=ref.must, when=ref.when + ) + updated_sources.append(new_ref) + continue + elif ref.src.name == "pythonw.exe": + w_launcher_path = ref.src.with_name("venvwlauncher.exe") + if w_launcher_path.exists(): + new_ref = ExePathRefToDest( + w_launcher_path, + dest=ref.dest, + targets=[ref.base, *ref.aliases], + must=ref.must, + when=ref.when, + ) + updated_sources.append(new_ref) + continue + # Keep the original ref unchanged + updated_sources.append(ref) + return updated_sources + return sources @classmethod def has_shim(cls, interpreter): @@ -73,7 +105,11 @@ def has_shim(cls, interpreter): @classmethod def shim(cls, interpreter): - shim = Path(interpreter.system_stdlib) / "venv" / "scripts" / "nt" / "python.exe" + root = Path(interpreter.system_stdlib) / "venv" / "scripts" / "nt" + # Before 3.13 the launcher was called python.exe, after is venvlauncher.exe + # https://github.com/python/cpython/issues/112984 + exe_name = "venvlauncher.exe" if interpreter.version_info >= (3, 13) else "python.exe" + shim = root / exe_name if shim.exists(): return shim return None @@ -98,7 +134,7 @@ def dll_and_pyd(cls, interpreter): for folder in folders: for file in folder.iterdir(): - if file.suffix in (".pyd", ".dll"): + if file.suffix in {".pyd", ".dll"}: yield PathRefToDest(file, cls.to_bin) @classmethod diff --git a/src/virtualenv/create/via_global_ref/builtin/cpython/mac_os.py b/src/virtualenv/create/via_global_ref/builtin/cpython/mac_os.py index 42c191fcb..0ddbf9a33 100644 --- a/src/virtualenv/create/via_global_ref/builtin/cpython/mac_os.py +++ b/src/virtualenv/create/via_global_ref/builtin/cpython/mac_os.py @@ -1,11 +1,12 @@ """The Apple Framework builds require their own customization.""" + from __future__ import annotations import logging import os import struct import subprocess -from abc import ABCMeta, abstractmethod +from abc import ABC, abstractmethod from pathlib import Path from textwrap import dedent @@ -14,12 +15,15 @@ PathRefToDest, RefMust, ) +from virtualenv.create.via_global_ref.builtin.via_global_self_do import BuiltinViaGlobalRefMeta -from .common import CPython, CPythonPosix, is_mac_os_framework +from .common import CPython, CPythonPosix, is_mac_os_framework, is_macos_brew from .cpython3 import CPython3 +LOGGER = logging.getLogger(__name__) + -class CPythonmacOsFramework(CPython, metaclass=ABCMeta): +class CPythonmacOsFramework(CPython, ABC): @classmethod def can_describe(cls, interpreter): return is_mac_os_framework(interpreter) and super().can_describe(interpreter) @@ -113,13 +117,13 @@ def fix_mach_o(exe, current, new, max_size): unneeded bits of information, however Mac OS X 10.5 and earlier cannot read this new Link Edit table format. """ try: - logging.debug("change Mach-O for %s from %s to %s", exe, current, new) + LOGGER.debug("change Mach-O for %s from %s to %s", exe, current, new) _builtin_change_mach_o(max_size)(exe, current, new) except Exception as e: # noqa: BLE001 - logging.warning("Could not call _builtin_change_mac_o: %s. Trying to call install_name_tool instead.", e) + LOGGER.warning("Could not call _builtin_change_mac_o: %s. Trying to call install_name_tool instead.", e) try: cmd = ["install_name_tool", "-change", current, new, exe] - subprocess.check_call(cmd) # noqa: S603 + subprocess.check_call(cmd) except Exception: logging.fatal("Could not call install_name_tool -- you must have Apple's development tools installed") raise @@ -206,7 +210,7 @@ def mach_o_change(at_path, what, value): # noqa: C901 def do_macho(file, bits, endian): # Read Mach-O header (the magic number is assumed read by the caller) - cpu_type, cpu_sub_type, file_type, n_commands, size_of_commands, flags = read_data(file, endian, 6) + _cpu_type, _cpu_sub_type, _file_type, n_commands, _size_of_commands, _flags = read_data(file, endian, 6) # 64-bits header has one more field. if bits == 64: # noqa: PLR2004 read_data(file, endian) @@ -239,7 +243,7 @@ def do_file(file, offset=0, size=maxint): n_fat_arch = read_data(file, BIG_ENDIAN) for _ in range(n_fat_arch): # Read arch header - cpu_type, cpu_sub_type, offset, size, align = read_data(file, BIG_ENDIAN, 5) + _cpu_type, _cpu_sub_type, offset, size, _align = read_data(file, BIG_ENDIAN, 5) do_file(file, offset, size) elif magic == MH_MAGIC: do_macho(file, 32, BIG_ENDIAN) @@ -258,7 +262,20 @@ def do_file(file, offset=0, size=maxint): return mach_o_change +class CPython3macOsBrew(CPython3, CPythonPosix): + @classmethod + def can_describe(cls, interpreter): + return is_macos_brew(interpreter) and super().can_describe(interpreter) + + @classmethod + def setup_meta(cls, interpreter): # noqa: ARG003 + meta = BuiltinViaGlobalRefMeta() + meta.copy_error = "Brew disables copy creation: https://github.com/Homebrew/homebrew-core/issues/138159" + return meta + + __all__ = [ - "CPythonmacOsFramework", + "CPython3macOsBrew", "CPython3macOsFramework", + "CPythonmacOsFramework", ] diff --git a/src/virtualenv/create/via_global_ref/builtin/graalpy/__init__.py b/src/virtualenv/create/via_global_ref/builtin/graalpy/__init__.py new file mode 100644 index 000000000..8bfe887e7 --- /dev/null +++ b/src/virtualenv/create/via_global_ref/builtin/graalpy/__init__.py @@ -0,0 +1,87 @@ +from __future__ import annotations + +from abc import ABC +from pathlib import Path + +from virtualenv.create.describe import PosixSupports, WindowsSupports +from virtualenv.create.via_global_ref.builtin.ref import PathRefToDest, RefMust, RefWhen +from virtualenv.create.via_global_ref.builtin.via_global_self_do import ViaGlobalRefVirtualenvBuiltin + + +class GraalPy(ViaGlobalRefVirtualenvBuiltin, ABC): + @classmethod + def can_describe(cls, interpreter): + return interpreter.implementation == "GraalVM" and super().can_describe(interpreter) + + @classmethod + def exe_stem(cls): + return "graalpy" + + @classmethod + def exe_names(cls, interpreter): + return { + cls.exe_stem(), + "python", + f"python{interpreter.version_info.major}", + f"python{interpreter.version_info.major}.{interpreter.version_info.minor}", + } + + @classmethod + def _executables(cls, interpreter): + host = Path(interpreter.system_executable) + targets = sorted(f"{name}{cls.suffix}" for name in cls.exe_names(interpreter)) + yield host, targets, RefMust.NA, RefWhen.ANY + + @classmethod + def sources(cls, interpreter): + yield from super().sources(interpreter) + python_dir = Path(interpreter.system_executable).resolve().parent + if python_dir.name in {"bin", "Scripts"}: + python_dir = python_dir.parent + + native_lib = cls._native_lib(python_dir / "lib", interpreter.platform) + if native_lib.exists(): + yield PathRefToDest(native_lib, dest=lambda self, s: self.bin_dir.parent / "lib" / s.name) + + for jvm_dir_name in ("jvm", "jvmlibs", "modules"): + jvm_dir = python_dir / jvm_dir_name + if jvm_dir.exists(): + yield PathRefToDest(jvm_dir, dest=lambda self, s: self.bin_dir.parent / s.name) + + @classmethod + def _shared_libs(cls, python_dir): + raise NotImplementedError + + def set_pyenv_cfg(self): + super().set_pyenv_cfg() + # GraalPy 24.0 and older had home without the bin + version = self.interpreter.version_info + if version.major == 3 and version.minor <= 10: # noqa: PLR2004 + home = Path(self.pyenv_cfg["home"]) + if home.name == "bin": + self.pyenv_cfg["home"] = str(home.parent) + + +class GraalPyPosix(GraalPy, PosixSupports): + @classmethod + def _native_lib(cls, lib_dir, platform): + if platform == "darwin": + return lib_dir / "libpythonvm.dylib" + return lib_dir / "libpythonvm.so" + + +class GraalPyWindows(GraalPy, WindowsSupports): + @classmethod + def _native_lib(cls, lib_dir, _platform): + return lib_dir / "pythonvm.dll" + + def set_pyenv_cfg(self): + # GraalPy needs an additional entry in pyvenv.cfg on Windows + super().set_pyenv_cfg() + self.pyenv_cfg["venvlauncher_command"] = self.interpreter.system_executable + + +__all__ = [ + "GraalPyPosix", + "GraalPyWindows", +] diff --git a/src/virtualenv/create/via_global_ref/builtin/pypy/common.py b/src/virtualenv/create/via_global_ref/builtin/pypy/common.py index c7f91e32b..ca4b45ff1 100644 --- a/src/virtualenv/create/via_global_ref/builtin/pypy/common.py +++ b/src/virtualenv/create/via_global_ref/builtin/pypy/common.py @@ -7,7 +7,7 @@ from virtualenv.create.via_global_ref.builtin.via_global_self_do import ViaGlobalRefVirtualenvBuiltin -class PyPy(ViaGlobalRefVirtualenvBuiltin, metaclass=abc.ABCMeta): +class PyPy(ViaGlobalRefVirtualenvBuiltin, abc.ABC): @classmethod def can_describe(cls, interpreter): return interpreter.implementation == "PyPy" and super().can_describe(interpreter) diff --git a/src/virtualenv/create/via_global_ref/builtin/pypy/pypy3.py b/src/virtualenv/create/via_global_ref/builtin/pypy/pypy3.py index 39f0ed5bb..fa61ebc95 100644 --- a/src/virtualenv/create/via_global_ref/builtin/pypy/pypy3.py +++ b/src/virtualenv/create/via_global_ref/builtin/pypy/pypy3.py @@ -9,7 +9,7 @@ from .common import PyPy -class PyPy3(PyPy, Python3Supports, metaclass=abc.ABCMeta): +class PyPy3(PyPy, Python3Supports, abc.ABC): @classmethod def exe_stem(cls): return "pypy3" diff --git a/src/virtualenv/create/via_global_ref/builtin/ref.py b/src/virtualenv/create/via_global_ref/builtin/ref.py index d3dca5d09..e2fd45ffe 100644 --- a/src/virtualenv/create/via_global_ref/builtin/ref.py +++ b/src/virtualenv/create/via_global_ref/builtin/ref.py @@ -7,7 +7,7 @@ from __future__ import annotations import os -from abc import ABCMeta, abstractmethod +from abc import ABC, abstractmethod from collections import OrderedDict from stat import S_IXGRP, S_IXOTH, S_IXUSR @@ -27,7 +27,7 @@ class RefWhen: SYMLINK = "symlink" -class PathRef(metaclass=ABCMeta): +class PathRef(ABC): """Base class that checks if a file reference can be symlink/copied.""" FS_SUPPORTS_SYMLINK = fs_supports_symlink() @@ -91,7 +91,7 @@ def method(self, symlinks): return symlink if symlinks else copy -class ExePathRef(PathRef, metaclass=ABCMeta): +class ExePathRef(PathRef, ABC): """Base class that checks if a executable can be references via symlink/copy.""" def __init__(self, src, must=RefMust.NA, when=RefWhen.ANY) -> None: @@ -137,7 +137,7 @@ def run(self, creator, symlinks): class ExePathRefToDest(PathRefToDest, ExePathRef): """Link a exe path on the file system.""" - def __init__(self, src, targets, dest, must=RefMust.NA, when=RefWhen.ANY) -> None: # noqa: PLR0913 + def __init__(self, src, targets, dest, must=RefMust.NA, when=RefWhen.ANY) -> None: ExePathRef.__init__(self, src, must, when) PathRefToDest.__init__(self, src, dest, must, when) if not self.FS_CASE_SENSITIVE: @@ -171,8 +171,8 @@ def __repr__(self) -> str: __all__ = [ "ExePathRef", "ExePathRefToDest", - "PathRefToDest", "PathRef", - "RefWhen", + "PathRefToDest", "RefMust", + "RefWhen", ] diff --git a/src/virtualenv/create/via_global_ref/builtin/via_global_self_do.py b/src/virtualenv/create/via_global_ref/builtin/via_global_self_do.py index 3c0f9cf8c..2f7f2f11a 100644 --- a/src/virtualenv/create/via_global_ref/builtin/via_global_self_do.py +++ b/src/virtualenv/create/via_global_ref/builtin/via_global_self_do.py @@ -1,6 +1,6 @@ from __future__ import annotations -from abc import ABCMeta +from abc import ABC from virtualenv.create.via_global_ref.api import ViaGlobalRefApi, ViaGlobalRefMeta from virtualenv.create.via_global_ref.builtin.ref import ( @@ -19,7 +19,7 @@ def __init__(self) -> None: self.sources = [] -class ViaGlobalRefVirtualenvBuiltin(ViaGlobalRefApi, VirtualenvBuiltin, metaclass=ABCMeta): +class ViaGlobalRefVirtualenvBuiltin(ViaGlobalRefApi, VirtualenvBuiltin, ABC): def __init__(self, options, interpreter) -> None: super().__init__(options, interpreter) self._sources = getattr(options.meta, "sources", None) # if we're created as a describer this might be missing diff --git a/src/virtualenv/create/via_global_ref/venv.py b/src/virtualenv/create/via_global_ref/venv.py index cc73a6f27..45866d676 100644 --- a/src/virtualenv/create/via_global_ref/venv.py +++ b/src/virtualenv/create/via_global_ref/venv.py @@ -10,14 +10,17 @@ from virtualenv.util.subprocess import run_cmd from .api import ViaGlobalRefApi, ViaGlobalRefMeta +from .builtin.cpython.mac_os import CPython3macOsBrew from .builtin.pypy.pypy3 import Pypy3Windows +LOGGER = logging.getLogger(__name__) + class Venv(ViaGlobalRefApi): def __init__(self, options, interpreter) -> None: self.describe = options.describe super().__init__(options, interpreter) - current = PythonInfo.current() + current = PythonInfo.current(options.app_data, options.cache) self.can_be_inline = interpreter is current and interpreter.executable == interpreter.system_executable self._context = None @@ -27,6 +30,8 @@ def _args(self): @classmethod def can_create(cls, interpreter): if interpreter.has_venv: + if CPython3macOsBrew.can_describe(interpreter): + return CPython3macOsBrew.setup_meta(interpreter) meta = ViaGlobalRefMeta() if interpreter.platform == "win32": meta = handle_store_python(meta, interpreter) @@ -55,7 +60,7 @@ def executables_for_win_pypy_less_v37(self): exe.run(creator, self.symlinks) def create_inline(self): - from venv import EnvBuilder + from venv import EnvBuilder # noqa: PLC0415 builder = EnvBuilder( system_site_packages=self.enable_system_site_package, @@ -67,7 +72,7 @@ def create_inline(self): def create_via_sub_process(self): cmd = self.get_host_create_cmd() - logging.info("using host built-in venv to create via %s", " ".join(cmd)) + LOGGER.info("using host built-in venv to create via %s", " ".join(cmd)) code, out, err = run_cmd(cmd) if code != 0: raise ProcessCallFailedError(code, out, err, cmd) @@ -76,8 +81,7 @@ def get_host_create_cmd(self): cmd = [self.interpreter.system_executable, "-m", "venv", "--without-pip"] if self.enable_system_site_package: cmd.append("--system-site-packages") - cmd.append("--symlinks" if self.symlinks else "--copies") - cmd.append(str(self.dest)) + cmd.extend(("--symlinks" if self.symlinks else "--copies", str(self.dest))) return cmd def set_pyenv_cfg(self): @@ -90,7 +94,7 @@ def __getattribute__(self, item): describe = object.__getattribute__(self, "describe") if describe is not None and hasattr(describe, item): element = getattr(describe, item) - if not callable(element) or item in ("script",): + if not callable(element) or item == "script": return element return object.__getattribute__(self, item) diff --git a/src/virtualenv/discovery/app_data.py b/src/virtualenv/discovery/app_data.py new file mode 100644 index 000000000..90de58284 --- /dev/null +++ b/src/virtualenv/discovery/app_data.py @@ -0,0 +1,23 @@ +from __future__ import annotations + +from contextlib import contextmanager +from typing import TYPE_CHECKING, Any, ContextManager, Protocol + +if TYPE_CHECKING: + from pathlib import Path + + +class AppData(Protocol): + """Protocol for application data store.""" + + def py_info(self, path: Path) -> Any: ... + + def py_info_clear(self) -> None: ... + + @contextmanager + def ensure_extracted(self, path: Path, to_folder: Path | None = None) -> ContextManager[Path]: ... + + @contextmanager + def extract(self, path: Path, to_folder: Path | None = None) -> ContextManager[Path]: ... + + def close(self) -> None: ... diff --git a/src/virtualenv/discovery/builtin.py b/src/virtualenv/discovery/builtin.py index 92d96a91b..3c6dcb2fd 100644 --- a/src/virtualenv/discovery/builtin.py +++ b/src/virtualenv/discovery/builtin.py @@ -3,23 +3,40 @@ import logging import os import sys +from contextlib import suppress +from pathlib import Path +from typing import TYPE_CHECKING -from virtualenv.info import IS_WIN +from platformdirs import user_data_path from .discover import Discover +from .info import IS_WIN, fs_path_id from .py_info import PythonInfo from .py_spec import PythonSpec +if TYPE_CHECKING: + from argparse import ArgumentParser + from collections.abc import Callable, Generator, Iterable, Mapping, Sequence + + from .app_data import AppData +LOGGER = logging.getLogger(__name__) + class Builtin(Discover): - def __init__(self, options) -> None: - super().__init__(options) - self.python_spec = options.python if options.python else [sys.executable] + python_spec: Sequence[str] + app_data: AppData + try_first_with: Sequence[str] + + def __init__(self, options, cache=None) -> None: + super().__init__(options, cache) + self.python_spec = options.python or [sys.executable] + if self._env.get("VIRTUALENV_PYTHON"): + self.python_spec = self.python_spec[1:] + self.python_spec[:1] # Rotate the list self.app_data = options.app_data self.try_first_with = options.try_first_with @classmethod - def add_parser_arguments(cls, parser): + def add_parser_arguments(cls, parser: ArgumentParser) -> None: parser.add_argument( "-p", "--python", @@ -41,9 +58,9 @@ def add_parser_arguments(cls, parser): help="try first these interpreters before starting the discovery", ) - def run(self): + def run(self) -> PythonInfo | None: for python_spec in self.python_spec: - result = get_interpreter(python_spec, self.try_first_with, self.app_data, self._env) + result = get_interpreter(python_spec, self.try_first_with, self.app_data, self.cache, self._env) if result is not None: return result return None @@ -53,26 +70,53 @@ def __repr__(self) -> str: return f"{self.__class__.__name__} discover of python_spec={spec!r}" -def get_interpreter(key, try_first_with, app_data=None, env=None): +def get_interpreter( + key, + try_first_with: Iterable[str], + app_data: AppData | None = None, + cache=None, + env: Mapping[str, str] | None = None, +) -> PythonInfo | None: spec = PythonSpec.from_string_spec(key) - logging.info("find interpreter for spec %r", spec) + LOGGER.info("find interpreter for spec %r", spec) proposed_paths = set() env = os.environ if env is None else env - for interpreter, impl_must_match in propose_interpreters(spec, try_first_with, app_data, env): + for interpreter, impl_must_match in propose_interpreters(spec, try_first_with, app_data, cache, env): key = interpreter.system_executable, impl_must_match if key in proposed_paths: continue - logging.info("proposed %s", interpreter) + LOGGER.info("proposed %s", interpreter) if interpreter.satisfies(spec, impl_must_match): - logging.debug("accepted %s", interpreter) + LOGGER.debug("accepted %s", interpreter) return interpreter proposed_paths.add(key) return None -def propose_interpreters(spec, try_first_with, app_data, env=None): # noqa: C901, PLR0912 - # 0. try with first +def propose_interpreters( # noqa: C901, PLR0912, PLR0915 + spec: PythonSpec, + try_first_with: Iterable[str], + app_data: AppData | None = None, + cache=None, + env: Mapping[str, str] | None = None, +) -> Generator[tuple[PythonInfo, bool], None, None]: + # 0. if it's a path and exists, and is absolute path, this is the only option we consider env = os.environ if env is None else env + tested_exes: set[str] = set() + if spec.is_abs: + try: + os.lstat(spec.path) # Windows Store Python does not work with os.path.exists, but does for os.lstat + except OSError: + pass + else: + exe_raw = os.path.abspath(spec.path) + exe_id = fs_path_id(exe_raw) + if exe_id not in tested_exes: + tested_exes.add(exe_id) + yield PythonInfo.from_exe(exe_raw, app_data, cache, env=env), True + return + + # 1. try with first for py_exe in try_first_with: path = os.path.abspath(py_exe) try: @@ -80,58 +124,92 @@ def propose_interpreters(spec, try_first_with, app_data, env=None): # noqa: C90 except OSError: pass else: - yield PythonInfo.from_exe(os.path.abspath(path), app_data, env=env), True + exe_raw = os.path.abspath(path) + exe_id = fs_path_id(exe_raw) + if exe_id in tested_exes: + continue + tested_exes.add(exe_id) + yield PythonInfo.from_exe(exe_raw, app_data, cache, env=env), True # 1. if it's a path and exists if spec.path is not None: try: os.lstat(spec.path) # Windows Store Python does not work with os.path.exists, but does for os.lstat except OSError: - if spec.is_abs: - raise + pass else: - yield PythonInfo.from_exe(os.path.abspath(spec.path), app_data, env=env), True + exe_raw = os.path.abspath(spec.path) + exe_id = fs_path_id(exe_raw) + if exe_id not in tested_exes: + tested_exes.add(exe_id) + yield PythonInfo.from_exe(exe_raw, app_data, cache, env=env), True if spec.is_abs: return else: # 2. otherwise try with the current - yield PythonInfo.current_system(app_data), True + current_python = PythonInfo.current_system(app_data, cache) + exe_raw = str(current_python.executable) + exe_id = fs_path_id(exe_raw) + if exe_id not in tested_exes: + tested_exes.add(exe_id) + yield current_python, True # 3. otherwise fallback to platform default logic if IS_WIN: - from .windows import propose_interpreters + from .windows import propose_interpreters # noqa: PLC0415 - for interpreter in propose_interpreters(spec, app_data, env): + for interpreter in propose_interpreters(spec, app_data, cache, env): + exe_raw = str(interpreter.executable) + exe_id = fs_path_id(exe_raw) + if exe_id in tested_exes: + continue + tested_exes.add(exe_id) yield interpreter, True - # finally just find on path, the path order matters (as the candidates are less easy to control by end user) - paths = get_paths(env) - tested_exes = set() - for pos, path in enumerate(paths): - path_str = str(path) - logging.debug(LazyPathDump(pos, path_str, env)) - for candidate, match in possible_specs(spec): - found = check_path(candidate, path_str) - if found is not None: - exe = os.path.abspath(found) - if exe not in tested_exes: - tested_exes.add(exe) - interpreter = PathPythonInfo.from_exe(exe, app_data, raise_on_error=False, env=env) - if interpreter is not None: - yield interpreter, match - - -def get_paths(env): + + # try to find on path, the path order matters (as the candidates are less easy to control by end user) + find_candidates = path_exe_finder(spec) + for pos, path in enumerate(get_paths(env)): + LOGGER.debug(LazyPathDump(pos, path, env)) + for exe, impl_must_match in find_candidates(path): + exe_raw = str(exe) + exe_id = fs_path_id(exe_raw) + if exe_id in tested_exes: + continue + tested_exes.add(exe_id) + interpreter = PathPythonInfo.from_exe(exe_raw, app_data, cache, raise_on_error=False, env=env) + if interpreter is not None: + yield interpreter, impl_must_match + + # otherwise try uv-managed python (~/.local/share/uv/python or platform equivalent) + if uv_python_dir := os.getenv("UV_PYTHON_INSTALL_DIR"): + uv_python_path = Path(uv_python_dir).expanduser() + elif xdg_data_home := os.getenv("XDG_DATA_HOME"): + uv_python_path = Path(xdg_data_home).expanduser() / "uv" / "python" + else: + uv_python_path = user_data_path("uv") / "python" + + for exe_path in uv_python_path.glob("*/bin/python"): + interpreter = PathPythonInfo.from_exe(str(exe_path), app_data, cache, raise_on_error=False, env=env) + if interpreter is not None: + yield interpreter, True + + +def get_paths(env: Mapping[str, str]) -> Generator[Path, None, None]: path = env.get("PATH", None) if path is None: try: path = os.confstr("CS_PATH") except (AttributeError, ValueError): path = os.defpath - return [] if not path else [p for p in path.split(os.pathsep) if os.path.exists(p)] + if path: + for p in map(Path, path.split(os.pathsep)): + with suppress(OSError): + if p.is_dir() and next(p.iterdir(), None): + yield p class LazyPathDump: - def __init__(self, pos, path, env) -> None: + def __init__(self, pos: int, path: Path, env: Mapping[str, str]) -> None: self.pos = pos self.path = path self.env = env @@ -140,35 +218,44 @@ def __repr__(self) -> str: content = f"discover PATH[{self.pos}]={self.path}" if self.env.get("_VIRTUALENV_DEBUG"): # this is the over the board debug content += " with =>" - for file_name in os.listdir(self.path): + for file_path in self.path.iterdir(): try: - file_path = os.path.join(self.path, file_name) - if os.path.isdir(file_path) or not os.access(file_path, os.X_OK): + if file_path.is_dir(): + continue + if IS_WIN: + pathext = self.env.get("PATHEXT", ".COM;.EXE;.BAT;.CMD").split(";") + if not any(file_path.name.upper().endswith(ext) for ext in pathext): + continue + elif not (file_path.stat().st_mode & os.X_OK): continue except OSError: pass content += " " - content += file_name + content += file_path.name return content -def check_path(candidate, path): - _, ext = os.path.splitext(candidate) - if sys.platform == "win32" and ext != ".exe": - candidate = candidate + ".exe" - if os.path.isfile(candidate): - return candidate - candidate = os.path.join(path, candidate) - if os.path.isfile(candidate): - return candidate - return None +def path_exe_finder(spec: PythonSpec) -> Callable[[Path], Generator[tuple[Path, bool], None, None]]: + """Given a spec, return a function that can be called on a path to find all matching files in it.""" + pat = spec.generate_re(windows=sys.platform == "win32") + direct = spec.str_spec + if sys.platform == "win32": + direct = f"{direct}.exe" + + def path_exes(path: Path) -> Generator[tuple[Path, bool], None, None]: + # 4. then maybe it's something exact on PATH - if it was direct lookup implementation no longer counts + direct_path = path / direct + if direct_path.exists(): + yield direct_path, False + # 5. or from the spec we can deduce if a name on path matches + for exe in path.iterdir(): + match = pat.fullmatch(exe.name) + if match: + # the implementation must match when we find โ€œpython[ver]โ€ + yield exe.absolute(), match["impl"] == "python" -def possible_specs(spec): - # 4. then maybe it's something exact on PATH - if it was direct lookup implementation no longer counts - yield spec.str_spec, False - # 5. or from the spec we can deduce a name on path that matches - yield from spec.generate_names() + return path_exes class PathPythonInfo(PythonInfo): @@ -176,7 +263,7 @@ class PathPythonInfo(PythonInfo): __all__ = [ - "get_interpreter", "Builtin", "PathPythonInfo", + "get_interpreter", ] diff --git a/src/virtualenv/discovery/cache.py b/src/virtualenv/discovery/cache.py new file mode 100644 index 000000000..eaf24cc14 --- /dev/null +++ b/src/virtualenv/discovery/cache.py @@ -0,0 +1,18 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, Protocol + +if TYPE_CHECKING: + from pathlib import Path + + +class Cache(Protocol): + """A protocol for a cache.""" + + def get(self, path: Path) -> Any: ... + + def set(self, path: Path, data: Any) -> None: ... + + def remove(self, path: Path) -> None: ... + + def clear(self) -> None: ... diff --git a/src/virtualenv/discovery/cached_py_info.py b/src/virtualenv/discovery/cached_py_info.py index 19e938f6c..645a5eb36 100644 --- a/src/virtualenv/discovery/cached_py_info.py +++ b/src/virtualenv/discovery/cached_py_info.py @@ -7,78 +7,103 @@ from __future__ import annotations +import hashlib +import importlib.util import logging import os import random +import subprocess import sys from collections import OrderedDict from pathlib import Path from shlex import quote from string import ascii_lowercase, ascii_uppercase, digits -from subprocess import Popen +from typing import TYPE_CHECKING -from virtualenv.app_data import AppDataDisabled -from virtualenv.discovery.py_info import PythonInfo -from virtualenv.util.subprocess import subprocess +from .py_info import PythonInfo + +if TYPE_CHECKING: + from .app_data import AppData + from .cache import Cache _CACHE = OrderedDict() _CACHE[Path(sys.executable)] = PythonInfo() - - -def from_exe(cls, app_data, exe, env=None, raise_on_error=True, ignore_cache=False): # noqa: FBT002, PLR0913 +LOGGER = logging.getLogger(__name__) + + +def from_exe( # noqa: PLR0913 + cls, + app_data: AppData, + exe: str, + env: dict[str, str] | None = None, + *, + raise_on_error: bool = True, + ignore_cache: bool = False, + cache: Cache, +) -> PythonInfo | None: env = os.environ if env is None else env - result = _get_from_cache(cls, app_data, exe, env, ignore_cache=ignore_cache) + result = _get_from_cache(cls, app_data, exe, env, cache, ignore_cache=ignore_cache) if isinstance(result, Exception): if raise_on_error: raise result - logging.info("%s", result) + LOGGER.info("%s", result) result = None return result -def _get_from_cache(cls, app_data, exe, env, ignore_cache=True): # noqa: FBT002 +def _get_from_cache(cls, app_data: AppData, exe: str, env, cache: Cache, *, ignore_cache: bool) -> PythonInfo: # noqa: PLR0913 # note here we cannot resolve symlinks, as the symlink may trigger different prefix information if there's a # pyenv.cfg somewhere alongside on python3.5+ exe_path = Path(exe) if not ignore_cache and exe_path in _CACHE: # check in the in-memory cache result = _CACHE[exe_path] else: # otherwise go through the app data cache - py_info = _get_via_file_cache(cls, app_data, exe_path, exe, env) - result = _CACHE[exe_path] = py_info + result = _CACHE[exe_path] = _get_via_file_cache(cls, app_data, exe_path, exe, env, cache) # independent if it was from the file or in-memory cache fix the original executable location if isinstance(result, PythonInfo): result.executable = exe return result -def _get_via_file_cache(cls, app_data, path, exe, env): - path_text = str(path) +def _get_via_file_cache(cls, app_data: AppData, path: Path, exe: str, env, cache: Cache) -> PythonInfo: # noqa: PLR0913 + # 1. get the hash of the probing script + spec = importlib.util.find_spec("virtualenv.discovery.py_info") + script = Path(spec.origin) + try: + py_info_hash = hashlib.sha256(script.read_bytes()).hexdigest() + except OSError: + py_info_hash = None + + # 2. get the mtime of the python executable try: path_modified = path.stat().st_mtime except OSError: path_modified = -1 - if app_data is None: - app_data = AppDataDisabled() - py_info, py_info_store = None, app_data.py_info(path) - with py_info_store.locked(): - if py_info_store.exists(): # if exists and matches load - data = py_info_store.read() - of_path, of_st_mtime, of_content = data["path"], data["st_mtime"], data["content"] - if of_path == path_text and of_st_mtime == path_modified: - py_info = cls._from_dict(of_content.copy()) - sys_exe = py_info.system_executable - if sys_exe is not None and not os.path.exists(sys_exe): - py_info_store.remove() - py_info = None - else: - py_info_store.remove() - if py_info is None: # if not loaded run and save - failure, py_info = _run_subprocess(cls, exe, app_data, env) - if failure is None: - data = {"st_mtime": path_modified, "path": path_text, "content": py_info._to_dict()} # noqa: SLF001 - py_info_store.write(data) - else: - py_info = failure + + # 3. check if we have a valid cache entry + py_info = None + data = cache.get(path) + if data is not None: + if data.get("path") == str(path) and data.get("st_mtime") == path_modified and data.get("hash") == py_info_hash: + py_info = cls._from_dict(data.get("content")) + sys_exe = py_info.system_executable + if sys_exe is not None and not os.path.exists(sys_exe): + py_info = None # if system executable is no longer there, this is not valid + if py_info is None: + cache.remove(path) # if cache is invalid, remove it + + if py_info is None: # if not loaded run and save + failure, py_info = _run_subprocess(cls, exe, app_data, env) + if failure is None: + data = { + "st_mtime": path_modified, + "path": str(path), + "content": py_info._to_dict(), # noqa: SLF001 + "hash": py_info_hash, + } + cache.set(path, data) + else: + py_info = failure return py_info @@ -87,11 +112,17 @@ def _get_via_file_cache(cls, app_data, path, exe, env): def gen_cookie(): return "".join( - random.choice(f"{ascii_lowercase}{ascii_uppercase}{digits}") for _ in range(COOKIE_LENGTH) # noqa: S311 + random.choice(f"{ascii_lowercase}{ascii_uppercase}{digits}") # noqa: S311 + for _ in range(COOKIE_LENGTH) ) -def _run_subprocess(cls, exe, app_data, env): +def _run_subprocess( + cls, + exe: str, + app_data: AppData, + env: dict[str, str], +) -> tuple[Exception | None, PythonInfo | None]: py_info_script = Path(os.path.abspath(__file__)).parent / "py_info.py" # Cookies allow to split the serialized stdout output generated by the script collecting the info from the output # generated by something else. The right way to deal with it is to create an anonymous pipe and pass its descriptor @@ -103,15 +134,15 @@ def _run_subprocess(cls, exe, app_data, env): start_cookie = gen_cookie() end_cookie = gen_cookie() - with app_data.ensure_extracted(py_info_script) as py_info_script: - cmd = [exe, str(py_info_script), start_cookie, end_cookie] + with app_data.ensure_extracted(py_info_script) as py_info_script_path: + cmd = [exe, str(py_info_script_path), start_cookie, end_cookie] # prevent sys.prefix from leaking into the child process - see https://bugs.python.org/issue22490 env = env.copy() env.pop("__PYVENV_LAUNCHER__", None) - logging.debug("get interpreter info via cmd: %s", LogCmd(cmd)) + LOGGER.debug("get interpreter info via cmd: %s", LogCmd(cmd)) try: - process = Popen( - cmd, # noqa: S603 + process = subprocess.Popen( + cmd, universal_newlines=True, stdin=subprocess.PIPE, stderr=subprocess.PIPE, @@ -165,8 +196,10 @@ def __repr__(self) -> str: return cmd_repr -def clear(app_data): - app_data.py_info_clear() +def clear(cache: Cache | None = None) -> None: + """Clear the cache.""" + if cache is not None: + cache.clear() _CACHE.clear() diff --git a/src/virtualenv/discovery/discover.py b/src/virtualenv/discovery/discover.py index b74ee6c01..de1b5fd0b 100644 --- a/src/virtualenv/discovery/discover.py +++ b/src/virtualenv/discovery/discover.py @@ -1,9 +1,9 @@ from __future__ import annotations -from abc import ABCMeta, abstractmethod +from abc import ABC, abstractmethod -class Discover(metaclass=ABCMeta): +class Discover(ABC): """Discover and provide the requested Python interpreter.""" @classmethod @@ -15,7 +15,7 @@ def add_parser_arguments(cls, parser): """ raise NotImplementedError - def __init__(self, options) -> None: + def __init__(self, options, cache=None) -> None: """ Create a new discovery mechanism. @@ -24,6 +24,7 @@ def __init__(self, options) -> None: self._has_run = False self._interpreter = None self._env = options.env + self.cache = cache @abstractmethod def run(self): diff --git a/src/virtualenv/discovery/info.py b/src/virtualenv/discovery/info.py new file mode 100644 index 000000000..c786a6a98 --- /dev/null +++ b/src/virtualenv/discovery/info.py @@ -0,0 +1,33 @@ +from __future__ import annotations + +import logging +import os +import sys +import tempfile + +_FS_CASE_SENSITIVE = None +LOGGER = logging.getLogger(__name__) +IS_WIN = sys.platform == "win32" + + +def fs_is_case_sensitive(): + """Check if the file system is case-sensitive.""" + global _FS_CASE_SENSITIVE # noqa: PLW0603 + + if _FS_CASE_SENSITIVE is None: + with tempfile.NamedTemporaryFile(prefix="TmP") as tmp_file: + _FS_CASE_SENSITIVE = not os.path.exists(tmp_file.name.lower()) + LOGGER.debug("filesystem is %scase-sensitive", "" if _FS_CASE_SENSITIVE else "not ") + return _FS_CASE_SENSITIVE + + +def fs_path_id(path: str) -> str: + """Get a case-normalized path identifier.""" + return path.casefold() if fs_is_case_sensitive() else path + + +__all__ = ( + "IS_WIN", + "fs_is_case_sensitive", + "fs_path_id", +) diff --git a/src/virtualenv/discovery/py_info.py b/src/virtualenv/discovery/py_info.py index 8b59130a4..797f88bdb 100644 --- a/src/virtualenv/discovery/py_info.py +++ b/src/virtualenv/discovery/py_info.py @@ -11,6 +11,7 @@ import os import platform import re +import struct import sys import sysconfig import warnings @@ -18,6 +19,7 @@ from string import digits VersionInfo = namedtuple("VersionInfo", ["major", "minor", "micro", "releaselevel", "serial"]) # noqa: PYI024 +LOGGER = logging.getLogger(__name__) def _get_path_extensions(): @@ -43,7 +45,10 @@ def abs_path(v): # this is a tuple in earlier, struct later, unify to our own named tuple self.version_info = VersionInfo(*sys.version_info) - self.architecture = 64 if sys.maxsize > 2**32 else 32 + # Use the same implementation as found in stdlib platform.architecture + # to account for platforms where the maximum integer is not equal the + # pointer size. + self.architecture = 32 if struct.calcsize("P") == 4 else 64 # noqa: PLR2004 # Used to determine some file names. # See `CPython3Windows.python_zip()`. @@ -51,6 +56,7 @@ def abs_path(v): self.version = sys.version self.os = os.name + self.free_threaded = sysconfig.get_config_var("Py_GIL_DISABLED") == 1 # information about the prefix - determines python home self.prefix = abs_path(getattr(sys, "prefix", None)) # prefix we think @@ -112,12 +118,16 @@ def abs_path(v): config_var_keys = set() for element in self.sysconfig_paths.values(): - for k in _CONF_VAR_RE.findall(element): - config_var_keys.add(k[1:-1]) + config_var_keys.update(k[1:-1] for k in _CONF_VAR_RE.findall(element)) config_var_keys.add("PYTHONFRAMEWORK") self.sysconfig_vars = {i: sysconfig.get_config_var(i or "") for i in config_var_keys} + if "TCL_LIBRARY" in os.environ: + self.tcl_lib, self.tk_lib = self._get_tcl_tk_libs() + else: + self.tcl_lib, self.tk_lib = None, None + confs = { k: (self.system_prefix if v is not None and v.startswith(self.prefix) else v) for k, v in self.sysconfig_vars.items() @@ -125,38 +135,113 @@ def abs_path(v): self.system_stdlib = self.sysconfig_path("stdlib", confs) self.system_stdlib_platform = self.sysconfig_path("platstdlib", confs) self.max_size = getattr(sys, "maxsize", getattr(sys, "maxint", None)) - self._creators = None + + @staticmethod + def _get_tcl_tk_libs(): + """ + Detects the tcl and tk libraries using tkinter. + + This works reliably but spins up tkinter, which is heavy if you don't need it. + """ + tcl_lib, tk_lib = None, None + try: + import tkinter as tk # noqa: PLC0415 + except ImportError: + pass + else: + try: + tcl = tk.Tcl() + tcl_lib = tcl.eval("info library") + + # Try to get TK library path directly first + try: + tk_lib = tcl.eval("set tk_library") + if tk_lib and os.path.isdir(tk_lib): + pass # We found it directly + else: + tk_lib = None # Reset if invalid + except tk.TclError: + tk_lib = None + + # If direct query failed, try constructing the path + if tk_lib is None: + tk_version = tcl.eval("package require Tk") + tcl_parent = os.path.dirname(tcl_lib) + + # Try different version formats + version_variants = [ + tk_version, # Full version like "8.6.12" + ".".join(tk_version.split(".")[:2]), # Major.minor like "8.6" + tk_version.split(".")[0], # Just major like "8" + ] + + for version in version_variants: + tk_lib_path = os.path.join(tcl_parent, f"tk{version}") + if not os.path.isdir(tk_lib_path): + continue + # Validate it's actually a TK directory + if os.path.exists(os.path.join(tk_lib_path, "tk.tcl")): + tk_lib = tk_lib_path + break + + except tk.TclError: + pass + + return tcl_lib, tk_lib def _fast_get_system_executable(self): """Try to get the system executable by just looking at properties.""" - if self.real_prefix or ( - self.base_prefix is not None and self.base_prefix != self.prefix - ): # if this is a virtual environment - if self.real_prefix is None: - base_executable = getattr(sys, "_base_executable", None) # some platforms may set this to help us - if base_executable is not None: # noqa: SIM102 # use the saved system executable if present - if sys.executable != base_executable: # we know we're in a virtual environment, cannot be us - if os.path.exists(base_executable): - return base_executable - # Python may return "python" because it was invoked from the POSIX virtual environment - # however some installs/distributions do not provide a version-less "python" binary in - # the system install location (see PEP 394) so try to fallback to a versioned binary. - # - # Gate this to Python 3.11 as `sys._base_executable` path resolution is now relative to - # the 'home' key from pyvenv.cfg which often points to the system install location. - major, minor = self.version_info.major, self.version_info.minor - if self.os == "posix" and (major, minor) >= (3, 11): - # search relative to the directory of sys._base_executable - base_dir = os.path.dirname(base_executable) - for base_executable in [ - os.path.join(base_dir, exe) for exe in (f"python{major}", f"python{major}.{minor}") - ]: - if os.path.exists(base_executable): - return base_executable - return None # in this case we just can't tell easily without poking around FS and calling them, bail # if we're not in a virtual environment, this is already a system python, so return the original executable # note we must choose the original and not the pure executable as shim scripts might throw us off - return self.original_executable + if not (self.real_prefix or (self.base_prefix is not None and self.base_prefix != self.prefix)): + return self.original_executable + + # if this is NOT a virtual environment, can't determine easily, bail out + if self.real_prefix is not None: + return None + + base_executable = getattr(sys, "_base_executable", None) # some platforms may set this to help us + if base_executable is None: # use the saved system executable if present + return None + + # we know we're in a virtual environment, can not be us + if sys.executable == base_executable: + return None + + # We're not in a venv and base_executable exists; use it directly + if os.path.exists(base_executable): + return base_executable + + # Try fallback for POSIX virtual environments + return self._try_posix_fallback_executable(base_executable) + + def _try_posix_fallback_executable(self, base_executable): + """ + Try to find a versioned Python binary as fallback for POSIX virtual environments. + + Python may return "python" because it was invoked from the POSIX virtual environment + however some installs/distributions do not provide a version-less "python" binary in + the system install location (see PEP 394) so try to fallback to a versioned binary. + + Gate this to Python 3.11 as `sys._base_executable` path resolution is now relative to + the 'home' key from pyvenv.cfg which often points to the system install location. + """ + major, minor = self.version_info.major, self.version_info.minor + if self.os != "posix" or (major, minor) < (3, 11): + return None + + # search relative to the directory of sys._base_executable + base_dir = os.path.dirname(base_executable) + candidates = [f"python{major}", f"python{major}.{minor}"] + if self.implementation == "PyPy": + candidates.extend(["pypy", "pypy3", f"pypy{major}", f"pypy{major}.{minor}"]) + + for candidate in candidates: + full_path = os.path.join(base_dir, candidate) + if os.path.exists(full_path): + return full_path + + return None # in this case we just can't tell easily without poking around FS and calling them, bail def install_path(self, key): result = self.distutils_install.get(key) @@ -175,8 +260,8 @@ def _distutils_install(): with warnings.catch_warnings(): # disable warning for PEP-632 warnings.simplefilter("ignore") try: - from distutils import dist - from distutils.command.install import SCHEME_KEYS + from distutils import dist # noqa: PLC0415 + from distutils.command.install import SCHEME_KEYS # noqa: PLC0415 except ImportError: # if removed or not installed ignore return {} @@ -214,7 +299,9 @@ def is_venv(self): return self.base_prefix is not None def sysconfig_path(self, key, config_var=None, sep=os.sep): - pattern = self.sysconfig_paths[key] + pattern = self.sysconfig_paths.get(key) + if pattern is None: + return "" if config_var is None: config_var = self.sysconfig_vars else: @@ -223,13 +310,6 @@ def sysconfig_path(self, key, config_var=None, sep=os.sep): config_var = base return pattern.format(**config_var).replace("/", sep) - def creators(self, refresh=False): # noqa: FBT002 - if self._creators is None or refresh is True: - from virtualenv.run.plugin.creators import CreatorSelector - - self._creators = CreatorSelector.for_interpreter(self) - return self._creators - @property def system_include(self): path = self.sysconfig_path( @@ -254,9 +334,6 @@ def system_prefix(self): def system_exec_prefix(self): return self.real_prefix or self.base_exec_prefix or self.exec_prefix - def __unicode__(self): - return repr(self) - def __repr__(self) -> str: return "{}({!r})".format( self.__class__.__name__, @@ -293,17 +370,22 @@ def __str__(self) -> str: @property def spec(self): - return "{}{}-{}".format(self.implementation, ".".join(str(i) for i in self.version_info), self.architecture) + return "{}{}{}-{}".format( + self.implementation, + ".".join(str(i) for i in self.version_info), + "t" if self.free_threaded else "", + self.architecture, + ) @classmethod - def clear_cache(cls, app_data): + def clear_cache(cls, cache=None): # this method is not used by itself, so here and called functions can import stuff locally - from virtualenv.discovery.cached_py_info import clear + from virtualenv.discovery.cached_py_info import clear # noqa: PLC0415 - clear(app_data) + clear(cache) cls._cache_exe_discovery.clear() - def satisfies(self, spec, impl_must_match): # noqa: C901 + def satisfies(self, spec, impl_must_match): # noqa: C901, PLR0911 """Check if a given specification can be satisfied by the this python interpreter instance.""" if spec.path: if self.executable == os.path.abspath(spec.path): @@ -329,6 +411,9 @@ def satisfies(self, spec, impl_must_match): # noqa: C901 if spec.architecture is not None and spec.architecture != self.architecture: return False + if spec.free_threaded is not None and spec.free_threaded != self.free_threaded: + return False + for our, req in zip(self.version_info[0:3], (spec.major, spec.minor, spec.micro)): if req is not None and our is not None and our != req: return False @@ -338,23 +423,35 @@ def satisfies(self, spec, impl_must_match): # noqa: C901 _current = None @classmethod - def current(cls, app_data=None): + def current(cls, app_data, cache): """ This locates the current host interpreter information. This might be different than what we run into in case the host python has been upgraded from underneath us. """ # noqa: D205 if cls._current is None: - cls._current = cls.from_exe(sys.executable, app_data, raise_on_error=True, resolve_to_host=False) + cls._current = cls.from_exe( + sys.executable, + app_data, + cache, + raise_on_error=True, + resolve_to_host=False, + ) return cls._current @classmethod - def current_system(cls, app_data=None): + def current_system(cls, app_data, cache) -> PythonInfo: """ This locates the current host interpreter information. This might be different than what we run into in case the host python has been upgraded from underneath us. """ # noqa: D205 if cls._current_system is None: - cls._current_system = cls.from_exe(sys.executable, app_data, raise_on_error=True, resolve_to_host=True) + cls._current_system = cls.from_exe( + sys.executable, + app_data, + cache, + raise_on_error=True, + resolve_to_host=True, + ) return cls._current_system def _to_json(self): @@ -362,8 +459,7 @@ def _to_json(self): return json.dumps(self._to_dict(), indent=2) def _to_dict(self): - data = {var: (getattr(self, var) if var not in ("_creators",) else None) for var in vars(self)} - + data = {var: getattr(self, var) for var in vars(self)} data["version_info"] = data["version_info"]._asdict() # namedtuple to dictionary return data @@ -371,7 +467,8 @@ def _to_dict(self): def from_exe( # noqa: PLR0913 cls, exe, - app_data=None, + app_data, + cache, raise_on_error=True, # noqa: FBT002 ignore_cache=False, # noqa: FBT002 resolve_to_host=True, # noqa: FBT002 @@ -379,18 +476,26 @@ def from_exe( # noqa: PLR0913 ): """Given a path to an executable get the python information.""" # this method is not used by itself, so here and called functions can import stuff locally - from virtualenv.discovery.cached_py_info import from_exe + from virtualenv.discovery.cached_py_info import from_exe as from_exe_cache # noqa: PLC0415 env = os.environ if env is None else env - proposed = from_exe(cls, app_data, exe, env=env, raise_on_error=raise_on_error, ignore_cache=ignore_cache) + proposed = from_exe_cache( + cls, + app_data, + exe, + env=env, + raise_on_error=raise_on_error, + ignore_cache=ignore_cache, + cache=cache, + ) if isinstance(proposed, PythonInfo) and resolve_to_host: try: - proposed = proposed._resolve_to_system(app_data, proposed) # noqa: SLF001 - except Exception as exception: # noqa: BLE001 + proposed = proposed._resolve_to_system(app_data, proposed, cache=cache) # noqa: SLF001 + except Exception as exception: if raise_on_error: raise - logging.info("ignore %s due cannot resolve system due to %r", proposed.original_executable, exception) + LOGGER.info("ignore %s due cannot resolve system due to %r", proposed.original_executable, exception) proposed = None return proposed @@ -408,7 +513,7 @@ def _from_dict(cls, data): return result @classmethod - def _resolve_to_system(cls, app_data, target): + def _resolve_to_system(cls, app_data, target, cache): start_executable = target.executable prefixes = OrderedDict() while target.system_executable is None: @@ -416,29 +521,29 @@ def _resolve_to_system(cls, app_data, target): if prefix in prefixes: if len(prefixes) == 1: # if we're linking back to ourselves accept ourselves with a WARNING - logging.info("%r links back to itself via prefixes", target) + LOGGER.info("%r links back to itself via prefixes", target) target.system_executable = target.executable break for at, (p, t) in enumerate(prefixes.items(), start=1): - logging.error("%d: prefix=%s, info=%r", at, p, t) - logging.error("%d: prefix=%s, info=%r", len(prefixes) + 1, prefix, target) + LOGGER.error("%d: prefix=%s, info=%r", at, p, t) + LOGGER.error("%d: prefix=%s, info=%r", len(prefixes) + 1, prefix, target) msg = "prefixes are causing a circle {}".format("|".join(prefixes.keys())) raise RuntimeError(msg) prefixes[prefix] = target - target = target.discover_exe(app_data, prefix=prefix, exact=False) + target = target.discover_exe(app_data, prefix=prefix, exact=False, cache=cache) if target.executable != target.system_executable: - target = cls.from_exe(target.system_executable, app_data) + target = cls.from_exe(target.system_executable, app_data, cache) target.executable = start_executable return target _cache_exe_discovery = {} # noqa: RUF012 - def discover_exe(self, app_data, prefix, exact=True, env=None): # noqa: FBT002 + def discover_exe(self, app_data, cache, prefix, exact=True, env=None): # noqa: FBT002 key = prefix, exact if key in self._cache_exe_discovery and prefix: - logging.debug("discover exe from cache %s - exact %s: %r", prefix, exact, self._cache_exe_discovery[key]) + LOGGER.debug("discover exe from cache %s - exact %s: %r", prefix, exact, self._cache_exe_discovery[key]) return self._cache_exe_discovery[key] - logging.debug("discover exe for %s in %s", self, prefix) + LOGGER.debug("discover exe for %s in %s", self, prefix) # we don't know explicitly here, do some guess work - our executable name should tell possible_names = self._find_possible_exe_names() possible_folders = self._find_possible_folders(prefix) @@ -446,7 +551,7 @@ def discover_exe(self, app_data, prefix, exact=True, env=None): # noqa: FBT002 env = os.environ if env is None else env for folder in possible_folders: for name in possible_names: - info = self._check_exe(app_data, folder, name, exact, discovered, env) + info = self._check_exe(app_data, cache, folder, name, exact, discovered, env) if info is not None: self._cache_exe_discovery[key] = info return info @@ -454,16 +559,23 @@ def discover_exe(self, app_data, prefix, exact=True, env=None): # noqa: FBT002 info = self._select_most_likely(discovered, self) folders = os.pathsep.join(possible_folders) self._cache_exe_discovery[key] = info - logging.debug("no exact match found, chosen most similar of %s within base folders %s", info, folders) + LOGGER.debug("no exact match found, chosen most similar of %s within base folders %s", info, folders) return info msg = "failed to detect {} in {}".format("|".join(possible_names), os.pathsep.join(possible_folders)) raise RuntimeError(msg) - def _check_exe(self, app_data, folder, name, exact, discovered, env): # noqa: PLR0913 + def _check_exe(self, app_data, cache, folder, name, exact, discovered, env): # noqa: PLR0913 exe_path = os.path.join(folder, name) if not os.path.exists(exe_path): return None - info = self.from_exe(exe_path, app_data, resolve_to_host=False, raise_on_error=False, env=env) + info = self.from_exe( + exe_path, + app_data, + cache, + resolve_to_host=False, + raise_on_error=False, + env=env, + ) if info is None: # ignore if for some reason we can't query return None for item in ["implementation", "architecture", "version_info"]: @@ -473,7 +585,7 @@ def _check_exe(self, app_data, folder, name, exact, discovered, env): # noqa: P if item == "version_info": found, searched = ".".join(str(i) for i in found), ".".join(str(i) for i in searched) executable = info.executable - logging.debug("refused interpreter %s because %s differs %s != %s", executable, item, found, searched) + LOGGER.debug("refused interpreter %s because %s differs %s != %s", executable, item, found, searched) if exact is False: discovered.append(info) break @@ -525,10 +637,14 @@ def _find_possible_exe_names(self): for name in self._possible_base(): for at in (3, 2, 1, 0): version = ".".join(str(i) for i in self.version_info[:at]) - for arch in [f"-{self.architecture}", ""]: - for ext in EXTENSIONS: - candidate = f"{name}{version}{arch}{ext}" - name_candidate[candidate] = None + mods = [""] + if self.free_threaded: + mods.append("t") + for mod in mods: + for arch in [f"-{self.architecture}", ""]: + for ext in EXTENSIONS: + candidate = f"{name}{version}{mod}{arch}{ext}" + name_candidate[candidate] = None return list(name_candidate.keys()) def _possible_base(self): @@ -543,7 +659,8 @@ def _possible_base(self): for base in possible_base: lower = base.lower() yield lower - from virtualenv.info import fs_is_case_sensitive + + from .info import fs_is_case_sensitive # noqa: PLC0415 if fs_is_case_sensitive(): if base != lower: diff --git a/src/virtualenv/discovery/py_spec.py b/src/virtualenv/discovery/py_spec.py index 04a5b0975..d8519c23d 100644 --- a/src/virtualenv/discovery/py_spec.py +++ b/src/virtualenv/discovery/py_spec.py @@ -2,32 +2,40 @@ from __future__ import annotations -import contextlib import os import re -from collections import OrderedDict -from virtualenv.info import fs_is_case_sensitive - -PATTERN = re.compile(r"^(?P[a-zA-Z]+)?(?P[0-9.]+)?(?:-(?P32|64))?$") +PATTERN = re.compile(r"^(?P[a-zA-Z]+)?(?P[0-9.]+)?(?Pt)?(?:-(?P32|64))?$") class PythonSpec: """Contains specification about a Python Interpreter.""" - def __init__(self, str_spec, implementation, major, minor, micro, architecture, path) -> None: # noqa: PLR0913 + def __init__( # noqa: PLR0913 + self, + str_spec: str, + implementation: str | None, + major: int | None, + minor: int | None, + micro: int | None, + architecture: int | None, + path: str | None, + *, + free_threaded: bool | None = None, + ) -> None: self.str_spec = str_spec self.implementation = implementation self.major = major self.minor = minor self.micro = micro + self.free_threaded = free_threaded self.architecture = architecture self.path = path @classmethod - def from_string_spec(cls, string_spec): # noqa: C901, PLR0912 - impl, major, minor, micro, arch, path = None, None, None, None, None, None - if os.path.isabs(string_spec): + def from_string_spec(cls, string_spec: str): # noqa: C901, PLR0912 + impl, major, minor, micro, threaded, arch, path = None, None, None, None, None, None, None + if os.path.isabs(string_spec): # noqa: PLR1702 path = string_spec else: ok = False @@ -53,6 +61,7 @@ def _int_or_none(val): major = int(str(version_data)[0]) # first digit major if version_data > 9: # noqa: PLR2004 minor = int(str(version_data)[1:]) + threaded = bool(groups["threaded"]) ok = True except ValueError: pass @@ -65,28 +74,29 @@ def _int_or_none(val): if not ok: path = string_spec - return cls(string_spec, impl, major, minor, micro, arch, path) - - def generate_names(self): - impls = OrderedDict() - if self.implementation: - # first consider implementation as it is - impls[self.implementation] = False - if fs_is_case_sensitive(): - # for case sensitive file systems consider lower and upper case versions too - # trivia: MacBooks and all pre 2018 Windows-es were case insensitive by default - impls[self.implementation.lower()] = False - impls[self.implementation.upper()] = False - impls["python"] = True # finally consider python as alias, implementation must match now - version = self.major, self.minor, self.micro - with contextlib.suppress(ValueError): - version = version[: version.index(None)] - - for impl, match in impls.items(): - for at in range(len(version), -1, -1): - cur_ver = version[0:at] - spec = f"{impl}{'.'.join(str(i) for i in cur_ver)}" - yield spec, match + return cls(string_spec, impl, major, minor, micro, arch, path, free_threaded=threaded) + + def generate_re(self, *, windows: bool) -> re.Pattern: + """Generate a regular expression for matching against a filename.""" + version = r"{}(\.{}(\.{})?)?".format( + *(r"\d+" if v is None else v for v in (self.major, self.minor, self.micro)) + ) + impl = "python" if self.implementation is None else f"python|{re.escape(self.implementation)}" + mod = "t?" if self.free_threaded else "" + suffix = r"\.exe" if windows else "" + version_conditional = ( + "?" + # Windows Python executables are almost always unversioned + if windows + # Spec is an empty string + or self.major is None + else "" + ) + # Try matching `direct` first, so the `direct` group is filled when possible. + return re.compile( + rf"(?P{impl})(?P{version}{mod}){version_conditional}{suffix}$", + flags=re.IGNORECASE, + ) @property def is_abs(self): @@ -100,6 +110,8 @@ def satisfies(self, spec): return False if spec.architecture is not None and spec.architecture != self.architecture: return False + if spec.free_threaded is not None and spec.free_threaded != self.free_threaded: + return False for our, req in zip((self.major, self.minor, self.micro), (spec.major, spec.minor, spec.micro)): if req is not None and our is not None and our != req: @@ -108,7 +120,7 @@ def satisfies(self, spec): def __repr__(self) -> str: name = type(self).__name__ - params = "implementation", "major", "minor", "micro", "architecture", "path" + params = "implementation", "major", "minor", "micro", "architecture", "path", "free_threaded" return f"{name}({', '.join(f'{k}={getattr(self, k)}' for k in params if getattr(self, k) is not None)})" diff --git a/src/virtualenv/discovery/windows/__init__.py b/src/virtualenv/discovery/windows/__init__.py index 9efd5b6ab..ef47a90e3 100644 --- a/src/virtualenv/discovery/windows/__init__.py +++ b/src/virtualenv/discovery/windows/__init__.py @@ -16,7 +16,7 @@ class Pep514PythonInfo(PythonInfo): """A Python information acquired from PEP-514.""" -def propose_interpreters(spec, cache_dir, env): +def propose_interpreters(spec, app_data, cache, env): # see if PEP-514 entries are good # start with higher python versions in an effort to use the latest version available @@ -27,16 +27,22 @@ def propose_interpreters(spec, cache_dir, env): reverse=True, ) - for name, major, minor, arch, exe, _ in existing: + for name, major, minor, arch, threaded, exe, _ in existing: # Map well-known/most common organizations to a Python implementation, use the org name as a fallback for # backwards compatibility. implementation = _IMPLEMENTATION_BY_ORG.get(name, name) # Pre-filtering based on Windows Registry metadata, for CPython only skip_pre_filter = implementation.lower() != "cpython" - registry_spec = PythonSpec(None, implementation, major, minor, None, arch, exe) + registry_spec = PythonSpec(None, implementation, major, minor, None, arch, exe, free_threaded=threaded) if skip_pre_filter or registry_spec.satisfies(spec): - interpreter = Pep514PythonInfo.from_exe(exe, cache_dir, env=env, raise_on_error=False) + interpreter = Pep514PythonInfo.from_exe( + exe, + app_data, + cache, + raise_on_error=False, + env=env, + ) if interpreter is not None and interpreter.satisfies(spec, impl_must_match=True): yield interpreter # Final filtering/matching using interpreter metadata diff --git a/src/virtualenv/discovery/windows/pep514.py b/src/virtualenv/discovery/windows/pep514.py index 8bc9e3060..a75dad36d 100644 --- a/src/virtualenv/discovery/windows/pep514.py +++ b/src/virtualenv/discovery/windows/pep514.py @@ -65,7 +65,8 @@ def process_tag(hive_name, company, company_key, tag, default_arch): exe_data = load_exe(hive_name, company, company_key, tag) if exe_data is not None: exe, args = exe_data - return company, major, minor, arch, exe, args + threaded = load_threaded(hive_name, company, tag, tag_key) + return company, major, minor, arch, threaded, exe, args return None return None return None @@ -138,6 +139,18 @@ def parse_version(version_str): raise ValueError(error) +def load_threaded(hive_name, company, tag, tag_key): + display_name = get_value(tag_key, "DisplayName") + if display_name is not None: + if isinstance(display_name, str): + if "freethreaded" in display_name.lower(): + return True + else: + key_path = f"{hive_name}/{company}/{tag}/DisplayName" + msg(key_path, f"display name is not string: {display_name!r}") + return bool(re.match(r"^\d+(\.\d+){0,2}t$", tag, flags=re.IGNORECASE)) + + def msg(path, what): LOGGER.warning("PEP-514 violation in Windows Registry at %s error: %s", path, what) diff --git a/src/virtualenv/info.py b/src/virtualenv/info.py index e0977768c..e3542a7e2 100644 --- a/src/virtualenv/info.py +++ b/src/virtualenv/info.py @@ -8,13 +8,14 @@ IMPLEMENTATION = platform.python_implementation() IS_PYPY = IMPLEMENTATION == "PyPy" +IS_GRAALPY = IMPLEMENTATION == "GraalVM" IS_CPYTHON = IMPLEMENTATION == "CPython" IS_WIN = sys.platform == "win32" IS_MAC_ARM64 = sys.platform == "darwin" and platform.machine() == "arm64" ROOT = os.path.realpath(os.path.join(os.path.abspath(__file__), os.path.pardir, os.path.pardir)) IS_ZIPAPP = os.path.isfile(ROOT) - _CAN_SYMLINK = _FS_CASE_SENSITIVE = _CFG_DIR = _DATA_DIR = None +LOGGER = logging.getLogger(__name__) def fs_is_case_sensitive(): @@ -23,7 +24,7 @@ def fs_is_case_sensitive(): if _FS_CASE_SENSITIVE is None: with tempfile.NamedTemporaryFile(prefix="TmP") as tmp_file: _FS_CASE_SENSITIVE = not os.path.exists(tmp_file.name.lower()) - logging.debug("filesystem is %scase-sensitive", "" if _FS_CASE_SENSITIVE else "not ") + LOGGER.debug("filesystem is %scase-sensitive", "" if _FS_CASE_SENSITIVE else "not ") return _FS_CASE_SENSITIVE @@ -33,29 +34,37 @@ def fs_supports_symlink(): if _CAN_SYMLINK is None: can = False if hasattr(os, "symlink"): - if IS_WIN: - with tempfile.NamedTemporaryFile(prefix="TmP") as tmp_file: - temp_dir = os.path.dirname(tmp_file.name) - dest = os.path.join(temp_dir, f"{tmp_file.name}-{'b'}") - try: - os.symlink(tmp_file.name, dest) - can = True - except (OSError, NotImplementedError): - pass - logging.debug("symlink on filesystem does%s work", "" if can else " not") - else: - can = True + # Creating a symlink can fail for a variety of reasons, indicating that the filesystem does not support it. + # E.g. on Linux with a VFAT partition mounted. + with tempfile.NamedTemporaryFile(prefix="TmP") as tmp_file: + temp_dir = os.path.dirname(tmp_file.name) + dest = os.path.join(temp_dir, f"{tmp_file.name}-{'b'}") + try: + os.symlink(tmp_file.name, dest) + can = True + except (OSError, NotImplementedError): + pass # symlink is not supported + finally: + if os.path.lexists(dest): + os.remove(dest) + LOGGER.debug("symlink on filesystem does%s work", "" if can else " not") _CAN_SYMLINK = can return _CAN_SYMLINK +def fs_path_id(path: str) -> str: + return path.casefold() if fs_is_case_sensitive() else path + + __all__ = ( - "IS_PYPY", "IS_CPYTHON", + "IS_GRAALPY", + "IS_MAC_ARM64", + "IS_PYPY", "IS_WIN", + "IS_ZIPAPP", + "ROOT", "fs_is_case_sensitive", + "fs_path_id", "fs_supports_symlink", - "ROOT", - "IS_ZIPAPP", - "IS_MAC_ARM64", ) diff --git a/src/virtualenv/report.py b/src/virtualenv/report.py index db1be9934..c9682a8f6 100644 --- a/src/virtualenv/report.py +++ b/src/virtualenv/report.py @@ -18,8 +18,7 @@ def setup_report(verbosity, show_pid=False): # noqa: FBT002 _clean_handlers(LOGGER) - if verbosity > MAX_LEVEL: - verbosity = MAX_LEVEL # pragma: no cover + verbosity = min(verbosity, MAX_LEVEL) # pragma: no cover level = LEVELS[verbosity] msg_format = "%(message)s" if level <= logging.DEBUG: @@ -34,7 +33,7 @@ def setup_report(verbosity, show_pid=False): # noqa: FBT002 stream_handler.setFormatter(formatter) LOGGER.addHandler(stream_handler) level_name = logging.getLevelName(level) - logging.debug("setup logging to %s", level_name) + LOGGER.debug("setup logging to %s", level_name) logging.getLogger("distlib").setLevel(logging.ERROR) return verbosity diff --git a/src/virtualenv/run/__init__.py b/src/virtualenv/run/__init__.py index ed0d6d0cd..208050c90 100644 --- a/src/virtualenv/run/__init__.py +++ b/src/virtualenv/run/__init__.py @@ -5,6 +5,7 @@ from functools import partial from virtualenv.app_data import make_app_data +from virtualenv.cache import FileCache from virtualenv.config.cli.parser import VirtualEnvConfigParser from virtualenv.report import LEVELS, setup_report from virtualenv.run.session import Session @@ -48,6 +49,7 @@ def session_via_cli(args, options=None, setup_logging=True, env=None): # noqa: env = os.environ if env is None else env parser, elements = build_parser(args, options, setup_logging, env) options = parser.parse_args(args) + options.py_version = parser._interpreter.version_info # noqa: SLF001 creator, seeder, activators = tuple(e.create(options) for e in elements) # create types return Session( options.verbosity, @@ -129,11 +131,13 @@ def load_app_data(args, parser, options): options, _ = parser.parse_known_args(args, namespace=options) if options.reset_app_data: options.app_data.reset() + + options.cache = FileCache(store_factory=options.app_data.py_info, clearer=options.app_data.py_info_clear) return options def add_version_flag(parser): - import virtualenv + import virtualenv # noqa: PLC0415 parser.add_argument( "--version", @@ -153,6 +157,9 @@ def _do_report_setup(parser, args, setup_logging): verbosity = verbosity_group.add_mutually_exclusive_group() verbosity.add_argument("-v", "--verbose", action="count", dest="verbose", help="increase verbosity", default=2) verbosity.add_argument("-q", "--quiet", action="count", dest="quiet", help="decrease verbosity", default=0) + # do not configure logging if only help is requested, as no logging is required for this + if args and any(i in args for i in ("-h", "--help")): + return option, _ = parser.parse_known_args(args) if setup_logging: setup_report(option.verbosity) diff --git a/src/virtualenv/run/plugin/base.py b/src/virtualenv/run/plugin/base.py index 16114f60e..97c4792ec 100644 --- a/src/virtualenv/run/plugin/base.py +++ b/src/virtualenv/run/plugin/base.py @@ -2,15 +2,9 @@ import sys from collections import OrderedDict +from importlib.metadata import entry_points -if sys.version_info >= (3, 8): - from importlib.metadata import entry_points - - importlib_metadata_version = () -else: - from importlib_metadata import entry_points, version - - importlib_metadata_version = tuple(int(i) for i in version("importlib_metadata").split(".")[:2]) +importlib_metadata_version = () class PluginLoader: @@ -66,6 +60,6 @@ def create(self, options): __all__ = [ - "PluginLoader", "ComponentBuilder", + "PluginLoader", ] diff --git a/src/virtualenv/run/plugin/creators.py b/src/virtualenv/run/plugin/creators.py index a0bcb028b..6bb11845d 100644 --- a/src/virtualenv/run/plugin/creators.py +++ b/src/virtualenv/run/plugin/creators.py @@ -86,6 +86,6 @@ def create(self, options): __all__ = [ - "CreatorSelector", "CreatorInfo", + "CreatorSelector", ] diff --git a/src/virtualenv/run/plugin/discovery.py b/src/virtualenv/run/plugin/discovery.py index c9e456469..b271faac5 100644 --- a/src/virtualenv/run/plugin/discovery.py +++ b/src/virtualenv/run/plugin/discovery.py @@ -16,10 +16,15 @@ def get_discover(parser, args): choices = _get_default_discovery(discover_types) # prefer the builtin if present, otherwise fallback to first defined type choices = sorted(choices, key=lambda a: 0 if a == "builtin" else 1) + try: + default_discovery = next(iter(choices)) + except StopIteration as e: + msg = "No discovery plugin found. Try reinstalling virtualenv to fix this issue." + raise RuntimeError(msg) from e discovery_parser.add_argument( "--discovery", choices=choices, - default=next(iter(choices)), + default=default_discovery, required=False, help="interpreter discovery method", ) @@ -27,7 +32,7 @@ def get_discover(parser, args): discover_class = discover_types[options.discovery] discover_class.add_parser_arguments(discovery_parser) options, _ = parser.parse_known_args(args, namespace=options) - return discover_class(options) + return discover_class(options, options.cache) def _get_default_discovery(discover_types): @@ -35,6 +40,6 @@ def _get_default_discovery(discover_types): __all__ = [ - "get_discover", "Discovery", + "get_discover", ] diff --git a/src/virtualenv/run/session.py b/src/virtualenv/run/session.py index 9ffd89082..def795328 100644 --- a/src/virtualenv/run/session.py +++ b/src/virtualenv/run/session.py @@ -3,6 +3,8 @@ import json import logging +LOGGER = logging.getLogger(__name__) + class Session: """Represents a virtual environment creation session.""" @@ -47,20 +49,20 @@ def run(self): self.creator.pyenv_cfg.write() def _create(self): - logging.info("create virtual environment via %s", self.creator) + LOGGER.info("create virtual environment via %s", self.creator) self.creator.run() - logging.debug(_DEBUG_MARKER) - logging.debug("%s", _Debug(self.creator)) + LOGGER.debug(_DEBUG_MARKER) + LOGGER.debug("%s", _Debug(self.creator)) def _seed(self): if self.seeder is not None and self.seeder.enabled: - logging.info("add seed packages via %s", self.seeder) + LOGGER.info("add seed packages via %s", self.seeder) self.seeder.run(self.creator) def _activate(self): if self.activators: active = ", ".join(type(i).__name__.replace("Activator", "") for i in self.activators) - logging.info("add activators for %s", active) + LOGGER.info("add activators for %s", active) for activator in self.activators: activator.generate(self.creator) diff --git a/src/virtualenv/seed/embed/base_embed.py b/src/virtualenv/seed/embed/base_embed.py index 5ff2c847b..72fc5a34a 100644 --- a/src/virtualenv/seed/embed/base_embed.py +++ b/src/virtualenv/seed/embed/base_embed.py @@ -1,15 +1,18 @@ from __future__ import annotations -from abc import ABCMeta +import logging +from abc import ABC +from argparse import SUPPRESS from pathlib import Path from virtualenv.seed.seeder import Seeder from virtualenv.seed.wheels import Version +LOGGER = logging.getLogger(__name__) PERIODIC_UPDATE_ON_BY_DEFAULT = True -class BaseEmbed(Seeder, metaclass=ABCMeta): +class BaseEmbed(Seeder, ABC): def __init__(self, options) -> None: super().__init__(options, enabled=options.no_seed is False) @@ -18,7 +21,11 @@ def __init__(self, options) -> None: self.pip_version = options.pip self.setuptools_version = options.setuptools - self.wheel_version = options.wheel + + # wheel version needs special handling + # on Python > 3.8, the default is None (as in not used) + # so we can differentiate between explicit and implicit none + self.wheel_version = options.wheel or "none" self.no_pip = options.no_pip self.no_setuptools = options.no_setuptools @@ -26,6 +33,15 @@ def __init__(self, options) -> None: self.app_data = options.app_data self.periodic_update = not options.no_periodic_update + if options.py_version[:2] >= (3, 9): + if options.wheel is not None or options.no_wheel: + LOGGER.warning( + "The --no-wheel and --wheel options are deprecated. " + "They have no effect for Python > 3.8 as wheel is no longer " + "bundled in virtualenv.", + ) + self.no_wheel = True + if not self.distribution_to_versions(): self.enabled = False @@ -41,7 +57,7 @@ def distribution_to_versions(self) -> dict[str, str]: return { distribution: getattr(self, f"{distribution}_version") for distribution in self.distributions() - if getattr(self, f"no_{distribution}") is False and getattr(self, f"{distribution}_version") != "none" + if getattr(self, f"no_{distribution}", None) is False and getattr(self, f"{distribution}_version") != "none" } @classmethod @@ -71,21 +87,28 @@ def add_parser_arguments(cls, parser, interpreter, app_data): # noqa: ARG003 default=[], ) for distribution, default in cls.distributions().items(): + help_ = f"version of {distribution} to install as seed: embed, bundle, none or exact version" if interpreter.version_info[:2] >= (3, 12) and distribution in {"wheel", "setuptools"}: default = "none" # noqa: PLW2901 + if interpreter.version_info[:2] >= (3, 9) and distribution == "wheel": + default = None # noqa: PLW2901 + help_ = SUPPRESS parser.add_argument( f"--{distribution}", dest=distribution, metavar="version", - help=f"version of {distribution} to install as seed: embed, bundle, none or exact version", + help=help_, default=default, ) for distribution in cls.distributions(): + help_ = f"do not install {distribution}" + if interpreter.version_info[:2] >= (3, 9) and distribution == "wheel": + help_ = SUPPRESS parser.add_argument( f"--no-{distribution}", dest=f"no_{distribution}", action="store_true", - help=f"do not install {distribution}", + help=help_, default=False, ) parser.add_argument( @@ -103,7 +126,7 @@ def __repr__(self) -> str: result += f"extra_search_dir={', '.join(str(i) for i in self.extra_search_dir)}," result += f"download={self.download}," for distribution in self.distributions(): - if getattr(self, f"no_{distribution}"): + if getattr(self, f"no_{distribution}", None): continue version = getattr(self, f"{distribution}_version", None) if version == "none": diff --git a/src/virtualenv/seed/embed/pip_invoke.py b/src/virtualenv/seed/embed/pip_invoke.py index 2625a01de..b733c5148 100644 --- a/src/virtualenv/seed/embed/pip_invoke.py +++ b/src/virtualenv/seed/embed/pip_invoke.py @@ -8,6 +8,8 @@ from virtualenv.seed.embed.base_embed import BaseEmbed from virtualenv.seed.wheels import Version, get_wheel, pip_wheel_env_run +LOGGER = logging.getLogger(__name__) + class PipInvoke(BaseEmbed): def __init__(self, options) -> None: @@ -23,8 +25,8 @@ def run(self, creator): @staticmethod def _execute(cmd, env): - logging.debug("pip seed by running: %s", LogCmd(cmd, env)) - process = Popen(cmd, env=env) # noqa: S603 + LOGGER.debug("pip seed by running: %s", LogCmd(cmd, env)) + process = Popen(cmd, env=env) process.communicate() if process.returncode != 0: msg = f"failed seed with code {process.returncode}" diff --git a/src/virtualenv/seed/embed/via_app_data/pip_install/base.py b/src/virtualenv/seed/embed/via_app_data/pip_install/base.py index cc3b73698..d5e87ad93 100644 --- a/src/virtualenv/seed/embed/via_app_data/pip_install/base.py +++ b/src/virtualenv/seed/embed/via_app_data/pip_install/base.py @@ -4,7 +4,7 @@ import os import re import zipfile -from abc import ABCMeta, abstractmethod +from abc import ABC, abstractmethod from configparser import ConfigParser from itertools import chain from pathlib import Path @@ -14,8 +14,10 @@ from virtualenv.util.path import safe_delete +LOGGER = logging.getLogger(__name__) -class PipInstall(metaclass=ABCMeta): + +class PipInstall(ABC): def __init__(self, wheel, creator, image_folder) -> None: self._wheel = wheel self._creator = creator @@ -40,11 +42,11 @@ def install(self, version_info): script_dir = self._creator.script_dir for name, module in self._console_scripts.items(): consoles.update(self._create_console_entry_point(name, module, script_dir, version_info)) - logging.debug("generated console scripts %s", " ".join(i.name for i in consoles)) + LOGGER.debug("generated console scripts %s", " ".join(i.name for i in consoles)) def build_image(self): # 1. first extract the wheel - logging.debug("build install image for %s to %s", self._wheel.name, self._image_dir) + LOGGER.debug("build install image for %s to %s", self._wheel.name, self._image_dir) with zipfile.ZipFile(str(self._wheel)) as zip_ref: self._shorten_path_if_needed(zip_ref) zip_ref.extractall(str(self._image_dir)) @@ -63,7 +65,7 @@ def _shorten_path_if_needed(self, zip_ref): if path_len > 260: # noqa: PLR2004 self._image_dir.mkdir(exist_ok=True) # to get a short path must exist - from virtualenv.util.path import get_short_path_name + from virtualenv.util.path import get_short_path_name # noqa: PLC0415 to_folder = get_short_path_name(to_folder) self._image_dir = Path(to_folder) @@ -151,7 +153,7 @@ def _uninstall_previous_version(self): @staticmethod def _uninstall_dist(dist): dist_base = dist.parent - logging.debug("uninstall existing distribution %s from %s", dist.stem, dist_base) + LOGGER.debug("uninstall existing distribution %s from %s", dist.stem, dist_base) top_txt = dist / "top_level.txt" # add top level packages at folder level paths = ( @@ -181,7 +183,7 @@ def clear(self): safe_delete(self._image_dir) def has_image(self): - return self._image_dir.exists() and next(self._image_dir.iterdir()) is not None + return self._image_dir.exists() and any(self._image_dir.iterdir()) class ScriptMakerCustom(ScriptMaker): @@ -194,7 +196,7 @@ def __init__(self, target_dir, version_info, executable, name) -> None: self.variants = {"", "X", "X.Y"} self._name = name - def _write_script(self, names, shebang, script_bytes, filenames, ext): # noqa: PLR0913 + def _write_script(self, names, shebang, script_bytes, filenames, ext): names.add(f"{self._name}{self.version_info[0]}.{self.version_info[1]}") super()._write_script(names, shebang, script_bytes, filenames, ext) diff --git a/src/virtualenv/seed/embed/via_app_data/pip_install/symlink.py b/src/virtualenv/seed/embed/via_app_data/pip_install/symlink.py index 6bc5e51c0..7eb9f5f47 100644 --- a/src/virtualenv/seed/embed/via_app_data/pip_install/symlink.py +++ b/src/virtualenv/seed/embed/via_app_data/pip_install/symlink.py @@ -16,7 +16,7 @@ def _sync(self, src, dst): def _generate_new_files(self): # create the pyc files, as the build image will be R/O cmd = [str(self._creator.exe), "-m", "compileall", str(self._image_dir)] - process = Popen(cmd, stdout=PIPE, stderr=PIPE) # noqa: S603 + process = Popen(cmd, stdout=PIPE, stderr=PIPE) process.communicate() # the root pyc is shared, so we'll not symlink that - but still add the pyc files to the RECORD for close root_py_cache = self._image_dir / "__pycache__" @@ -40,8 +40,7 @@ def _generate_new_files(self): def _fix_records(self, new_files): new_files.update(i for i in self._image_dir.iterdir()) extra_record_data_str = self._records_text(sorted(new_files, key=str)) - with open(str(self._dist_info / "RECORD"), "wb") as file_handler: - file_handler.write(extra_record_data_str.encode("utf-8")) + (self._dist_info / "RECORD").write_text(extra_record_data_str, encoding="utf-8") def build_image(self): super().build_image() diff --git a/src/virtualenv/seed/embed/via_app_data/via_app_data.py b/src/virtualenv/seed/embed/via_app_data/via_app_data.py index 7e58bfc6e..a2e9630c6 100644 --- a/src/virtualenv/seed/embed/via_app_data/via_app_data.py +++ b/src/virtualenv/seed/embed/via_app_data/via_app_data.py @@ -17,6 +17,8 @@ from .pip_install.copy import CopyPipInstall from .pip_install.symlink import SymlinkPipInstall +LOGGER = logging.getLogger(__name__) + class FromAppData(BaseEmbed): def __init__(self, options) -> None: @@ -46,7 +48,7 @@ def run(self, creator): def _install(name, wheel): try: - logging.debug("install %s from wheel %s via %s", name, wheel, installer_class.__name__) + LOGGER.debug("install %s from wheel %s via %s", name, wheel, installer_class.__name__) key = Path(installer_class.__name__) / wheel.path.stem wheel_img = self.app_data.wheel_image(creator.interpreter.version_release_str, key) installer = installer_class(wheel.path, creator, wheel_img) @@ -94,7 +96,7 @@ def _get(distribution, version): if result is not None: break except Exception as exception: - logging.exception("fail") + LOGGER.exception("fail") failure = exception if failure: if isinstance(failure, CalledProcessError): @@ -108,7 +110,7 @@ def _get(distribution, version): msg += output else: msg = repr(failure) - logging.error(msg) + LOGGER.error(msg) with lock: fail[distribution] = version else: diff --git a/src/virtualenv/seed/seeder.py b/src/virtualenv/seed/seeder.py index 01f943098..58fd8f416 100644 --- a/src/virtualenv/seed/seeder.py +++ b/src/virtualenv/seed/seeder.py @@ -1,9 +1,9 @@ from __future__ import annotations -from abc import ABCMeta, abstractmethod +from abc import ABC, abstractmethod -class Seeder(metaclass=ABCMeta): +class Seeder(ABC): """A seeder will install some seed packages into a virtual environment.""" def __init__(self, options, enabled) -> None: diff --git a/src/virtualenv/seed/wheels/__init__.py b/src/virtualenv/seed/wheels/__init__.py index 94b4cc93b..20a398a30 100644 --- a/src/virtualenv/seed/wheels/__init__.py +++ b/src/virtualenv/seed/wheels/__init__.py @@ -4,8 +4,8 @@ from .util import Version, Wheel __all__ = [ - "get_wheel", - "pip_wheel_env_run", "Version", "Wheel", + "get_wheel", + "pip_wheel_env_run", ] diff --git a/src/virtualenv/seed/wheels/acquire.py b/src/virtualenv/seed/wheels/acquire.py index c5ed731d1..eb2fb5b45 100644 --- a/src/virtualenv/seed/wheels/acquire.py +++ b/src/virtualenv/seed/wheels/acquire.py @@ -12,6 +12,8 @@ from .periodic_update import add_wheel_to_update_log from .util import Version, Wheel, discover_wheels +LOGGER = logging.getLogger(__name__) + def get_wheel( # noqa: PLR0913 distribution, @@ -50,7 +52,7 @@ def get_wheel( # noqa: PLR0913 def download_wheel(distribution, version_spec, for_py_version, search_dirs, app_data, to_folder, env): # noqa: PLR0913 to_download = f"{distribution}{version_spec or ''}" - logging.debug("download wheel %s %s to %s", to_download, for_py_version, to_folder) + LOGGER.debug("download wheel %s %s to %s", to_download, for_py_version, to_folder) cmd = [ sys.executable, "-m", @@ -69,13 +71,13 @@ def download_wheel(distribution, version_spec, for_py_version, search_dirs, app_ ] # pip has no interface in python - must be a new sub-process env = pip_wheel_env_run(search_dirs, app_data, env) - process = Popen(cmd, env=env, stdout=PIPE, stderr=PIPE, universal_newlines=True, encoding="utf-8") # noqa: S603 + process = Popen(cmd, env=env, stdout=PIPE, stderr=PIPE, universal_newlines=True, encoding="utf-8") out, err = process.communicate() if process.returncode != 0: kwargs = {"output": out, "stderr": err} raise CalledProcessError(process.returncode, cmd, **kwargs) result = _find_downloaded_wheel(distribution, version_spec, for_py_version, to_folder, out) - logging.debug("downloaded wheel %s", result.name) + LOGGER.debug("downloaded wheel %s", result.name) return result @@ -107,7 +109,7 @@ def find_compatible_in_house(distribution, version_spec, for_py_version, in_fold def pip_wheel_env_run(search_dirs, app_data, env): env = env.copy() - env.update({"PIP_USE_WHEEL": "1", "PIP_USER": "0", "PIP_NO_INPUT": "1"}) + env.update({"PIP_USE_WHEEL": "1", "PIP_USER": "0", "PIP_NO_INPUT": "1", "PYTHONIOENCODING": "utf-8"}) wheel = get_wheel( distribution="pip", version=None, @@ -126,7 +128,7 @@ def pip_wheel_env_run(search_dirs, app_data, env): __all__ = [ - "get_wheel", "download_wheel", + "get_wheel", "pip_wheel_env_run", ] diff --git a/src/virtualenv/seed/wheels/bundle.py b/src/virtualenv/seed/wheels/bundle.py index d54ebccda..523e45ca2 100644 --- a/src/virtualenv/seed/wheels/bundle.py +++ b/src/virtualenv/seed/wheels/bundle.py @@ -45,6 +45,6 @@ def from_dir(distribution, version, for_py_version, directories): __all__ = [ - "load_embed_wheel", "from_bundle", + "load_embed_wheel", ] diff --git a/src/virtualenv/seed/wheels/embed/__init__.py b/src/virtualenv/seed/wheels/embed/__init__.py index fcbb35cc4..0dbab1477 100644 --- a/src/virtualenv/seed/wheels/embed/__init__.py +++ b/src/virtualenv/seed/wheels/embed/__init__.py @@ -6,48 +6,51 @@ BUNDLE_FOLDER = Path(__file__).absolute().parent BUNDLE_SUPPORT = { - "3.7": { - "pip": "pip-23.2.1-py3-none-any.whl", - "setuptools": "setuptools-68.0.0-py3-none-any.whl", - "wheel": "wheel-0.41.2-py3-none-any.whl", - }, "3.8": { - "pip": "pip-23.2.1-py3-none-any.whl", - "setuptools": "setuptools-68.1.2-py3-none-any.whl", - "wheel": "wheel-0.41.2-py3-none-any.whl", + "pip": "pip-25.0.1-py3-none-any.whl", + "setuptools": "setuptools-75.3.2-py3-none-any.whl", + "wheel": "wheel-0.45.1-py3-none-any.whl", }, "3.9": { - "pip": "pip-23.2.1-py3-none-any.whl", - "setuptools": "setuptools-68.1.2-py3-none-any.whl", - "wheel": "wheel-0.41.2-py3-none-any.whl", + "pip": "pip-25.2-py3-none-any.whl", + "setuptools": "setuptools-80.9.0-py3-none-any.whl", }, "3.10": { - "pip": "pip-23.2.1-py3-none-any.whl", - "setuptools": "setuptools-68.1.2-py3-none-any.whl", - "wheel": "wheel-0.41.2-py3-none-any.whl", + "pip": "pip-25.2-py3-none-any.whl", + "setuptools": "setuptools-80.9.0-py3-none-any.whl", }, "3.11": { - "pip": "pip-23.2.1-py3-none-any.whl", - "setuptools": "setuptools-68.1.2-py3-none-any.whl", - "wheel": "wheel-0.41.2-py3-none-any.whl", + "pip": "pip-25.2-py3-none-any.whl", + "setuptools": "setuptools-80.9.0-py3-none-any.whl", }, "3.12": { - "pip": "pip-23.2.1-py3-none-any.whl", - "setuptools": "setuptools-68.1.2-py3-none-any.whl", - "wheel": "wheel-0.41.2-py3-none-any.whl", + "pip": "pip-25.2-py3-none-any.whl", + "setuptools": "setuptools-80.9.0-py3-none-any.whl", + }, + "3.13": { + "pip": "pip-25.2-py3-none-any.whl", + "setuptools": "setuptools-80.9.0-py3-none-any.whl", + }, + "3.14": { + "pip": "pip-25.2-py3-none-any.whl", + "setuptools": "setuptools-80.9.0-py3-none-any.whl", }, } -MAX = "3.7" +MAX = "3.8" def get_embed_wheel(distribution, for_py_version): - path = BUNDLE_FOLDER / (BUNDLE_SUPPORT.get(for_py_version, {}) or BUNDLE_SUPPORT[MAX]).get(distribution) + mapping = BUNDLE_SUPPORT.get(for_py_version, {}) or BUNDLE_SUPPORT[MAX] + wheel_file = mapping.get(distribution) + if wheel_file is None: + return None + path = BUNDLE_FOLDER / wheel_file return Wheel.from_path(path) __all__ = [ - "get_embed_wheel", + "BUNDLE_FOLDER", "BUNDLE_SUPPORT", "MAX", - "BUNDLE_FOLDER", + "get_embed_wheel", ] diff --git a/src/virtualenv/seed/wheels/embed/pip-23.2.1-py3-none-any.whl b/src/virtualenv/seed/wheels/embed/pip-23.2.1-py3-none-any.whl deleted file mode 100644 index ba28ef02e..000000000 Binary files a/src/virtualenv/seed/wheels/embed/pip-23.2.1-py3-none-any.whl and /dev/null differ diff --git a/src/virtualenv/seed/wheels/embed/pip-25.0.1-py3-none-any.whl b/src/virtualenv/seed/wheels/embed/pip-25.0.1-py3-none-any.whl new file mode 100644 index 000000000..8d3b0043e Binary files /dev/null and b/src/virtualenv/seed/wheels/embed/pip-25.0.1-py3-none-any.whl differ diff --git a/src/virtualenv/seed/wheels/embed/pip-25.2-py3-none-any.whl b/src/virtualenv/seed/wheels/embed/pip-25.2-py3-none-any.whl new file mode 100644 index 000000000..e14bb3f37 Binary files /dev/null and b/src/virtualenv/seed/wheels/embed/pip-25.2-py3-none-any.whl differ diff --git a/src/virtualenv/seed/wheels/embed/setuptools-68.0.0-py3-none-any.whl b/src/virtualenv/seed/wheels/embed/setuptools-68.0.0-py3-none-any.whl deleted file mode 100644 index 81f15459b..000000000 Binary files a/src/virtualenv/seed/wheels/embed/setuptools-68.0.0-py3-none-any.whl and /dev/null differ diff --git a/src/virtualenv/seed/wheels/embed/setuptools-68.1.2-py3-none-any.whl b/src/virtualenv/seed/wheels/embed/setuptools-68.1.2-py3-none-any.whl deleted file mode 100644 index e9fdff1cd..000000000 Binary files a/src/virtualenv/seed/wheels/embed/setuptools-68.1.2-py3-none-any.whl and /dev/null differ diff --git a/src/virtualenv/seed/wheels/embed/setuptools-75.3.2-py3-none-any.whl b/src/virtualenv/seed/wheels/embed/setuptools-75.3.2-py3-none-any.whl new file mode 100644 index 000000000..1b66a67ff Binary files /dev/null and b/src/virtualenv/seed/wheels/embed/setuptools-75.3.2-py3-none-any.whl differ diff --git a/src/virtualenv/seed/wheels/embed/setuptools-80.9.0-py3-none-any.whl b/src/virtualenv/seed/wheels/embed/setuptools-80.9.0-py3-none-any.whl new file mode 100644 index 000000000..2412ad4a3 Binary files /dev/null and b/src/virtualenv/seed/wheels/embed/setuptools-80.9.0-py3-none-any.whl differ diff --git a/src/virtualenv/seed/wheels/embed/wheel-0.41.2-py3-none-any.whl b/src/virtualenv/seed/wheels/embed/wheel-0.41.2-py3-none-any.whl deleted file mode 100644 index f2befff47..000000000 Binary files a/src/virtualenv/seed/wheels/embed/wheel-0.41.2-py3-none-any.whl and /dev/null differ diff --git a/src/virtualenv/seed/wheels/embed/wheel-0.45.1-py3-none-any.whl b/src/virtualenv/seed/wheels/embed/wheel-0.45.1-py3-none-any.whl new file mode 100644 index 000000000..589308a21 Binary files /dev/null and b/src/virtualenv/seed/wheels/embed/wheel-0.45.1-py3-none-any.whl differ diff --git a/src/virtualenv/seed/wheels/periodic_update.py b/src/virtualenv/seed/wheels/periodic_update.py index 95937e636..ac627b3bd 100644 --- a/src/virtualenv/seed/wheels/periodic_update.py +++ b/src/virtualenv/seed/wheels/periodic_update.py @@ -1,6 +1,5 @@ """Periodically update bundled versions.""" - from __future__ import annotations import json @@ -23,6 +22,7 @@ from virtualenv.seed.wheels.util import Wheel from virtualenv.util.subprocess import CREATE_NO_WINDOW +LOGGER = logging.getLogger(__name__) GRACE_PERIOD_CI = timedelta(hours=1) # prevent version switch in the middle of a CI run GRACE_PERIOD_MINOR = timedelta(days=28) UPDATE_PERIOD = timedelta(days=14) @@ -46,7 +46,7 @@ def periodic_update( # noqa: PLR0913 def _update_wheel(ver): updated_wheel = Wheel(app_data.house / ver.filename) - logging.debug("using %supdated wheel %s", "periodically " if updated_wheel else "", updated_wheel) + LOGGER.debug("using %supdated wheel %s", "periodically " if updated_wheel else "", updated_wheel) return updated_wheel u_log = UpdateLog.from_app_data(app_data, distribution, for_py_version) @@ -80,10 +80,10 @@ def handle_auto_update(distribution, for_py_version, wheel, search_dirs, app_dat def add_wheel_to_update_log(wheel, for_py_version, app_data): embed_update_log = app_data.embed_update_log(wheel.distribution, for_py_version) - logging.debug("adding %s information to %s", wheel.name, embed_update_log.file) + LOGGER.debug("adding %s information to %s", wheel.name, embed_update_log.file) u_log = UpdateLog.from_dict(embed_update_log.read()) if any(version.filename == wheel.name for version in u_log.versions): - logging.warning("%s already present in %s", wheel.name, embed_update_log.file) + LOGGER.warning("%s already present in %s", wheel.name, embed_update_log.file) return # we don't need a release date for sources other than "periodic" version = NewVersion(wheel.name, datetime.now(tz=timezone.utc), None, "download") @@ -102,7 +102,7 @@ def load_datetime(value): return None if value is None else datetime.strptime(value, DATETIME_FMT).replace(tzinfo=timezone.utc) -class NewVersion: +class NewVersion: # noqa: PLW1641 def __init__(self, filename, found_date, release_date, source) -> None: self.filename = filename self.found_date = found_date @@ -143,7 +143,7 @@ def __repr__(self) -> str: ) def __eq__(self, other): - return type(self) == type(other) and all( + return type(self) == type(other) and all( # noqa: E721 getattr(self, k) == getattr(other, k) for k in ["filename", "release_date", "found_date", "source"] ) @@ -220,8 +220,8 @@ def trigger_update(distribution, for_py_version, wheel, search_dirs, app_data, e kwargs = {"stdout": pipe, "stderr": pipe} if not debug and sys.platform == "win32": kwargs["creationflags"] = CREATE_NO_WINDOW - process = Popen(cmd, **kwargs) # noqa: S603 - logging.info( + process = Popen(cmd, **kwargs) + LOGGER.info( "triggered periodic upgrade of %s%s (for python %s) via background process having PID %d", distribution, "" if wheel is None else f"=={wheel.version}", @@ -240,7 +240,7 @@ def do_update(distribution, for_py_version, embed_filename, app_data, search_dir try: versions = _run_do_update(app_data, distribution, embed_filename, for_py_version, periodic, search_dirs) finally: - logging.debug("done %s %s with %s", distribution, for_py_version, versions) + LOGGER.debug("done %s %s with %s", distribution, for_py_version, versions) return versions @@ -252,7 +252,7 @@ def _run_do_update( # noqa: C901, PLR0913 periodic, search_dirs, ): - from virtualenv.seed.wheels import acquire + from virtualenv.seed.wheels import acquire # noqa: PLC0415 wheel_filename = None if embed_filename is None else Path(embed_filename) embed_version = None if wheel_filename is None else Wheel(wheel_filename).version_tuple @@ -298,7 +298,7 @@ def _run_do_update( # noqa: C901, PLR0913 break release_date = release_date_for_wheel_path(dest.path) last = NewVersion(filename=dest.path.name, release_date=release_date, found_date=download_time, source=source) - logging.info("detected %s in %s", last, datetime.now(tz=timezone.utc) - download_time) + LOGGER.info("detected %s in %s", last, datetime.now(tz=timezone.utc) - download_time) versions.append(last) filenames.add(last.filename) last_wheel = last.wheel @@ -326,7 +326,7 @@ def release_date_for_wheel_path(dest): upload_time = content["releases"][wheel.version][0]["upload_time"] return datetime.strptime(upload_time, "%Y-%m-%dT%H:%M:%S").replace(tzinfo=timezone.utc) except Exception as exception: # noqa: BLE001 - logging.error("could not load release date %s because %r", content, exception) # noqa: TRY400 + LOGGER.error("could not load release date %s because %r", content, exception) # noqa: TRY400 return None @@ -354,9 +354,9 @@ def _pypi_get_distribution_info(distribution): content = json.load(file_handler) break except URLError as exception: - logging.error("failed to access %s because %r", url, exception) # noqa: TRY400 + LOGGER.error("failed to access %s because %r", url, exception) # noqa: TRY400 except Exception as exception: # noqa: BLE001 - logging.error("failed to access %s because %r", url, exception) # noqa: TRY400 + LOGGER.error("failed to access %s because %r", url, exception) # noqa: TRY400 return content @@ -376,7 +376,7 @@ def manual_upgrade(app_data, env): def _run_manual_upgrade(app_data, distribution, for_py_version, env): start = datetime.now(tz=timezone.utc) - from .bundle import from_bundle + from .bundle import from_bundle # noqa: PLC0415 current = from_bundle( distribution=distribution, @@ -387,7 +387,7 @@ def _run_manual_upgrade(app_data, distribution, for_py_version, env): do_periodic_update=False, env=env, ) - logging.warning( + LOGGER.warning( "upgrade %s for python %s with current %s", distribution, for_py_version, @@ -411,18 +411,18 @@ def _run_manual_upgrade(app_data, distribution, for_py_version, env): args.append("\n".join(f"\t{v}" for v in versions)) ver_update = "new entries found:\n%s" if versions else "no new versions found" msg = f"upgraded %s for python %s in %s {ver_update}" - logging.warning(msg, *args) + LOGGER.warning(msg, *args) __all__ = [ - "add_wheel_to_update_log", - "periodic_update", - "do_update", - "manual_upgrade", "NewVersion", "UpdateLog", - "load_datetime", + "add_wheel_to_update_log", + "do_update", "dump_datetime", - "trigger_update", + "load_datetime", + "manual_upgrade", + "periodic_update", "release_date_for_wheel_path", + "trigger_update", ] diff --git a/src/virtualenv/seed/wheels/util.py b/src/virtualenv/seed/wheels/util.py index cfc0098a9..2bc01ae27 100644 --- a/src/virtualenv/seed/wheels/util.py +++ b/src/virtualenv/seed/wheels/util.py @@ -115,7 +115,7 @@ def as_version_spec(version): __all__ = [ - "discover_wheels", "Version", "Wheel", + "discover_wheels", ] diff --git a/src/virtualenv/util/error.py b/src/virtualenv/util/error.py index 7b23509bd..a317ddc18 100644 --- a/src/virtualenv/util/error.py +++ b/src/virtualenv/util/error.py @@ -1,6 +1,5 @@ """Errors.""" - from __future__ import annotations diff --git a/src/virtualenv/util/lock.py b/src/virtualenv/util/lock.py index c15b5f188..b250e032f 100644 --- a/src/virtualenv/util/lock.py +++ b/src/virtualenv/util/lock.py @@ -4,13 +4,15 @@ import logging import os -from abc import ABCMeta, abstractmethod +from abc import ABC, abstractmethod from contextlib import contextmanager, suppress from pathlib import Path from threading import Lock, RLock from filelock import FileLock, Timeout +LOGGER = logging.getLogger(__name__) + class _CountedFileLock(FileLock): def __init__(self, lock_file) -> None: @@ -27,23 +29,29 @@ def acquire(self, timeout=None, poll_interval=0.05): if not self.thread_safe.acquire(timeout=-1 if timeout is None else timeout): raise Timeout(self.lock_file) if self.count == 0: - super().acquire(timeout, poll_interval) + try: + super().acquire(timeout, poll_interval) + except BaseException: + self.thread_safe.release() + raise self.count += 1 def release(self, force=False): # noqa: FBT002 with self.thread_safe: if self.count > 0: - self.thread_safe.release() - if self.count == 1: - super().release(force=force) - self.count = max(self.count - 1, 0) + if self.count == 1: + super().release(force=force) + self.count -= 1 + if self.count == 0: + # if we have no more users of this lock, release the thread lock + self.thread_safe.release() _lock_store = {} _store_lock = Lock() -class PathLockBase(metaclass=ABCMeta): +class PathLockBase(ABC): def __init__(self, folder) -> None: path = Path(folder) self.path = path.resolve() if path.exists() else path @@ -51,11 +59,8 @@ def __init__(self, folder) -> None: def __repr__(self) -> str: return f"{self.__class__.__name__}({self.path})" - def __div__(self, other): - return type(self)(self.path / other) - def __truediv__(self, other): - return self.__div__(other) + return type(self)(self.path / other) @abstractmethod def __enter__(self): @@ -119,7 +124,7 @@ def _lock_file(self, lock, no_block=False): # noqa: FBT002 except Timeout: if no_block: raise - logging.debug("lock file %s present, will block until released", lock.lock_file) + LOGGER.debug("lock file %s present, will block until released", lock.lock_file) lock.release() # release the acquire try from above lock.acquire() diff --git a/src/virtualenv/util/path/__init__.py b/src/virtualenv/util/path/__init__.py index dc827f3c6..c144d8dfd 100644 --- a/src/virtualenv/util/path/__init__.py +++ b/src/virtualenv/util/path/__init__.py @@ -5,12 +5,12 @@ from ._win import get_short_path_name __all__ = [ - "ensure_dir", - "symlink", "copy", "copytree", + "ensure_dir", + "get_short_path_name", "make_exe", - "set_tree", "safe_delete", - "get_short_path_name", + "set_tree", + "symlink", ] diff --git a/src/virtualenv/util/path/_sync.py b/src/virtualenv/util/path/_sync.py index c9334aded..02a6f6e9e 100644 --- a/src/virtualenv/util/path/_sync.py +++ b/src/virtualenv/util/path/_sync.py @@ -6,10 +6,12 @@ import sys from stat import S_IWUSR +LOGGER = logging.getLogger(__name__) + def ensure_dir(path): if not path.exists(): - logging.debug("create folder %s", str(path)) + LOGGER.debug("create folder %s", str(path)) os.makedirs(str(path)) @@ -20,16 +22,16 @@ def ensure_safe_to_do(src, dest): if not dest.exists(): return if dest.is_dir() and not dest.is_symlink(): - logging.debug("remove directory %s", dest) + LOGGER.debug("remove directory %s", dest) safe_delete(dest) else: - logging.debug("remove file %s", dest) + LOGGER.debug("remove file %s", dest) dest.unlink() def symlink(src, dest): ensure_safe_to_do(src, dest) - logging.debug("symlink %s", _Debug(src, dest)) + LOGGER.debug("symlink %s", _Debug(src, dest)) dest.symlink_to(src, target_is_directory=src.is_dir()) @@ -37,7 +39,7 @@ def copy(src, dest): ensure_safe_to_do(src, dest) is_dir = src.is_dir() method = copytree if is_dir else shutil.copy - logging.debug("copy %s", _Debug(src, dest)) + LOGGER.debug("copy %s", _Debug(src, dest)) method(str(src), str(dest)) @@ -58,7 +60,7 @@ def onerror(func, path, exc_info): # noqa: ARG001 os.chmod(path, S_IWUSR) func(path) else: - raise + raise # noqa: PLE0704 kwargs = {"onexc" if sys.version_info >= (3, 12) else "onerror": onerror} shutil.rmtree(str(dest), ignore_errors=True, **kwargs) @@ -74,10 +76,10 @@ def __str__(self) -> str: __all__ = [ - "ensure_dir", - "symlink", "copy", - "symlink", "copytree", + "ensure_dir", "safe_delete", + "symlink", + "symlink", ] diff --git a/src/virtualenv/util/path/_win.py b/src/virtualenv/util/path/_win.py index e7385519d..6404cda64 100644 --- a/src/virtualenv/util/path/_win.py +++ b/src/virtualenv/util/path/_win.py @@ -3,16 +3,16 @@ def get_short_path_name(long_name): """Gets the short path name of a given long path - http://stackoverflow.com/a/23598461/200291.""" - import ctypes - from ctypes import wintypes + import ctypes # noqa: PLC0415 + from ctypes import wintypes # noqa: PLC0415 - _GetShortPathNameW = ctypes.windll.kernel32.GetShortPathNameW # noqa: N806 - _GetShortPathNameW.argtypes = [wintypes.LPCWSTR, wintypes.LPWSTR, wintypes.DWORD] - _GetShortPathNameW.restype = wintypes.DWORD + GetShortPathNameW = ctypes.windll.kernel32.GetShortPathNameW # noqa: N806 + GetShortPathNameW.argtypes = [wintypes.LPCWSTR, wintypes.LPWSTR, wintypes.DWORD] + GetShortPathNameW.restype = wintypes.DWORD output_buf_size = 0 while True: output_buf = ctypes.create_unicode_buffer(output_buf_size) - needed = _GetShortPathNameW(long_name, output_buf, output_buf_size) + needed = GetShortPathNameW(long_name, output_buf, output_buf_size) if output_buf_size >= needed: return output_buf.value output_buf_size = needed diff --git a/src/virtualenv/util/subprocess/__init__.py b/src/virtualenv/util/subprocess/__init__.py index 03e5370e8..e6d5fc885 100644 --- a/src/virtualenv/util/subprocess/__init__.py +++ b/src/virtualenv/util/subprocess/__init__.py @@ -8,7 +8,7 @@ def run_cmd(cmd): try: process = subprocess.Popen( - cmd, # noqa: S603 + cmd, universal_newlines=True, stdin=subprocess.PIPE, stderr=subprocess.PIPE, @@ -25,6 +25,6 @@ def run_cmd(cmd): __all__ = ( - "run_cmd", "CREATE_NO_WINDOW", + "run_cmd", ) diff --git a/src/virtualenv/util/zipapp.py b/src/virtualenv/util/zipapp.py index 3049b8400..183dd07db 100644 --- a/src/virtualenv/util/zipapp.py +++ b/src/virtualenv/util/zipapp.py @@ -6,6 +6,8 @@ from virtualenv.info import IS_WIN, ROOT +LOGGER = logging.getLogger(__name__) + def read(full_path): sub_file = _get_path_within_zip(full_path) @@ -14,7 +16,7 @@ def read(full_path): def extract(full_path, dest): - logging.debug("extract %s to %s", full_path, dest) + LOGGER.debug("extract %s to %s", full_path, dest) sub_file = _get_path_within_zip(full_path) with zipfile.ZipFile(ROOT, "r") as zip_file: info = zip_file.getinfo(sub_file) @@ -23,8 +25,12 @@ def extract(full_path, dest): def _get_path_within_zip(full_path): - full_path = os.path.abspath(str(full_path)) - sub_file = full_path[len(ROOT) + 1 :] + full_path = os.path.realpath(os.path.abspath(str(full_path))) + prefix = f"{ROOT}{os.sep}" + if not full_path.startswith(prefix): + msg = f"full_path={full_path} should start with prefix={prefix}." + raise RuntimeError(msg) + sub_file = full_path[len(prefix) :] if IS_WIN: # paths are always UNIX separators, even on Windows, though __file__ still follows platform default sub_file = sub_file.replace(os.sep, "/") @@ -32,6 +38,6 @@ def _get_path_within_zip(full_path): __all__ = [ - "read", "extract", + "read", ] diff --git a/tasks/__main__zipapp.py b/tasks/__main__zipapp.py index a34fc5c33..9118e2007 100644 --- a/tasks/__main__zipapp.py +++ b/tasks/__main__zipapp.py @@ -4,22 +4,25 @@ import os import sys import zipfile +from functools import cached_property +from importlib.abc import SourceLoader +from importlib.util import spec_from_file_location ABS_HERE = os.path.abspath(os.path.dirname(__file__)) -NEW_IMPORT_SYSTEM = sys.version_info[0] >= 3 # noqa: PLR2004 class VersionPlatformSelect: def __init__(self) -> None: - self.archive = ABS_HERE - self._zip_file = zipfile.ZipFile(ABS_HERE, "r") + zipapp = ABS_HERE + self.archive = zipapp + self._zip_file = zipfile.ZipFile(zipapp) self.modules = self._load("modules.json") self.distributions = self._load("distributions.json") self.__cache = {} def _load(self, of_file): version = ".".join(str(i) for i in sys.version_info[0:2]) - per_version = json.loads(self.get_data(of_file).decode("utf-8")) + per_version = json.loads(self.get_data(of_file).decode()) all_platforms = per_version[version] if version in per_version else per_version["3.9"] content = all_platforms.get("==any", {}) # start will all platforms not_us = f"!={sys.platform}" @@ -65,25 +68,66 @@ def find_distributions(self, context): def __repr__(self) -> str: return f"{self.__class__.__name__}(path={ABS_HERE})" - def _register_distutils_finder(self): + def _register_distutils_finder(self): # noqa: C901 if "distlib" not in self.modules: return + class Resource: + def __init__(self, path: str, name: str, loader: SourceLoader) -> None: + self.path = os.path.join(path, name) + self._name = name + self.loader = loader + + @cached_property + def name(self) -> str: + return os.path.basename(self._name) + + @property + def bytes(self) -> bytes: + return self.loader.get_data(self._name) + + @property + def is_container(self) -> bool: + return len(self.resources) > 1 + + @cached_property + def resources(self) -> list[str]: + return [ + i.filename + for i in ( + (j for j in zip_file.filelist if j.filename.startswith(f"{self._name}/")) + if self._name + else zip_file.filelist + ) + ] + class DistlibFinder: def __init__(self, path, loader) -> None: self.path = path self.loader = loader def find(self, name): - class Resource: - def __init__(self, content) -> None: - self.bytes = content - - full_path = os.path.join(self.path, name) - return Resource(self.loader.get_data(full_path)) - - from distlib.resources import register_finder - + return Resource(self.path, name, self.loader) + + def iterator(self, resource_name): + resource = self.find(resource_name) + if resource is not None: + todo = [resource] + while todo: + resource = todo.pop(0) + yield resource + if resource.is_container: + resource_name = resource.name + for name in resource.resources: + child = self.find(f"{resource_name}/{name}" if resource_name else name) + if child.is_container: + todo.append(child) + else: + yield child + + from distlib.resources import register_finder # noqa: PLC0415 + + zip_file = self._zip_file register_finder(self, lambda module: DistlibFinder(os.path.dirname(module.__file__), self)) @@ -93,10 +137,7 @@ def __init__(self, content) -> None: def versioned_distribution_class(): global _VER_DISTRIBUTION_CLASS # noqa: PLW0603 if _VER_DISTRIBUTION_CLASS is None: - if sys.version_info >= (3, 8): - from importlib.metadata import Distribution - else: - from importlib_metadata import Distribution + from importlib.metadata import Distribution # noqa: PLC0415 class VersionedDistribution(Distribution): def __init__(self, file_loader, dist_path) -> None: @@ -113,48 +154,22 @@ def locate_file(self, path): return _VER_DISTRIBUTION_CLASS -if NEW_IMPORT_SYSTEM: - from importlib.abc import SourceLoader - from importlib.util import spec_from_file_location - - class VersionedFindLoad(VersionPlatformSelect, SourceLoader): - def find_spec(self, fullname, path, target=None): # noqa: ARG002 - zip_path = self.find_mod(fullname) - if zip_path is not None: - return spec_from_file_location(name=fullname, loader=self) - return None - - def module_repr(self, module): - raise NotImplementedError - -else: - from imp import new_module - - class VersionedFindLoad(VersionPlatformSelect): - def find_module(self, fullname, path=None): # noqa: ARG002 - return self if self.find_mod(fullname) else None +class VersionedFindLoad(VersionPlatformSelect, SourceLoader): + def find_spec(self, fullname, path, target=None): # noqa: ARG002 + zip_path = self.find_mod(fullname) + if zip_path is not None: + return spec_from_file_location(name=fullname, loader=self) + return None - def load_module(self, fullname): - filename = self.get_filename(fullname) - code = self.get_data(filename) - mod = sys.modules.setdefault(fullname, new_module(fullname)) - mod.__file__ = filename - mod.__loader__ = self - is_package = filename.endswith("__init__.py") - if is_package: - mod.__path__ = [os.path.dirname(filename)] - mod.__package__ = fullname - else: - mod.__package__ = fullname.rpartition(".")[0] - exec(code, mod.__dict__) # noqa: S102 - return mod + def module_repr(self, module): + raise NotImplementedError def run(): with VersionedFindLoad() as finder: sys.meta_path.insert(0, finder) finder._register_distutils_finder() # noqa: SLF001 - from virtualenv.__main__ import run as run_virtualenv + from virtualenv.__main__ import run as run_virtualenv # noqa: PLC0415, PLC2701 run_virtualenv() diff --git a/tasks/make_zipapp.py b/tasks/make_zipapp.py index bc74834e5..608efcf8c 100644 --- a/tasks/make_zipapp.py +++ b/tasks/make_zipapp.py @@ -1,4 +1,5 @@ """https://docs.python.org/3/library/zipapp.html.""" + from __future__ import annotations import argparse @@ -22,7 +23,7 @@ HERE = Path(__file__).parent.absolute() -VERSIONS = [f"3.{i}" for i in range(10, 6, -1)] +VERSIONS = [f"3.{i}" for i in range(13, 7, -1)] def main(): @@ -48,22 +49,22 @@ def create_zipapp(dest, packages): zip_app.writestr("__main__.py", (HERE / "__main__zipapp.py").read_bytes()) bio.seek(0) zipapp.create_archive(bio, dest) - print(f"zipapp created at {dest}") # noqa: T201 + print(f"zipapp created at {dest} with size {os.path.getsize(dest) / 1024 / 1024:.2f}MB") # noqa: T201 -def write_packages_to_zipapp(base, dist, modules, packages, zip_app): # noqa: C901 +def write_packages_to_zipapp(base, dist, modules, packages, zip_app): # noqa: C901, PLR0912 has = set() - for name, p_w_v in packages.items(): + for name, p_w_v in packages.items(): # noqa: PLR1702 for platform, w_v in p_w_v.items(): for wheel_data in w_v.values(): wheel = wheel_data.wheel with zipfile.ZipFile(str(wheel)) as wheel_zip: for filename in wheel_zip.namelist(): - if name in ("virtualenv",): + if name == "virtualenv": dest = PurePosixPath(filename) else: dest = base / wheel.stem / filename - if dest.suffix in (".so", ".pyi"): + if dest.suffix in {".so", ".pyi"}: continue if dest.suffix == ".py": key = filename[:-3].replace("/", ".").replace("__init__", "").rstrip(".") @@ -92,7 +93,7 @@ def __init__(self, into) -> None: self.into = into self.collected = defaultdict(lambda: defaultdict(dict)) self.pip_cmd = [str(Path(sys.executable).parent / "pip")] - self._cmd = [*self.pip_cmd, "download", "-q", "--no-deps", "--dest", str(self.into)] + self._cmd = [*self.pip_cmd, "download", "-q", "--no-deps", "--no-cache-dir", "--dest", str(self.into)] def run(self, target, versions): whl = self.build_sdist(target) @@ -159,7 +160,9 @@ def get_dependencies(whl, version): req = Requirement(dep) markers = getattr(req.marker, "_markers", ()) or () if any( - m for m in markers if isinstance(m, tuple) and len(m) == 3 and m[0].value == "extra" # noqa: PLR2004 + m + for m in markers + if isinstance(m, tuple) and len(m) == 3 and m[0].value == "extra" # noqa: PLR2004 ): continue py_versions = WheelDownloader._marker_at(markers, "python_version") @@ -177,7 +180,7 @@ def get_dependencies(whl, version): platforms = [] platform_positions = WheelDownloader._marker_at(markers, "sys_platform") deleted = 0 - for pos in platform_positions: # can only be ore meaningfully + for pos in platform_positions: # can only be or meaningfully platform = f"{markers[pos][1].value}{markers[pos][2].value}" deleted += WheelDownloader._del_marker_at(markers, pos - deleted) platforms.append(platform) @@ -236,7 +239,7 @@ def _build_sdist(self, folder, target): def run_suppress_output(cmd, stop_print_on_fail=False): # noqa: FBT002 process = subprocess.Popen( - cmd, # noqa: S603 + cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True, @@ -276,7 +279,7 @@ def get_wheels_for_support_versions(folder): class WheelForVersion: def __init__(self, wheel=None, versions=None) -> None: self.wheel = wheel - self.versions = versions if versions else [] + self.versions = versions or [] def __repr__(self) -> str: return f"{self.__class__.__name__}({self.wheel!r}, {self.versions!r})" diff --git a/tasks/release.py b/tasks/release.py index 21c92ec3f..feda8b3c2 100644 --- a/tasks/release.py +++ b/tasks/release.py @@ -1,4 +1,5 @@ """Handles creating a release PR.""" + from __future__ import annotations from pathlib import Path @@ -53,7 +54,7 @@ def get_upstream(repo: Repo) -> Remote: def release_changelog(repo: Repo, version: Version) -> Commit: print("generate release commit") # noqa: T201 - check_call(["towncrier", "build", "--yes", "--version", version.public], cwd=str(ROOT_SRC_DIR)) # noqa: S603, S607 + check_call(["towncrier", "build", "--yes", "--version", version.public], cwd=str(ROOT_SRC_DIR)) # noqa: S607 return repo.index.commit(f"release {version}") diff --git a/tasks/update_embedded.py b/tasks/update_embedded.py index e97d90d83..933e033f6 100755 --- a/tasks/update_embedded.py +++ b/tasks/update_embedded.py @@ -3,6 +3,7 @@ from __future__ import annotations import codecs +import locale import os import re from zlib import crc32 as _crc32 @@ -19,12 +20,12 @@ def crc32(data): gzip = codecs.lookup("zlib") b64 = codecs.lookup("base64") -file_regex = re.compile(r'# file (.*?)\n([a-zA-Z][a-zA-Z0-9_]+) = convert\(\n {4}"""\n(.*?)"""\n\)', re.S) +file_regex = re.compile(r'# file (.*?)\n([a-zA-Z][a-zA-Z0-9_]+) = convert\(\n {4}"""\n(.*?)"""\n\)', re.DOTALL) file_template = '# file {filename}\n{variable} = convert(\n """\n{data}"""\n)' def rebuild(script_path): - with open(script_path) as current_fh: + with script_path.open(encoding=locale.getpreferredencoding(False)) as current_fh: # noqa: FBT003 script_content = current_fh.read() script_parts = [] match_end = 0 @@ -49,7 +50,7 @@ def handle_file(previous_content, filename, variable_name, previous_encoded): print(f"Found file {filename}") # noqa: T201 current_path = os.path.realpath(os.path.join(here, "..", "src", "virtualenv_embedded", filename)) _, file_type = os.path.splitext(current_path) - keep_line_ending = file_type in (".bat",) + keep_line_ending = file_type == ".bat" with open(current_path, encoding="utf-8", newline="" if keep_line_ending else None) as current_fh: current_text = current_fh.read() current_crc = crc32(current_text) @@ -68,8 +69,7 @@ def handle_file(previous_content, filename, variable_name, previous_encoded): def report(exit_code, new, next_match, current, script_path): if new != current: print("Content updated; overwriting... ", end="") # noqa: T201 - with open(script_path, "w") as current_fh: - current_fh.write(new) + script_path.write_bytes(new) print("done.") # noqa: T201 else: print("No changes in content") # noqa: T201 diff --git a/tasks/upgrade_wheels.py b/tasks/upgrade_wheels.py index ee5c15b64..e752b1dd1 100644 --- a/tasks/upgrade_wheels.py +++ b/tasks/upgrade_wheels.py @@ -15,18 +15,19 @@ STRICT = "UPGRADE_ADVISORY" not in os.environ BUNDLED = ["pip", "setuptools", "wheel"] -SUPPORT = [(3, i) for i in range(7, 13)] +SUPPORT = [(3, i) for i in range(8, 15)] DEST = Path(__file__).resolve().parents[1] / "src" / "virtualenv" / "seed" / "wheels" / "embed" def download(ver, dest, package): subprocess.call( - [ # noqa: S603 + [ sys.executable, "-m", "pip", "--disable-pip-version-check", "download", + "--no-cache-dir", "--only-binary=:all:", "--python-version", ver, @@ -37,7 +38,7 @@ def download(ver, dest, package): ) -def run(): # noqa: C901 +def run(): # noqa: C901, PLR0912 old_batch = {i.name for i in DEST.iterdir() if i.suffix == ".whl"} with TemporaryDirectory() as temp: temp_path = Path(temp) @@ -49,6 +50,8 @@ def run(): # noqa: C901 into.mkdir() folders[into] = support_ver for package in BUNDLED: + if package == "wheel" and support >= (3, 9): + continue thread = Thread(target=download, args=(support_ver, str(into), package)) targets.append(thread) thread.start() @@ -66,8 +69,8 @@ def run(): # noqa: C901 added = collect_package_versions(new_packages) removed = collect_package_versions(remove_packages) - outcome = (1 if STRICT else 0) if (added or removed) else 0 + print(f"Outcome {outcome} added {added} removed {removed}") # noqa: T201 lines = ["Upgrade embedded wheels:", ""] for key, versions in added.items(): text = f"* {key} to {fmt_version(versions)}" @@ -89,8 +92,10 @@ def run(): # noqa: C901 if (folder / package).exists(): support_table[version].append(package) support_table = {k: OrderedDict((i.split("-")[0], i) for i in v) for k, v in support_table.items()} - bundle = ",".join( - f"{v!r}: {{ {','.join(f'{p!r}: {f!r}' for p, f in line.items())} }}" for v, line in support_table.items() + nl = "\n" + bundle = "".join( + f"\n {v!r}: {{{nl}{''.join(f' {p!r}: {f!r},{nl}' for p, f in line.items())} }}," + for v, line in support_table.items() ) msg = dedent( f""" @@ -104,10 +109,13 @@ def run(): # noqa: C901 def get_embed_wheel(distribution, for_py_version): - path = BUNDLE_FOLDER / (BUNDLE_SUPPORT.get(for_py_version, {{}}) or BUNDLE_SUPPORT[MAX]).get(distribution) + mapping = BUNDLE_SUPPORT.get(for_py_version, {{}}) or BUNDLE_SUPPORT[MAX] + wheel_file = mapping.get(distribution) + if wheel_file is None: + return None + path = BUNDLE_FOLDER / wheel_file return Wheel.from_path(path) - __all__ = [ "get_embed_wheel", "BUNDLE_SUPPORT", @@ -119,8 +127,8 @@ def get_embed_wheel(distribution, for_py_version): ) dest_target = DEST / "__init__.py" dest_target.write_text(msg, encoding="utf-8") - - subprocess.run([sys.executable, "-m", "black", str(dest_target)], check=False) # noqa: S603 + subprocess.run([sys.executable, "-m", "ruff", "check", str(dest_target), "--fix", "--unsafe-fixes"]) + subprocess.run([sys.executable, "-m", "ruff", "format", str(dest_target), "--preview"]) raise SystemExit(outcome) diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/conftest.py b/tests/conftest.py index 1175aa952..4fcb25da8 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -12,20 +12,23 @@ import pytest from virtualenv.app_data import AppDataDiskFolder +from virtualenv.cache import FileCache from virtualenv.discovery.py_info import PythonInfo -from virtualenv.info import IS_PYPY, IS_WIN, fs_supports_symlink +from virtualenv.info import IS_GRAALPY, IS_PYPY, IS_WIN, fs_supports_symlink from virtualenv.report import LOGGER +from virtualenv.run.plugin.creators import CreatorSelector def pytest_addoption(parser): parser.addoption("--int", action="store_true", default=False, help="run integration tests") + parser.addoption("--skip-slow", action="store_true", default=False, help="skip slow tests") def pytest_configure(config): """Ensure randomly is called before we re-order""" manager = config.pluginmanager - order = manager.hook.pytest_collection_modifyitems._hookimpls # noqa: SLF001 + order = manager.hook.pytest_collection_modifyitems.get_hookimpls() dest = next((i for i, p in enumerate(order) if p.plugin is manager.getplugin("randomly")), None) if dest is not None: from_pos = next(i for i, p in enumerate(order) if p.plugin is manager.getplugin(__file__)) @@ -46,6 +49,11 @@ def pytest_collection_modifyitems(config, items): if item.location[0].startswith(int_location): item.add_marker(pytest.mark.skip(reason="need --int option to run")) + if config.getoption("--skip-slow"): + for item in items: + if "slow" in [mark.name for mark in item.iter_markers()]: + item.add_marker(pytest.mark.skip(reason="skipped because --skip-slow was passed")) + @pytest.fixture(scope="session") def has_symlink_support(tmp_path_factory): # noqa: ARG001 @@ -58,7 +66,7 @@ def link_folder(has_symlink_support): return os.symlink if sys.platform == "win32": # on Windows junctions may be used instead - import _winapi + import _winapi # noqa: PLC0415 return getattr(_winapi, "CreateJunction", None) return None @@ -118,15 +126,16 @@ def _check_cwd_not_changed_by_test(): @pytest.fixture(autouse=True) def _ensure_py_info_cache_empty(session_app_data): - PythonInfo.clear_cache(session_app_data) + cache = FileCache(session_app_data.py_info, session_app_data.py_info_clear) + PythonInfo.clear_cache(cache) yield - PythonInfo.clear_cache(session_app_data) + PythonInfo.clear_cache(cache) @contextmanager def change_os_environ(key, value): env_var = key - previous = os.environ[env_var] if env_var in os.environ else None + previous = os.environ.get(env_var, None) os.environ[env_var] = value try: yield @@ -197,7 +206,7 @@ def coverage_env(monkeypatch, link, request): """ if COVERAGE_RUN and "_no_coverage" not in request.fixturenames: # we inject right after creation, we cannot collect coverage on site.py - used for helper scripts, such as debug - from virtualenv import run + from virtualenv import run # noqa: PLC0415 def _session_via_cli(args, options, setup_logging, env=None): session = prev_run(args, options, setup_logging, env) @@ -206,7 +215,7 @@ def _session_via_cli(args, options, setup_logging, env=None): def create_run(): result = old_run() obj["cov"] = EnableCoverage(link) - obj["cov"].__enter__(session.creator) + obj["cov"].__enter__(session.creator) # noqa: PLC2801 return result monkeypatch.setattr(session.creator, "run", create_run) @@ -234,7 +243,7 @@ def finish(): # _no_coverage tells coverage_env to disable coverage injection for _no_coverage user. -@pytest.fixture() +@pytest.fixture def _no_coverage(): pass @@ -275,7 +284,11 @@ def is_inside_ci(): @pytest.fixture(scope="session") def special_char_name(): - base = "e-$ รจั€ั‚๐Ÿš’โ™žไธญ็‰‡-j" + base = "'\";&&e-$ รจั€ั‚๐Ÿš’โ™žไธญ็‰‡-j" + if IS_WIN: + # get rid of invalid characters on Windows + base = base.replace('"', "") + base = base.replace(";", "") # workaround for pypy3 https://bitbucket.org/pypy/pypy/issues/3147/venv-non-ascii-support-windows encoding = "ascii" if IS_WIN else sys.getfilesystemencoding() # let's not include characters that the file system cannot encode) @@ -291,14 +304,15 @@ def special_char_name(): return result -@pytest.fixture() +@pytest.fixture def special_name_dir(tmp_path, special_char_name): return Path(str(tmp_path)) / special_char_name @pytest.fixture(scope="session") def current_creators(session_app_data): - return PythonInfo.current_system(session_app_data).creators() + cache = FileCache(session_app_data.py_info, session_app_data.py_info_clear) + return CreatorSelector.for_interpreter(PythonInfo.current_system(session_app_data, cache)) @pytest.fixture(scope="session") @@ -332,7 +346,7 @@ def change_env_var(key, value): del os.environ[key] # pragma: no cover -@pytest.fixture() +@pytest.fixture def temp_app_data(monkeypatch, tmp_path): app_data = tmp_path / "app-data" monkeypatch.setenv("VIRTUALENV_OVERRIDE_APP_DATA", str(app_data)) @@ -344,21 +358,22 @@ def for_py_version(): return f"{sys.version_info.major}.{sys.version_info.minor}" -@pytest.fixture() +@pytest.fixture def _skip_if_test_in_system(session_app_data): - current = PythonInfo.current(session_app_data) + cache = FileCache(session_app_data.py_info, session_app_data.py_info_clear) + current = PythonInfo.current(session_app_data, cache) if current.system_executable is not None: pytest.skip("test not valid if run under system") -if IS_PYPY: +if IS_PYPY or IS_GRAALPY: - @pytest.fixture() + @pytest.fixture def time_freeze(freezer): return freezer.move_to else: - @pytest.fixture() + @pytest.fixture def time_freeze(time_machine): return lambda s: time_machine.move_to(s, tick=False) diff --git a/tests/integration/test_cachedir_tag.py b/tests/integration/test_cachedir_tag.py new file mode 100644 index 000000000..b3b6a1dfe --- /dev/null +++ b/tests/integration/test_cachedir_tag.py @@ -0,0 +1,32 @@ +from __future__ import annotations + +import shutil +import sys +from subprocess import check_output, run +from typing import TYPE_CHECKING + +import pytest + +from virtualenv import cli_run + +if TYPE_CHECKING: + from pathlib import Path + +# gtar => gnu-tar on macOS +TAR = next((target for target in ("gtar", "tar") if shutil.which(target)), None) + + +def compatible_is_tar_present() -> bool: + return TAR and "--exclude-caches" in check_output(args=[TAR, "--help"], text=True, encoding="utf-8") + + +@pytest.mark.skipif(sys.platform == "win32", reason="Windows does not have tar") +@pytest.mark.skipif(not compatible_is_tar_present(), reason="Compatible tar is not installed") +def test_cachedir_tag_ignored_by_tag(tmp_path: Path) -> None: + venv = tmp_path / ".venv" + cli_run(["--activators", "", "--without-pip", str(venv)]) + + args = [TAR, "--create", "--file", "/dev/null", "--exclude-caches", "--verbose", venv.name] + tar_result = run(args=args, capture_output=True, text=True, encoding="utf-8", cwd=tmp_path) + assert tar_result.stdout == ".venv/\n.venv/CACHEDIR.TAG\n" + assert tar_result.stderr == f"{TAR}: .venv/: contains a cache directory tag CACHEDIR.TAG; contents not dumped\n" diff --git a/tests/integration/test_zipapp.py b/tests/integration/test_zipapp.py index 5bcddf519..e40ecca91 100644 --- a/tests/integration/test_zipapp.py +++ b/tests/integration/test_zipapp.py @@ -6,42 +6,51 @@ from pathlib import Path import pytest -from flaky import flaky +from virtualenv.cache import FileCache from virtualenv.discovery.py_info import PythonInfo +from virtualenv.info import fs_supports_symlink from virtualenv.run import cli_run HERE = Path(__file__).parent -CURRENT = PythonInfo.current_system() @pytest.fixture(scope="session") -def zipapp_build_env(tmp_path_factory): +def current_info(session_app_data): + cache = FileCache(session_app_data.py_info, session_app_data.py_info_clear) + return PythonInfo.current_system(session_app_data, cache) + + +@pytest.fixture(scope="session") +def zipapp_build_env(tmp_path_factory, current_info): create_env_path = None - if CURRENT.implementation != "PyPy": - exe = CURRENT.executable # guaranteed to contain a recent enough pip (tox.ini) + if current_info.implementation not in {"PyPy", "GraalVM"}: + exe = current_info.executable # guaranteed to contain a recent enough pip (tox.ini) else: create_env_path = tmp_path_factory.mktemp("zipapp-create-env") exe, found = None, False # prefer CPython as builder as pypy is slow for impl in ["cpython", ""]: - for version in range(11, 6, -1): - with suppress(Exception): - # create a virtual environment which is also guaranteed to contain a recent enough pip (bundled) - session = cli_run( - [ - "-vvv", - "-p", - f"{impl}3.{version}", - "--activators", - "", - str(create_env_path), - "--no-download", - "--no-periodic-update", - ], - ) - exe = str(session.creator.exe) - found = True + for threaded in ["", "t"]: + for version in range(11, 6, -1): + with suppress(Exception): + # create a virtual environment which is also guaranteed to contain a recent enough pip (bundled) + session = cli_run( + [ + "-vvv", + "-p", + f"{impl}3.{version}{threaded}", + "--activators", + "", + str(create_env_path), + "--no-download", + "--no-periodic-update", + ], + ) + exe = str(session.creator.exe) + found = True + break + if found: break if found: break @@ -74,7 +83,7 @@ def zipapp_test_env(tmp_path_factory): shutil.rmtree(str(base_path)) -@pytest.fixture() +@pytest.fixture def call_zipapp(zipapp, tmp_path, zipapp_test_env, temp_app_data): # noqa: ARG001 def _run(*args): cmd = [str(zipapp_test_env), str(zipapp), "-vv", str(tmp_path / "env"), *list(args)] @@ -83,13 +92,31 @@ def _run(*args): return _run -@flaky(max_runs=2, min_passes=1) +@pytest.fixture +def call_zipapp_symlink(zipapp, tmp_path, zipapp_test_env, temp_app_data): # noqa: ARG001 + def _run(*args): + symlinked = zipapp.parent / "symlinked_virtualenv.pyz" + symlinked.symlink_to(str(zipapp)) + cmd = [str(zipapp_test_env), str(symlinked), "-vv", str(tmp_path / "env"), *list(args)] + subprocess.check_call(cmd) + + return _run + + +@pytest.mark.skipif(not fs_supports_symlink(), reason="symlink not supported") +def test_zipapp_in_symlink(capsys, call_zipapp_symlink): + call_zipapp_symlink("--reset-app-data") + _out, err = capsys.readouterr() + assert not err + + def test_zipapp_help(call_zipapp, capsys): call_zipapp("-h") - out, err = capsys.readouterr() + _out, err = capsys.readouterr() assert not err +@pytest.mark.slow @pytest.mark.parametrize("seeder", ["app-data", "pip"]) def test_zipapp_create(call_zipapp, seeder): call_zipapp("--seeder", seeder) diff --git a/tests/unit/activation/conftest.py b/tests/unit/activation/conftest.py index 4811d0727..a7186896f 100644 --- a/tests/unit/activation/conftest.py +++ b/tests/unit/activation/conftest.py @@ -6,7 +6,6 @@ import sys from os.path import dirname, normcase from pathlib import Path -from shlex import quote from subprocess import Popen import pytest @@ -15,7 +14,7 @@ class ActivationTester: - def __init__(self, of_class, session, cmd, activate_script, extension) -> None: # noqa: PLR0913 + def __init__(self, of_class, session, cmd, activate_script, extension) -> None: self.of_class = of_class self._creator = session.creator self._version_cmd = [cmd, "--version"] @@ -41,13 +40,13 @@ def get_version(self, raise_on_fail): encoding="utf-8", ) out, err = process.communicate() - except Exception as exception: # noqa: BLE001 + except Exception as exception: self._version = exception if raise_on_fail: raise return RuntimeError(f"{self} is not available due {exception}") else: - result = out if out else err + result = out or err self._version = result return result return self._version @@ -78,7 +77,8 @@ def __call__(self, monkeypatch, tmp_path): try: process = Popen(invoke, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env=env) raw_, _ = process.communicate() - raw = raw_.decode() + raw = raw_.decode(errors="replace") + assert process.returncode == 0, raw except subprocess.CalledProcessError as exception: output = exception.output + exception.stderr assert not exception.returncode, output # noqa: PT017 @@ -130,13 +130,13 @@ def _get_test_lines(self, activate_script): ] def assert_output(self, out, raw, tmp_path): - # pre-activation + """Compare _get_test_lines() with the expected values.""" assert out[0], raw assert out[1] == "None", raw assert out[2] == "None", raw - # post-activation - expected = self._creator.exe.parent / os.path.basename(sys.executable) - assert self.norm_path(out[3]) == self.norm_path(expected), raw + # self.activate_call(activate_script) runs at this point + python_exe = self._creator.exe.parent / os.path.basename(sys.executable) + assert self.norm_path(out[3]) == self.norm_path(python_exe), raw assert self.norm_path(out[4]) == self.norm_path(self._creator.dest).replace("\\\\", "\\"), raw assert out[5] == self._creator.env_name # Some attempts to test the prompt output print more than 1 line. @@ -153,7 +153,7 @@ def assert_output(self, out, raw, tmp_path): assert out[-1] == "None", raw def quote(self, s): - return quote(s) + return self.of_class.quote(s) def python_cmd(self, cmd): return f"{os.path.basename(sys.executable)} -c {self.quote(cmd)}" @@ -182,7 +182,7 @@ def norm_path(path): if sys.platform != "win32": result = path else: - from ctypes import create_unicode_buffer, windll + from ctypes import create_unicode_buffer, windll # noqa: PLC0415 buffer_cont = create_unicode_buffer(256) get_long_path_name = windll.kernel32.GetLongPathNameW @@ -212,8 +212,8 @@ def __call__(self, monkeypatch, tmp_path): stderr=subprocess.PIPE, env=env, ) - out, _err = process.communicate() - err = _err.decode("utf-8") + _out, err_ = process.communicate() + err = err_.decode("utf-8") assert process.returncode assert self.non_source_fail_message in err @@ -229,9 +229,19 @@ def raise_on_non_source_class(): @pytest.fixture(scope="session", params=[True, False], ids=["with_prompt", "no_prompt"]) -def activation_python(request, tmp_path_factory, special_char_name, current_fastest): +def activation_python(request, tmp_path_factory, special_char_name, current_fastest, session_app_data): dest = os.path.join(str(tmp_path_factory.mktemp("activation-tester-env")), special_char_name) - cmd = ["--without-pip", dest, "--creator", current_fastest, "-vv", "--no-periodic-update"] + cmd = [ + "--without-pip", + dest, + "--creator", + current_fastest, + "-vv", + "--no-periodic-update", + "--app-data", + str(session_app_data), + ] + # `params` is accessed here. https://docs.pytest.org/en/stable/reference/reference.html#pytest-fixture if request.param: cmd += ["--prompt", special_char_name] session = cli_run(cmd) @@ -240,7 +250,7 @@ def activation_python(request, tmp_path_factory, special_char_name, current_fast return session -@pytest.fixture() +@pytest.fixture def activation_tester(activation_python, monkeypatch, tmp_path, is_inside_ci): def _tester(tester_class): tester = tester_class(activation_python) diff --git a/tests/unit/activation/test_bash.py b/tests/unit/activation/test_bash.py index 4c92b733c..0a9c588cf 100644 --- a/tests/unit/activation/test_bash.py +++ b/tests/unit/activation/test_bash.py @@ -1,5 +1,7 @@ from __future__ import annotations +from argparse import Namespace + import pytest from virtualenv.activation import BashActivator @@ -7,7 +9,63 @@ @pytest.mark.skipif(IS_WIN, reason="Github Actions ships with WSL bash") -def test_bash(raise_on_non_source_class, activation_tester): +@pytest.mark.parametrize( + ("tcl_lib", "tk_lib", "present"), + [ + ("/path/to/tcl", "/path/to/tk", True), + (None, None, False), + ], +) +def test_bash_tkinter_generation(tmp_path, tcl_lib, tk_lib, present): + # GIVEN + class MockInterpreter: + pass + + interpreter = MockInterpreter() + interpreter.tcl_lib = tcl_lib + interpreter.tk_lib = tk_lib + + class MockCreator: + def __init__(self, dest): + self.dest = dest + self.bin_dir = dest / "bin" + self.bin_dir.mkdir() + self.interpreter = interpreter + self.pyenv_cfg = {} + self.env_name = "my-env" + + creator = MockCreator(tmp_path) + options = Namespace(prompt=None) + activator = BashActivator(options) + + # WHEN + activator.generate(creator) + content = (creator.bin_dir / "activate").read_text(encoding="utf-8") + + # THEN + # The teardown logic is always present in deactivate() + assert "unset _OLD_VIRTUAL_TCL_LIBRARY" in content + assert "unset _OLD_VIRTUAL_TK_LIBRARY" in content + + if present: + assert 'if [ /path/to/tcl != "" ]; then' in content + assert "TCL_LIBRARY=/path/to/tcl" in content + assert "export TCL_LIBRARY" in content + + assert 'if [ /path/to/tk != "" ]; then' in content + assert "TK_LIBRARY=/path/to/tk" in content + assert "export TK_LIBRARY" in content + else: + # When not present, the if condition is false, so the block is not executed + assert "if [ '' != \"\" ]; then" in content, content + assert "TCL_LIBRARY=''" in content + # The export is inside the if, so this is fine + assert "export TCL_LIBRARY" in content + + +@pytest.mark.skipif(IS_WIN, reason="Github Actions ships with WSL bash") +@pytest.mark.parametrize("hashing_enabled", [True, False]) +def test_bash(raise_on_non_source_class, hashing_enabled, activation_tester): class Bash(raise_on_non_source_class): def __init__(self, session) -> None: super().__init__( @@ -18,6 +76,11 @@ def __init__(self, session) -> None: "sh", "You must source this script: $ source ", ) + self.deactivate += " || exit 1" + self._invoke_script.append("-h" if hashing_enabled else "+h") + + def activate_call(self, script): + return super().activate_call(script) + " || exit 1" def print_prompt(self): return self.print_os_env_var("PS1") diff --git a/tests/unit/activation/test_batch.py b/tests/unit/activation/test_batch.py index 10dc41b1a..db2595860 100644 --- a/tests/unit/activation/test_batch.py +++ b/tests/unit/activation/test_batch.py @@ -1,12 +1,57 @@ from __future__ import annotations -from shlex import quote +from argparse import Namespace import pytest from virtualenv.activation import BatchActivator +@pytest.mark.parametrize( + ("tcl_lib", "tk_lib", "present"), + [ + ("C:\\tcl", "C:\\tk", True), + (None, None, False), + ], +) +def test_batch_tkinter_generation(tmp_path, tcl_lib, tk_lib, present): + # GIVEN + class MockInterpreter: + os = "nt" + + interpreter = MockInterpreter() + interpreter.tcl_lib = tcl_lib + interpreter.tk_lib = tk_lib + + class MockCreator: + def __init__(self, dest): + self.dest = dest + self.bin_dir = dest / "bin" + self.bin_dir.mkdir() + self.interpreter = interpreter + self.pyenv_cfg = {} + self.env_name = "my-env" + + creator = MockCreator(tmp_path) + options = Namespace(prompt=None) + activator = BatchActivator(options) + + # WHEN + activator.generate(creator) + activate_content = (creator.bin_dir / "activate.bat").read_text(encoding="utf-8") + deactivate_content = (creator.bin_dir / "deactivate.bat").read_text(encoding="utf-8") + + # THEN + if present: + assert '@if NOT "C:\\tcl"=="" @set "TCL_LIBRARY=C:\\tcl"' in activate_content + assert '@if NOT "C:\\tk"=="" @set "TK_LIBRARY=C:\\tk"' in activate_content + assert "if defined _OLD_VIRTUAL_TCL_LIBRARY" in deactivate_content + assert "if defined _OLD_VIRTUAL_TK_LIBRARY" in deactivate_content + else: + assert '@if NOT ""=="" @set "TCL_LIBRARY="' in activate_content + assert '@if NOT ""=="" @set "TK_LIBRARY="' in activate_content + + @pytest.mark.usefixtures("activation_python") def test_batch(activation_tester_class, activation_tester, tmp_path): version_script = tmp_path / "version.bat" @@ -23,14 +68,62 @@ def __init__(self, session) -> None: self.unix_line_ending = False def _get_test_lines(self, activate_script): - # for BATCH utf-8 support need change the character code page to 650001 - return ["@echo off", "", "chcp 65001 1>NUL", *super()._get_test_lines(activate_script)] + return ["@echo off", *super()._get_test_lines(activate_script)] + + def quote(self, s): + if '"' in s or " " in s: + text = s.replace('"', r"\"") + return f'"{text}"' + return s + + def print_prompt(self): + return 'echo "%PROMPT%"' + + activation_tester(Batch) + + +@pytest.mark.usefixtures("activation_python") +def test_batch_output(activation_tester_class, activation_tester, tmp_path): + version_script = tmp_path / "version.bat" + version_script.write_text("ver", encoding="utf-8") + + class Batch(activation_tester_class): + def __init__(self, session) -> None: + super().__init__(BatchActivator, session, None, "activate.bat", "bat") + self._version_cmd = [str(version_script)] + self._invoke_script = [] + self.deactivate = "call deactivate" + self.activate_cmd = "call" + self.pydoc_call = f"call {self.pydoc_call}" + self.unix_line_ending = False + + def _get_test_lines(self, activate_script): + """ + Build intermediary script which will be then called. + In the script just activate environment, call echo to get current + echo setting, and then deactivate. This ensures that echo setting + is preserved and no unwanted output appears. + """ + intermediary_script_path = str(tmp_path / "intermediary.bat") + activate_script_quoted = self.quote(str(activate_script)) + return [ + "@echo on", + f"@echo @call {activate_script_quoted} > {intermediary_script_path}", + f"@echo @echo >> {intermediary_script_path}", + f"@echo @deactivate >> {intermediary_script_path}", + f"@call {intermediary_script_path}", + ] + + def assert_output(self, out, raw, tmp_path): # noqa: ARG002 + assert out[0] == "ECHO is on.", raw def quote(self, s): - """double quotes needs to be single, and single need to be double""" - return "".join(("'" if c == '"' else ('"' if c == "'" else c)) for c in quote(s)) + if '"' in s or " " in s: + text = s.replace('"', r"\"") + return f'"{text}"' + return s def print_prompt(self): - return "echo %PROMPT%" + return 'echo "%PROMPT%"' activation_tester(Batch) diff --git a/tests/unit/activation/test_csh.py b/tests/unit/activation/test_csh.py index 309ae811e..5cea684ec 100644 --- a/tests/unit/activation/test_csh.py +++ b/tests/unit/activation/test_csh.py @@ -1,9 +1,66 @@ from __future__ import annotations +import sys +from argparse import Namespace +from shutil import which +from subprocess import check_output + +import pytest +from packaging.version import Version + from virtualenv.activation import CShellActivator +@pytest.mark.parametrize( + ("tcl_lib", "tk_lib", "present"), + [ + ("/path/to/tcl", "/path/to/tk", True), + (None, None, False), + ], +) +def test_cshell_tkinter_generation(tmp_path, tcl_lib, tk_lib, present): + # GIVEN + class MockInterpreter: + pass + + interpreter = MockInterpreter() + interpreter.tcl_lib = tcl_lib + interpreter.tk_lib = tk_lib + + class MockCreator: + def __init__(self, dest): + self.dest = dest + self.bin_dir = dest / "bin" + self.bin_dir.mkdir() + self.interpreter = interpreter + self.pyenv_cfg = {} + self.env_name = "my-env" + + creator = MockCreator(tmp_path) + options = Namespace(prompt=None) + activator = CShellActivator(options) + + # WHEN + activator.generate(creator) + content = (creator.bin_dir / "activate.csh").read_text(encoding="utf-8") + + if present: + assert "test $?_OLD_VIRTUAL_TCL_LIBRARY != 0" in content + assert "test $?_OLD_VIRTUAL_TK_LIBRARY != 0" in content + assert "setenv TCL_LIBRARY /path/to/tcl" in content + assert "setenv TK_LIBRARY /path/to/tk" in content + else: + assert "setenv TCL_LIBRARY ''" in content + + def test_csh(activation_tester_class, activation_tester): + exe = f"tcsh{'.exe' if sys.platform == 'win32' else ''}" + if which(exe): + version_text = check_output([exe, "--version"], text=True, encoding="utf-8") + version = Version(version_text.split(" ")[1]) + if version >= Version("6.24.14"): + pytest.skip("https://github.com/tcsh-org/tcsh/issues/117") + class Csh(activation_tester_class): def __init__(self, session) -> None: super().__init__(CShellActivator, session, "csh", "activate.csh", "csh") diff --git a/tests/unit/activation/test_fish.py b/tests/unit/activation/test_fish.py index 6a92a2790..c15a8f513 100644 --- a/tests/unit/activation/test_fish.py +++ b/tests/unit/activation/test_fish.py @@ -1,11 +1,57 @@ from __future__ import annotations +import os +import sys +from argparse import Namespace + import pytest from virtualenv.activation import FishActivator from virtualenv.info import IS_WIN +@pytest.mark.parametrize( + ("tcl_lib", "tk_lib", "present"), + [ + ("/path/to/tcl", "/path/to/tk", True), + (None, None, False), + ], +) +def test_fish_tkinter_generation(tmp_path, tcl_lib, tk_lib, present): + # GIVEN + class MockInterpreter: + pass + + interpreter = MockInterpreter() + interpreter.tcl_lib = tcl_lib + interpreter.tk_lib = tk_lib + + class MockCreator: + def __init__(self, dest): + self.dest = dest + self.bin_dir = dest / "bin" + self.bin_dir.mkdir() + self.interpreter = interpreter + self.pyenv_cfg = {} + self.env_name = "my-env" + + creator = MockCreator(tmp_path) + options = Namespace(prompt=None) + activator = FishActivator(options) + + # WHEN + activator.generate(creator) + content = (creator.bin_dir / "activate.fish").read_text(encoding="utf-8") + + # THEN + if present: + assert "set -gx TCL_LIBRARY '/path/to/tcl'" in content + assert "set -gx TK_LIBRARY '/path/to/tk'" in content + else: + assert "if test -n ''\n if set -q TCL_LIBRARY;" in content + assert "if test -n ''\n if set -q TK_LIBRARY;" in content + + @pytest.mark.skipif(IS_WIN, reason="we have not setup fish in CI yet") def test_fish(activation_tester_class, activation_tester, monkeypatch, tmp_path): monkeypatch.setenv("HOME", str(tmp_path)) @@ -20,4 +66,54 @@ def __init__(self, session) -> None: def print_prompt(self): return "fish_prompt" + def _get_test_lines(self, activate_script): + return [ + self.print_python_exe(), + self.print_os_env_var("VIRTUAL_ENV"), + self.print_os_env_var("VIRTUAL_ENV_PROMPT"), + self.print_os_env_var("PATH"), + self.activate_call(activate_script), + self.print_python_exe(), + self.print_os_env_var("VIRTUAL_ENV"), + self.print_os_env_var("VIRTUAL_ENV_PROMPT"), + self.print_os_env_var("PATH"), + self.print_prompt(), + # \\ loads documentation from the virtualenv site packages + self.pydoc_call, + self.deactivate, + self.print_python_exe(), + self.print_os_env_var("VIRTUAL_ENV"), + self.print_os_env_var("VIRTUAL_ENV_PROMPT"), + self.print_os_env_var("PATH"), + "", # just finish with an empty new line + ] + + def assert_output(self, out, raw, _): + """Compare _get_test_lines() with the expected values.""" + assert out[0], raw + assert out[1] == "None", raw + assert out[2] == "None", raw + # self.activate_call(activate_script) runs at this point + expected = self._creator.exe.parent / os.path.basename(sys.executable) + assert self.norm_path(out[4]) == self.norm_path(expected), raw + assert self.norm_path(out[5]) == self.norm_path(self._creator.dest).replace("\\\\", "\\"), raw + assert out[6] == self._creator.env_name + # Some attempts to test the prompt output print more than 1 line. + # So we need to check if the prompt exists on any of them. + prompt_text = f"({self._creator.env_name}) " + assert any(prompt_text in line for line in out[7:-5]), raw + + assert out[-5] == "wrote pydoc_test.html", raw + content = tmp_path / "pydoc_test.html" + assert content.exists(), raw + # post deactivation, same as before + assert out[-4] == out[0], raw + assert out[-3] == "None", raw + assert out[-2] == "None", raw + + # Check that the PATH is restored + assert out[3] == out[13], raw + # Check that PATH changed after activation + assert out[3] != out[8], raw + activation_tester(Fish) diff --git a/tests/unit/activation/test_nushell.py b/tests/unit/activation/test_nushell.py index fbf75e397..08c5cb1a1 100644 --- a/tests/unit/activation/test_nushell.py +++ b/tests/unit/activation/test_nushell.py @@ -1,11 +1,48 @@ from __future__ import annotations +from argparse import Namespace from shutil import which from virtualenv.activation import NushellActivator from virtualenv.info import IS_WIN +def test_nushell_tkinter_generation(tmp_path): + # GIVEN + class MockInterpreter: + pass + + interpreter = MockInterpreter() + interpreter.tcl_lib = "/path/to/tcl" + interpreter.tk_lib = "/path/to/tk" + quoted_tcl_path = NushellActivator.quote(interpreter.tcl_lib) + quoted_tk_path = NushellActivator.quote(interpreter.tk_lib) + + class MockCreator: + def __init__(self, dest): + self.dest = dest + self.bin_dir = dest / "bin" + self.bin_dir.mkdir() + self.interpreter = interpreter + self.pyenv_cfg = {} + self.env_name = "my-env" + + creator = MockCreator(tmp_path) + options = Namespace(prompt=None) + activator = NushellActivator(options) + + # WHEN + activator.generate(creator) + content = (creator.bin_dir / "activate.nu").read_text(encoding="utf-8") + + # THEN + expected_tcl = f"let $new_env = $new_env | insert TCL_LIBRARY {quoted_tcl_path}" + expected_tk = f"let $new_env = $new_env | insert TK_LIBRARY {quoted_tk_path}" + + assert expected_tcl in content + assert expected_tk in content + + def test_nushell(activation_tester_class, activation_tester): class Nushell(activation_tester_class): def __init__(self, session) -> None: diff --git a/tests/unit/activation/test_powershell.py b/tests/unit/activation/test_powershell.py index c454d6943..2a48956cf 100644 --- a/tests/unit/activation/test_powershell.py +++ b/tests/unit/activation/test_powershell.py @@ -1,14 +1,60 @@ from __future__ import annotations import sys -from shlex import quote +from argparse import Namespace import pytest from virtualenv.activation import PowerShellActivator -@pytest.mark.slow() +@pytest.mark.parametrize( + ("tcl_lib", "tk_lib", "present"), + [ + ("C:\\tcl", "C:\\tk", True), + (None, None, False), + ], +) +def test_powershell_tkinter_generation(tmp_path, tcl_lib, tk_lib, present): + # GIVEN + class MockInterpreter: + os = "nt" + + interpreter = MockInterpreter() + interpreter.tcl_lib = tcl_lib + interpreter.tk_lib = tk_lib + + class MockCreator: + def __init__(self, dest): + self.dest = dest + self.bin_dir = dest / "bin" + self.bin_dir.mkdir() + self.interpreter = interpreter + self.pyenv_cfg = {} + self.env_name = "my-env" + + creator = MockCreator(tmp_path) + options = Namespace(prompt=None) + activator = PowerShellActivator(options) + + # WHEN + activator.generate(creator) + content = (creator.bin_dir / "activate.ps1").read_text(encoding="utf-8-sig") + + # THEN + if present: + assert "if ('C:\\tcl' -ne \"\")" in content + assert "$env:TCL_LIBRARY = 'C:\\tcl'" in content + assert "if ('C:\\tk' -ne \"\")" in content + assert "$env:TK_LIBRARY = 'C:\\tk'" in content + assert "if (Test-Path variable:_OLD_VIRTUAL_TCL_LIBRARY)" in content + assert "if (Test-Path variable:_OLD_VIRTUAL_TK_LIBRARY)" in content + else: + assert "if ('' -ne \"\")" in content + assert "$env:TCL_LIBRARY = ''" in content + + +@pytest.mark.slow def test_powershell(activation_tester_class, activation_tester, monkeypatch): monkeypatch.setenv("TERM", "xterm") @@ -19,14 +65,9 @@ def __init__(self, session) -> None: self._version_cmd = [cmd, "-c", "$PSVersionTable"] self._invoke_script = [cmd, "-ExecutionPolicy", "ByPass", "-File"] self.activate_cmd = "." - self.script_encoding = "utf-16" - - def quote(self, s): - """powershell double quote needed for quotes within single quotes""" - return quote(s).replace('"', '""') + self.script_encoding = "utf-8-sig" def _get_test_lines(self, activate_script): - # for BATCH utf-8 support need change the character code page to 650001 return super()._get_test_lines(activate_script) def invoke_script(self): @@ -35,4 +76,19 @@ def invoke_script(self): def print_prompt(self): return "prompt" + def quote(self, s): + """ + Tester will pass strings to native commands on Windows so extra + parsing rules are used. Check `PowerShellActivator.quote` for more + details. + """ + text = PowerShellActivator.quote(s) + return text.replace('"', '""') if sys.platform == "win32" else text + + def activate_call(self, script): + # Commands are called without quotes in PowerShell + cmd = self.activate_cmd + scr = self.quote(str(script)) + return f"{cmd} {scr}".strip() + activation_tester(PowerShell) diff --git a/tests/unit/activation/test_python_activator.py b/tests/unit/activation/test_python_activator.py index f773f74a6..24a3561c5 100644 --- a/tests/unit/activation/test_python_activator.py +++ b/tests/unit/activation/test_python_activator.py @@ -18,7 +18,7 @@ def __init__(self, session) -> None: sys.executable, activate_script="activate_this.py", extension="py", - non_source_fail_message="You must use exec(open(this_file).read(), {'__file__': this_file}))", + non_source_fail_message="You must use import runpy; runpy.run_path(this_file)", ) self.unix_line_ending = not IS_WIN @@ -36,6 +36,7 @@ def _get_test_lines(activate_script): import os import sys import platform + import runpy def print_r(value): print(repr(value)) @@ -47,10 +48,7 @@ def print_r(value): file_at = {str(activate_script)!r} # CPython 2 requires non-ascii path open to be unicode - with open(file_at, "r", encoding='utf-8') as file_handler: - content = file_handler.read() - exec(content, {{"__file__": file_at}}) - + runpy.run_path(file_at) print_r(os.environ.get("VIRTUAL_ENV")) print_r(os.environ.get("VIRTUAL_ENV_PROMPT")) print_r(os.environ.get("PATH").split(os.pathsep)) diff --git a/tests/unit/config/cli/test_parser.py b/tests/unit/config/cli/test_parser.py index cb656c386..1dc7e055a 100644 --- a/tests/unit/config/cli/test_parser.py +++ b/tests/unit/config/cli/test_parser.py @@ -10,7 +10,7 @@ from virtualenv.run import session_via_cli -@pytest.fixture() +@pytest.fixture def gen_parser_no_conf_env(monkeypatch, tmp_path): keys_to_delete = {key for key in os.environ if key.startswith("VIRTUALENV_")} for key in keys_to_delete: diff --git a/tests/unit/config/test___main__.py b/tests/unit/config/test___main__.py index a4d0b7d34..0bd2569d6 100644 --- a/tests/unit/config/test___main__.py +++ b/tests/unit/config/test___main__.py @@ -26,10 +26,10 @@ def test_main(): assert out -@pytest.fixture() +@pytest.fixture def raise_on_session_done(mocker): def _func(exception): - from virtualenv.run import session_via_cli + from virtualenv.run import session_via_cli # noqa: PLC0415 prev_session = session_via_cli @@ -52,6 +52,16 @@ def test_fail_no_traceback(raise_on_session_done, tmp_path, capsys): assert err == "err\n" +def test_discovery_fails_no_discovery_plugin(mocker, tmp_path, capsys): + mocker.patch("virtualenv.run.plugin.discovery.Discovery.entry_points_for", return_value={}) + with pytest.raises(SystemExit) as context: + run_with_catch([str(tmp_path)]) + assert context.value.code == 1 + out, err = capsys.readouterr() + assert "RuntimeError: No discovery plugin found. Try reinstalling virtualenv to fix this issue." in out + assert not err + + def test_fail_with_traceback(raise_on_session_done, tmp_path, capsys): raise_on_session_done(TypeError("something bad")) @@ -64,7 +74,7 @@ def test_fail_with_traceback(raise_on_session_done, tmp_path, capsys): @pytest.mark.usefixtures("session_app_data") def test_session_report_full(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: - run_with_catch([str(tmp_path), "--setuptools", "bundle", "--wheel", "bundle"]) + run_with_catch([str(tmp_path), "--setuptools", "bundle"]) out, err = capsys.readouterr() assert not err lines = out.splitlines() @@ -72,7 +82,7 @@ def test_session_report_full(tmp_path: Path, capsys: pytest.CaptureFixture[str]) r"created virtual environment .* in \d+ms", r" creator .*", r" seeder .*", - r" added seed packages: .*pip==.*, setuptools==.*, wheel==.*", + r" added seed packages: .*pip==.*, setuptools==.*", r" activators .*", ] _match_regexes(lines, regexes) diff --git a/tests/unit/config/test_env_var.py b/tests/unit/config/test_env_var.py index 751944bb4..8ec16c607 100644 --- a/tests/unit/config/test_env_var.py +++ b/tests/unit/config/test_env_var.py @@ -5,12 +5,15 @@ import pytest +from virtualenv.cache import FileCache from virtualenv.config.cli.parser import VirtualEnvOptions from virtualenv.config.ini import IniConfig +from virtualenv.create.via_global_ref.builtin.cpython.common import is_macos_brew +from virtualenv.discovery.py_info import PythonInfo from virtualenv.run import session_via_cli -@pytest.fixture() +@pytest.fixture def _empty_conf(tmp_path, monkeypatch): conf = tmp_path / "conf.ini" monkeypatch.setenv(IniConfig.VIRTUALENV_CONFIG_FILE_ENV_VAR, str(conf)) @@ -74,9 +77,12 @@ def test_extra_search_dir_via_env_var(tmp_path, monkeypatch): @pytest.mark.usefixtures("_empty_conf") -def test_value_alias(monkeypatch, mocker): - from virtualenv.config.cli.parser import VirtualEnvConfigParser +def test_value_alias(monkeypatch, mocker, session_app_data): + from virtualenv.config.cli.parser import VirtualEnvConfigParser # noqa: PLC0415 + cache = FileCache(session_app_data.py_info, session_app_data.py_info_clear) + if is_macos_brew(PythonInfo.current_system(session_app_data, cache)): + pytest.skip(reason="no copy on brew") prev = VirtualEnvConfigParser._fix_default # noqa: SLF001 def func(self, action): diff --git a/tests/unit/config/test_ini.py b/tests/unit/config/test_ini.py index d3608a6f5..a1621fa30 100644 --- a/tests/unit/config/test_ini.py +++ b/tests/unit/config/test_ini.py @@ -10,7 +10,11 @@ @pytest.mark.skipif(not fs_supports_symlink(), reason="symlink is not supported") -@pytest.mark.xfail(IS_PYPY and IS_WIN and sys.version_info[0:2] == (3, 9), reason="symlink is not supported") +@pytest.mark.xfail( + # https://doc.pypy.org/en/latest/install.html?highlight=symlink#download-a-pre-built-pypy + IS_PYPY and IS_WIN and sys.version_info[0:2] >= (3, 9), + reason="symlink is not supported", +) def test_ini_can_be_overwritten_by_flag(tmp_path, monkeypatch): custom_ini = tmp_path / "conf.ini" custom_ini.write_text( diff --git a/tests/unit/create/conftest.py b/tests/unit/create/conftest.py index 58d390c5c..4c7bf45c0 100644 --- a/tests/unit/create/conftest.py +++ b/tests/unit/create/conftest.py @@ -14,25 +14,31 @@ import pytest +from virtualenv.cache import FileCache from virtualenv.discovery.py_info import PythonInfo -CURRENT = PythonInfo.current_system() +@pytest.fixture(scope="session") +def current_info(session_app_data): + cache = FileCache(session_app_data.py_info, session_app_data.py_info_clear) + return PythonInfo.current_system(session_app_data, cache) -def root(tmp_path_factory, session_app_data): # noqa: ARG001 - return CURRENT.system_executable +def root(tmp_path_factory, session_app_data, current_info): # noqa: ARG001 + return current_info.system_executable -def venv(tmp_path_factory, session_app_data): - if CURRENT.is_venv: + +def venv(tmp_path_factory, session_app_data, current_info): + if current_info.is_venv: return sys.executable - root_python = root(tmp_path_factory, session_app_data) + root_python = root(tmp_path_factory, session_app_data, current_info) dest = tmp_path_factory.mktemp("venv") process = Popen([str(root_python), "-m", "venv", "--without-pip", str(dest)]) process.communicate() # sadly creating a virtual environment does not tell us where the executable lives in general case # so discover using some heuristic - return CURRENT.discover_exe(prefix=str(dest)).original_executable + cache = FileCache(session_app_data.py_info, session_app_data.py_info_clear) + return current_info.discover_exe(session_app_data, cache, prefix=str(dest)).original_executable PYTHON = { @@ -42,8 +48,8 @@ def venv(tmp_path_factory, session_app_data): @pytest.fixture(params=list(PYTHON.values()), ids=list(PYTHON.keys()), scope="session") -def python(request, tmp_path_factory, session_app_data): - result = request.param(tmp_path_factory, session_app_data) +def python(request, tmp_path_factory, session_app_data, current_info): + result = request.param(tmp_path_factory, session_app_data, current_info) if isinstance(result, Exception): pytest.skip(f"could not resolve interpreter based on {request.param.__name__} because {result}") if result is None: diff --git a/tests/unit/create/test_creator.py b/tests/unit/create/test_creator.py index b1f2921ce..0e80514b1 100644 --- a/tests/unit/create/test_creator.py +++ b/tests/unit/create/test_creator.py @@ -11,9 +11,9 @@ import stat import subprocess import sys +import textwrap import zipfile from collections import OrderedDict -from itertools import product from pathlib import Path from stat import S_IREAD, S_IRGRP, S_IROTH from textwrap import dedent @@ -22,14 +22,24 @@ import pytest from virtualenv.__main__ import run, run_with_catch +from virtualenv.cache import FileCache from virtualenv.create.creator import DEBUG_SCRIPT, Creator, get_env_debug_info from virtualenv.create.pyenv_cfg import PyEnvCfg +from virtualenv.create.via_global_ref import api +from virtualenv.create.via_global_ref.builtin.cpython.common import is_macos_brew from virtualenv.create.via_global_ref.builtin.cpython.cpython3 import CPython3Posix from virtualenv.discovery.py_info import PythonInfo from virtualenv.info import IS_PYPY, IS_WIN, fs_is_case_sensitive from virtualenv.run import cli_run, session_via_cli +from virtualenv.run.plugin.creators import CreatorSelector -CURRENT = PythonInfo.current_system() +logger = logging.getLogger(__name__) + + +@pytest.fixture(scope="session") +def current_info(session_app_data): + cache = FileCache(session_app_data.py_info, session_app_data.py_info_clear) + return PythonInfo.current_system(session_app_data, cache) def test_os_path_sep_not_allowed(tmp_path, capsys): @@ -76,145 +86,216 @@ def test_destination_not_write_able(tmp_path, capsys): def cleanup_sys_path(paths): - from virtualenv.create.creator import HERE + from virtualenv.create.creator import HERE # noqa: PLC0415 paths = [p.resolve() for p in (Path(os.path.abspath(i)) for i in paths) if p.exists()] to_remove = [Path(HERE)] if os.environ.get("PYCHARM_HELPERS_DIR"): - to_remove.append(Path(os.environ["PYCHARM_HELPERS_DIR"]).parent) - to_remove.append(Path(os.path.expanduser("~")) / ".PyCharm") + to_remove.extend((Path(os.environ["PYCHARM_HELPERS_DIR"]).parent, Path(os.path.expanduser("~")) / ".PyCharm")) return [i for i in paths if not any(str(i).startswith(str(t)) for t in to_remove)] @pytest.fixture(scope="session") -def system(session_app_data): - return get_env_debug_info(Path(CURRENT.system_executable), DEBUG_SCRIPT, session_app_data, os.environ) +def system(session_app_data, current_info): + return get_env_debug_info(Path(current_info.system_executable), DEBUG_SCRIPT, session_app_data, os.environ) -CURRENT_CREATORS = [i for i in CURRENT.creators().key_to_class if i != "builtin"] -CREATE_METHODS = [] -for k, v in CURRENT.creators().key_to_meta.items(): - if k in CURRENT_CREATORS: - if v.can_copy: - CREATE_METHODS.append((k, "copies")) - if v.can_symlink: - CREATE_METHODS.append((k, "symlinks")) +@pytest.fixture(scope="session") +def current_creator_keys(current_info): + return [i for i in CreatorSelector.for_interpreter(current_info).key_to_class if i != "builtin"] -@pytest.mark.parametrize( - ("creator", "isolated"), - [pytest.param(*i, id=f"{'-'.join(i[0])}-{i[1]}") for i in product(CREATE_METHODS, ["isolated", "global"])], -) +@pytest.fixture(scope="session") +def create_methods(current_creator_keys, current_info): + methods = [] + for k, v in CreatorSelector.for_interpreter(current_info).key_to_meta.items(): + if k in current_creator_keys: + if v.can_copy: + if ( + k == "venv" + and current_info.implementation == "PyPy" + and current_info.pypy_version_info >= [7, 3, 13] + ): # https://github.com/pypy/pypy/issues/4019 + continue + methods.append((k, "copies")) + if v.can_symlink: + methods.append((k, "symlinks")) + return methods + + +@pytest.fixture +def python_case(request, current_info): + """Resolve the python under test based on a param value.""" + case = request.param + if case == "venv": + # keep the original skip condition + if sys.executable == current_info.system_executable: + pytest.skip("system") + return sys.executable, "venv" + if case == "root": + return current_info.system_executable, "root" + msg = f"unknown python_case: {case}" + raise RuntimeError(msg) + + +@pytest.mark.parametrize("isolated", ["isolated", "global"]) +@pytest.mark.parametrize("python_case", ["venv", "root"], indirect=True) def test_create_no_seed( # noqa: C901, PLR0912, PLR0913, PLR0915 - python, - creator, - isolated, system, coverage_env, special_name_dir, + create_methods, + current_info, + session_app_data, + isolated, + python_case, ): - dest = special_name_dir - creator_key, method = creator - cmd = [ - "-v", - "-v", - "-p", - str(python), - str(dest), - "--without-pip", - "--activators", - "", - "--creator", - creator_key, - f"--{method}", - ] - if isolated == "global": - cmd.append("--system-site-packages") - result = cli_run(cmd) - creator = result.creator - coverage_env() - if IS_PYPY: - # pypy cleans up file descriptors periodically so our (many) subprocess calls impact file descriptor limits - # force a close of these on system where the limit is low-ish (e.g. MacOS 256) - gc.collect() - purelib = creator.purelib - patch_files = {purelib / f"{'_virtualenv'}.{i}" for i in ("py", "pyc", "pth")} - patch_files.add(purelib / "__pycache__") - content = set(creator.purelib.iterdir()) - patch_files - assert not content, "\n".join(str(i) for i in content) - assert creator.env_name == str(dest.name) - debug = creator.debug - assert "exception" not in debug, f"{debug.get('exception')}\n{debug.get('out')}\n{debug.get('err')}" - sys_path = cleanup_sys_path(debug["sys"]["path"]) - system_sys_path = cleanup_sys_path(system["sys"]["path"]) - our_paths = set(sys_path) - set(system_sys_path) - our_paths_repr = "\n".join(repr(i) for i in our_paths) - - # ensure we have at least one extra path added - assert len(our_paths) >= 1, our_paths_repr - # ensure all additional paths are related to the virtual environment - for path in our_paths: - msg = "\n".join(str(p) for p in system_sys_path) - msg = f"\n{path!s}\ndoes not start with {dest!s}\nhas:\n{msg}" - assert str(path).startswith(str(dest)), msg - # ensure there's at least a site-packages folder as part of the virtual environment added - assert any(p for p in our_paths if p.parts[-1] == "site-packages"), our_paths_repr - - # ensure the global site package is added or not, depending on flag - global_sys_path = system_sys_path[-1] - if isolated == "isolated": - msg = "\n".join(str(j) for j in sys_path) - msg = f"global sys path {global_sys_path!s} is in virtual environment sys path:\n{msg}" - assert global_sys_path not in sys_path, msg - else: - common = [] - for left, right in zip(reversed(system_sys_path), reversed(sys_path)): - if left == right: - common.append(left) - else: - break + python_exe, python_id = python_case + logger.info("running no seed test for %s-%s", python_id, isolated) + + for creator_key, method in create_methods: + dest = special_name_dir / f"{creator_key}-{method}-{isolated}" + cmd = [ + "-v", + "-v", + "-p", + str(python_exe), + str(dest), + "--without-pip", + "--activators", + "", + "--creator", + creator_key, + f"--{method}", + ] + if isolated == "global": + cmd.append("--system-site-packages") + result = cli_run(cmd) + creator = result.creator + coverage_env() + if IS_PYPY: + # pypy cleans up file descriptors periodically so our (many) subprocess calls impact file descriptor limits + # force a close of these on system where the limit is low-ish (e.g. MacOS 256) + gc.collect() + purelib = creator.purelib + patch_files = {purelib / f"{'_virtualenv'}.{i}" for i in ("py", "pyc", "pth")} + patch_files.add(purelib / "__pycache__") + content = set(creator.purelib.iterdir()) - patch_files + assert not content, "\n".join(str(i) for i in content) + assert creator.env_name == str(dest.name) + debug = creator.debug + assert "exception" not in debug, f"{debug.get('exception')}\n{debug.get('out')}\n{debug.get('err')}" + sys_path = cleanup_sys_path(debug["sys"]["path"]) + system_sys_path = cleanup_sys_path(system["sys"]["path"]) + our_paths = set(sys_path) - set(system_sys_path) + our_paths_repr = "\n".join(repr(i) for i in our_paths) + + # ensure we have at least one extra path added + assert len(our_paths) >= 1, our_paths_repr + # ensure all additional paths are related to the virtual environment + for path in our_paths: + msg = "\n".join(str(p) for p in system_sys_path) + msg = f"\n{path!s}\ndoes not start with {dest!s}\nhas:\n{msg}" + assert str(path).startswith(str(dest)), msg + # ensure there's at least a site-packages folder as part of the virtual environment added + assert any(p for p in our_paths if p.parts[-1] == "site-packages"), our_paths_repr + + # ensure the global site package is added or not, depending on flag + global_sys_path = system_sys_path[-1] + if isolated == "isolated": + msg = "\n".join(str(j) for j in sys_path) + msg = f"global sys path {global_sys_path!s} is in virtual environment sys path:\n{msg}" + assert global_sys_path not in sys_path, msg + else: + common = [] + for left, right in zip(reversed(system_sys_path), reversed(sys_path)): + if left == right: + common.append(left) + else: + break + + def list_to_str(iterable): + return [str(i) for i in iterable] + + assert common, "\n".join(difflib.unified_diff(list_to_str(sys_path), list_to_str(system_sys_path))) + + # test that the python executables in the bin directory are either: + # - files + # - absolute symlinks outside of the venv + # - relative symlinks inside of the venv + if sys.platform == "win32": + exes = ("python.exe",) + else: + exes = ( + "python", + f"python{sys.version_info.major}", + f"python{sys.version_info.major}.{sys.version_info.minor}", + ) + if creator_key == "venv": + # for venv some repackaging does not includes the pythonx.y + exes = exes[:-1] + for exe in exes: + exe_path = creator.bin_dir / exe + assert exe_path.exists(), "\n".join(str(i) for i in creator.bin_dir.iterdir()) + if not exe_path.is_symlink(): # option 1: a real file + continue # it was a file + link = os.readlink(str(exe_path)) + if not os.path.isabs(link): # option 2: a relative symlink + continue + # option 3: an absolute symlink, should point outside the venv + assert not link.startswith(str(creator.dest)) + + if IS_WIN and current_info.implementation == "CPython": + python_w = creator.exe.parent / "pythonw.exe" + assert python_w.exists() + assert python_w.read_bytes() != creator.exe.read_bytes() + + if creator_key != "venv" and CPython3Posix.pyvenv_launch_patch_active( + PythonInfo.from_exe( + python_exe, + session_app_data, + FileCache(session_app_data.py_info, session_app_data.py_info_clear), + ), + ): + result = subprocess.check_output( + [str(creator.exe), "-c", 'import os; print(os.environ.get("__PYVENV_LAUNCHER__"))'], + text=True, + ).strip() + assert result == "None" + + git_ignore = (dest / ".gitignore").read_text(encoding="utf-8") + if creator_key == "venv" and sys.version_info >= (3, 13): + comment = "# Created by venv; see https://docs.python.org/3/library/venv.html" + else: + comment = "# created by virtualenv automatically" + assert git_ignore.splitlines() == [comment, "*"] + + +def test_create_cachedir_tag(tmp_path): + cachedir_tag_file = tmp_path / "CACHEDIR.TAG" + cli_run([str(tmp_path), "--without-pip", "--activators", ""]) - def list_to_str(iterable): - return [str(i) for i in iterable] + expected = """ + Signature: 8a477f597d28d172789f06886806bc55 + # This file is a cache directory tag created by Python virtualenv. + # For information about cache directory tags, see: + # https://bford.info/cachedir/ + """ + assert cachedir_tag_file.read_text(encoding="utf-8") == textwrap.dedent(expected).strip() - assert common, "\n".join(difflib.unified_diff(list_to_str(sys_path), list_to_str(system_sys_path))) - # test that the python executables in the bin directory are either: - # - files - # - absolute symlinks outside of the venv - # - relative symlinks inside of the venv - if sys.platform == "win32": - exes = ("python.exe",) - else: - exes = ("python", f"python{sys.version_info.major}", f"python{sys.version_info.major}.{sys.version_info.minor}") - if creator_key == "venv": - # for venv some repackaging does not includes the pythonx.y - exes = exes[:-1] - for exe in exes: - exe_path = creator.bin_dir / exe - assert exe_path.exists(), "\n".join(str(i) for i in creator.bin_dir.iterdir()) - if not exe_path.is_symlink(): # option 1: a real file - continue # it was a file - link = os.readlink(str(exe_path)) - if not os.path.isabs(link): # option 2: a relative symlink - continue - # option 3: an absolute symlink, should point outside the venv - assert not link.startswith(str(creator.dest)) - - if IS_WIN and CURRENT.implementation == "CPython": - python_w = creator.exe.parent / "pythonw.exe" - assert python_w.exists() - assert python_w.read_bytes() != creator.exe.read_bytes() - - if CPython3Posix.pyvenv_launch_patch_active(PythonInfo.from_exe(python)) and creator_key != "venv": - result = subprocess.check_output( - [str(creator.exe), "-c", 'import os; print(os.environ.get("__PYVENV_LAUNCHER__"))'], - text=True, - ).strip() - assert result == "None" - - git_ignore = (dest / ".gitignore").read_text(encoding="utf-8") - assert git_ignore.splitlines() == ["# created by virtualenv automatically", "*"] +def test_create_cachedir_tag_exists(tmp_path: Path) -> None: + cachedir_tag_file = tmp_path / "CACHEDIR.TAG" + cachedir_tag_file.write_text("magic", encoding="utf-8") + cli_run([str(tmp_path), "--without-pip", "--activators", ""]) + assert cachedir_tag_file.read_text(encoding="utf-8") == "magic" + + +def test_create_cachedir_tag_exists_override(tmp_path: Path) -> None: + cachedir_tag_file = tmp_path / "CACHEDIR.TAG" + cachedir_tag_file.write_text("magic", encoding="utf-8") + cli_run([str(tmp_path), "--without-pip", "--activators", ""]) + assert cachedir_tag_file.read_text(encoding="utf-8") == "magic" def test_create_vcs_ignore_exists(tmp_path): @@ -237,8 +318,9 @@ def test_create_vcs_ignore_exists_override(tmp_path): assert git_ignore.read_text(encoding="utf-8") == "magic" -@pytest.mark.skipif(not CURRENT.has_venv, reason="requires interpreter with venv") -def test_venv_fails_not_inline(tmp_path, capsys, mocker): +def test_venv_fails_not_inline(tmp_path, capsys, mocker, current_info): + if not current_info.has_venv: + pytest.skip("requires interpreter with venv") if hasattr(os, "geteuid") and os.geteuid() == 0: pytest.skip("no way to check permission restriction when running under root") @@ -254,7 +336,7 @@ def _session_via_cli(args, options=None, setup_logging=True, env=None): cfg = str(cfg_path) try: os.chmod(cfg, stat.S_IREAD | stat.S_IRGRP | stat.S_IROTH) - cmd = ["-p", str(CURRENT.executable), str(tmp_path), "--without-pip", "--creator", "venv"] + cmd = ["-p", str(current_info.executable), str(tmp_path), "--without-pip", "--creator", "venv"] with pytest.raises(SystemExit) as context: run(cmd) assert context.value.code != 0 @@ -265,46 +347,45 @@ def _session_via_cli(args, options=None, setup_logging=True, env=None): assert "Error:" in err, err -@pytest.mark.parametrize("creator", CURRENT_CREATORS) @pytest.mark.parametrize("clear", [True, False], ids=["clear", "no_clear"]) -def test_create_clear_resets(tmp_path, creator, clear, caplog): +def test_create_clear_resets(tmp_path, clear, caplog, current_creator_keys): caplog.set_level(logging.DEBUG) - if creator == "venv" and clear is False: - pytest.skip("venv without clear might fail") - marker = tmp_path / "magic" - cmd = [str(tmp_path), "--seeder", "app-data", "--without-pip", "--creator", creator, "-vvv"] - cli_run(cmd) + for creator in current_creator_keys: + if creator == "venv" and clear is False: + pytest.skip("venv without clear might fail") + marker = tmp_path / creator / "magic" + cmd = [str(tmp_path / creator), "--seeder", "app-data", "--without-pip", "--creator", creator, "-vvv"] + cli_run(cmd) - marker.write_text("", encoding="utf-8") # if we a marker file this should be gone on a clear run, remain otherwise - assert marker.exists() + marker.write_text("", encoding="utf-8") + assert marker.exists() - cli_run(cmd + (["--clear"] if clear else [])) - assert marker.exists() is not clear + cli_run(cmd + (["--clear"] if clear else [])) + assert marker.exists() is not clear -@pytest.mark.parametrize("creator", CURRENT_CREATORS) @pytest.mark.parametrize("prompt", [None, "magic"]) -def test_prompt_set(tmp_path, creator, prompt): - cmd = [str(tmp_path), "--seeder", "app-data", "--without-pip", "--creator", creator] - if prompt is not None: - cmd.extend(["--prompt", "magic"]) - - result = cli_run(cmd) - actual_prompt = tmp_path.name if prompt is None else prompt - cfg = PyEnvCfg.from_file(result.creator.pyenv_cfg.path) - if prompt is None: - assert "prompt" not in cfg - elif creator != "venv": - assert "prompt" in cfg, list(cfg.content.keys()) - assert cfg["prompt"] == actual_prompt - - -@pytest.mark.parametrize("creator", CURRENT_CREATORS) -def test_home_path_is_exe_parent(tmp_path, creator): - cmd = [str(tmp_path), "--seeder", "app-data", "--without-pip", "--creator", creator] - - result = cli_run(cmd) - cfg = PyEnvCfg.from_file(result.creator.pyenv_cfg.path) +def test_prompt_set(tmp_path, prompt, current_creator_keys): + for creator in current_creator_keys: + cmd = [str(tmp_path / creator), "--seeder", "app-data", "--without-pip", "--creator", creator] + if prompt is not None: + cmd.extend(["--prompt", "magic"]) + + result = cli_run(cmd) + actual_prompt = tmp_path.name if prompt is None else prompt + cfg = PyEnvCfg.from_file(result.creator.pyenv_cfg.path) + if prompt is None: + assert "prompt" not in cfg + elif creator != "venv": + assert "prompt" in cfg, list(cfg.content.keys()) + assert cfg["prompt"] == actual_prompt + + +def test_home_path_is_exe_parent(tmp_path, current_creator_keys): + for creator in current_creator_keys: + cmd = [str(tmp_path / creator), "--seeder", "app-data", "--without-pip", "--creator", creator] + result = cli_run(cmd) + cfg = PyEnvCfg.from_file(result.creator.pyenv_cfg.path) # Cannot assume "home" path is a specific value as path resolution may change # between versions (symlinks, framework paths, etc) but we can check that a @@ -364,24 +445,21 @@ def test_create_long_path(tmp_path): subprocess.check_call([str(result.creator.script("pip")), "--version"]) -@pytest.mark.parametrize("creator", sorted(set(PythonInfo.current_system().creators().key_to_class) - {"builtin"})) -@pytest.mark.usefixtures("session_app_data") -def test_create_distutils_cfg(creator, tmp_path, monkeypatch): - result = cli_run( - [ - str(tmp_path / "venv"), - "--activators", - "", - "--creator", - creator, - "--setuptools", - "bundle", - "--wheel", - "bundle", - ], - ) - - app = Path(__file__).parent / "console_app" +@pytest.mark.slow +def test_create_distutils_cfg(tmp_path, monkeypatch, current_creator_keys): + for creator in current_creator_keys: + result = cli_run( + [ + str(tmp_path / creator / "venv"), + "--activators", + "", + "--creator", + creator, + "--setuptools", + "bundle", + ], + ) + app = Path(__file__).parent / "console_app" dest = tmp_path / "console_app" shutil.copytree(str(app), str(dest)) @@ -430,7 +508,10 @@ def list_files(path): return result -def test_zip_importer_can_import_setuptools(tmp_path): +@pytest.mark.skip(reason="https://github.com/pypa/setuptools/issues/4640") +def test_zip_importer_can_import_setuptools(tmp_path, current_info): + if is_macos_brew(current_info): + pytest.skip("no copy on brew") """We're patching the loaders so might fail on r/o loaders, such as zipimporter on CPython<3.8""" result = cli_run( [str(tmp_path / "venv"), "--activators", "", "--no-pip", "--no-wheel", "--copies", "--setuptools", "bundle"], @@ -477,10 +558,9 @@ def test_no_preimport_threading(tmp_path): # verify that .pth files in site-packages/ are always processed even if $PYTHONPATH points to it. def test_pth_in_site_vs_python_path(tmp_path): session = cli_run([str(tmp_path)]) - site_packages = str(session.creator.purelib) + site_packages = session.creator.purelib # install test.pth that sets sys.testpth='ok' - with open(os.path.join(site_packages, "test.pth"), "w", encoding="utf-8") as f: - f.write('import sys; sys.testpth="ok"\n') + (session.creator.purelib / "test.pth").write_text('import sys; sys.testpth="ok"\n', encoding="utf-8") # verify that test.pth is activated when interpreter is run out = subprocess.check_output( [str(session.creator.exe), "-c", r"import sys; print(sys.testpth)"], @@ -490,7 +570,7 @@ def test_pth_in_site_vs_python_path(tmp_path): assert out == "ok\n" # same with $PYTHONPATH pointing to site_packages env = os.environ.copy() - path = [site_packages] + path = [str(site_packages)] if "PYTHONPATH" in env: path.append(env["PYTHONPATH"]) env["PYTHONPATH"] = os.pathsep.join(path) @@ -640,9 +720,10 @@ def _get_sys_path(flag=None): # (specifically venv scripts delivered with Python itself) are not writable. # # https://github.com/pypa/virtualenv/issues/2419 -@pytest.mark.skipif("venv" not in CURRENT_CREATORS, reason="test needs venv creator") -def test_venv_creator_without_write_perms(tmp_path, mocker): - from virtualenv.run.session import Session +def test_venv_creator_without_write_perms(tmp_path, mocker, current_creator_keys): + if "venv" not in current_creator_keys: + pytest.skip("test needs venv creator") + from virtualenv.run.session import Session # noqa: PLC0415 prev = Session._create # noqa: SLF001 @@ -656,3 +737,65 @@ def func(self): cmd = [str(tmp_path), "--seeder", "app-data", "--without-pip", "--creator", "venv"] cli_run(cmd) + + +def test_fallback_to_copies_if_symlink_unsupported(tmp_path, python, mocker, session_app_data): + """Test that creating a virtual environment falls back to copies when filesystem has no symlink support.""" + cache = FileCache(session_app_data.py_info, session_app_data.py_info_clear) + if is_macos_brew(PythonInfo.from_exe(python, session_app_data, cache)): + pytest.skip("brew python on darwin may not support copies, which is tested separately") + + # Given a filesystem that does not support symlinks + mocker.patch("virtualenv.create.via_global_ref.api.fs_supports_symlink", return_value=False) + + # When creating a virtual environment (no method specified) + cmd = [ + "-v", + "-p", + str(python), + str(tmp_path), + "--without-pip", + "--activators", + "", + ] + result = cli_run(cmd) + + # Then the creation should succeed and the creator should report it used copies + assert result.creator is not None + assert result.creator.symlinks is False + + +def test_fail_gracefully_if_no_method_supported(tmp_path, python, mocker, session_app_data): + """Test that virtualenv fails gracefully when no creation method is supported.""" + # Given a filesystem that does not support symlinks + mocker.patch("virtualenv.create.via_global_ref.api.fs_supports_symlink", return_value=False) + + cache = FileCache(session_app_data.py_info, session_app_data.py_info_clear) + # And a creator that does not support copying + if not is_macos_brew(PythonInfo.from_exe(python, session_app_data, cache)): + original_init = api.ViaGlobalRefMeta.__init__ + + def new_init(self, *args, **kwargs): + original_init(self, *args, **kwargs) + self.copy_error = "copying is not supported" + + mocker.patch("virtualenv.create.via_global_ref.api.ViaGlobalRefMeta.__init__", new=new_init) + + # When creating a virtual environment + with pytest.raises(RuntimeError) as excinfo: + cli_run( + [ + "-p", + str(python), + str(tmp_path), + "--without-pip", + ], + ) + + # Then a RuntimeError should be raised with a detailed message + assert "neither symlink or copy method supported" in str(excinfo.value) + assert "symlink: the filesystem does not supports symlink" in str(excinfo.value) + if is_macos_brew(PythonInfo.from_exe(python, session_app_data, cache)): + assert "copy: Brew disables copy creation" in str(excinfo.value) + else: + assert "copy: copying is not supported" in str(excinfo.value) diff --git a/tests/unit/create/test_interpreters.py b/tests/unit/create/test_interpreters.py index b6b58df4d..b842891f1 100644 --- a/tests/unit/create/test_interpreters.py +++ b/tests/unit/create/test_interpreters.py @@ -5,11 +5,12 @@ import pytest +from virtualenv.cache import FileCache from virtualenv.discovery.py_info import PythonInfo from virtualenv.run import cli_run -@pytest.mark.slow() +@pytest.mark.slow def test_failed_to_find_bad_spec(): of_id = uuid4().hex with pytest.raises(RuntimeError) as context: @@ -18,15 +19,14 @@ def test_failed_to_find_bad_spec(): assert repr(context.value) == msg -SYSTEM = PythonInfo.current_system() - - -@pytest.mark.parametrize( - "of_id", - ({sys.executable} if sys.executable != SYSTEM.executable else set()) | {SYSTEM.implementation}, -) -def test_failed_to_find_implementation(of_id, mocker): - mocker.patch("virtualenv.run.plugin.creators.CreatorSelector._OPTIONS", return_value={}) - with pytest.raises(RuntimeError) as context: - cli_run(["-p", of_id]) - assert repr(context.value) == repr(RuntimeError(f"No virtualenv implementation for {PythonInfo.current_system()}")) +def test_failed_to_find_implementation(mocker, session_app_data): + cache = FileCache(session_app_data.py_info, session_app_data.py_info_clear) + system = PythonInfo.current_system(session_app_data, cache) + of_ids = ({sys.executable} if sys.executable != system.executable else set()) | {system.implementation} + for of_id in of_ids: + mocker.patch("virtualenv.run.plugin.creators.CreatorSelector._OPTIONS", return_value={}) + with pytest.raises(RuntimeError) as context: + cli_run(["-p", of_id]) + assert repr(context.value) == repr( + RuntimeError(f"No virtualenv implementation for {PythonInfo.current_system(session_app_data, cache)}"), + ) diff --git a/tests/unit/create/via_global_ref/builtin/conftest.py b/tests/unit/create/via_global_ref/builtin/conftest.py index bb505db28..a5808c58b 100644 --- a/tests/unit/create/via_global_ref/builtin/conftest.py +++ b/tests/unit/create/via_global_ref/builtin/conftest.py @@ -11,16 +11,16 @@ sys.path.append(str(Path(__file__).parent)) -@pytest.fixture() +@pytest.fixture def py_info(py_info_name): return read_fixture(py_info_name) -@pytest.fixture() +@pytest.fixture def mock_files(mocker): return lambda paths, files: path.mock_files(mocker, paths, files) -@pytest.fixture() +@pytest.fixture def mock_pypy_libs(mocker): return lambda pypy, libs: path.mock_pypy_libs(mocker, pypy, libs) diff --git a/tests/unit/create/via_global_ref/builtin/cpython/cpython3_win_embed.json b/tests/unit/create/via_global_ref/builtin/cpython/cpython3_win_embed.json index e8d0d01c9..c75c6f4fc 100644 --- a/tests/unit/create/via_global_ref/builtin/cpython/cpython3_win_embed.json +++ b/tests/unit/create/via_global_ref/builtin/cpython/cpython3_win_embed.json @@ -57,5 +57,6 @@ "system_stdlib": "c:\\path\\to\\python\\Lib", "system_stdlib_platform": "c:\\path\\to\\python\\Lib", "max_size": 9223372036854775807, - "_creators": null + "_creators": null, + "free_threaded": false } diff --git a/tests/unit/create/via_global_ref/builtin/cpython/test_cpython3_win.py b/tests/unit/create/via_global_ref/builtin/cpython/test_cpython3_win.py index f831de114..4367ebc50 100644 --- a/tests/unit/create/via_global_ref/builtin/cpython/test_cpython3_win.py +++ b/tests/unit/create/via_global_ref/builtin/cpython/test_cpython3_win.py @@ -99,3 +99,11 @@ def test_no_python_zip_if_not_exists(py_info, mock_files): sources = tuple(CPython3Windows.sources(interpreter=py_info)) assert python_zip in py_info.path assert not contains_ref(sources, python_zip) + + +@pytest.mark.parametrize("py_info_name", ["cpython3_win_embed"]) +def test_python3_exe_present(py_info, mock_files): + mock_files(CPYTHON3_PATH, [py_info.system_executable]) + sources = tuple(CPython3Windows.sources(interpreter=py_info)) + assert contains_exe(sources, py_info.system_executable, "python3.exe") + assert contains_exe(sources, py_info.system_executable, "python3") diff --git a/tests/unit/create/via_global_ref/builtin/pypy/deb_pypy37.json b/tests/unit/create/via_global_ref/builtin/pypy/deb_pypy37.json index eb694a840..4e6ca4dda 100644 --- a/tests/unit/create/via_global_ref/builtin/pypy/deb_pypy37.json +++ b/tests/unit/create/via_global_ref/builtin/pypy/deb_pypy37.json @@ -60,5 +60,6 @@ "system_stdlib": "/usr/lib/pypy3/lib-python/3.7", "system_stdlib_platform": "/usr/lib/pypy3/lib-python/3.7", "max_size": 9223372036854775807, - "_creators": null + "_creators": null, + "free_threaded": false } diff --git a/tests/unit/create/via_global_ref/builtin/pypy/deb_pypy38.json b/tests/unit/create/via_global_ref/builtin/pypy/deb_pypy38.json index 478d79977..070867210 100644 --- a/tests/unit/create/via_global_ref/builtin/pypy/deb_pypy38.json +++ b/tests/unit/create/via_global_ref/builtin/pypy/deb_pypy38.json @@ -21,7 +21,11 @@ "original_executable": "/usr/bin/pypy3", "system_executable": "/usr/bin/pypy3", "has_venv": true, - "path": ["/usr/lib/pypy3.8", "/usr/local/lib/pypy3.8/dist-packages", "/usr/lib/python3/dist-packages"], + "path": [ + "/usr/lib/pypy3.8", + "/usr/local/lib/pypy3.8/dist-packages", + "/usr/lib/python3/dist-packages" + ], "file_system_encoding": "utf-8", "stdout_encoding": "UTF-8", "sysconfig_scheme": null, @@ -56,5 +60,6 @@ "system_stdlib": "/usr/lib/pypy3.8", "system_stdlib_platform": "/usr/lib/pypy3.8", "max_size": 9223372036854775807, - "_creators": null + "_creators": null, + "free_threaded": false } diff --git a/tests/unit/create/via_global_ref/builtin/pypy/portable_pypy38.json b/tests/unit/create/via_global_ref/builtin/pypy/portable_pypy38.json index 2264fa432..136c1b4f6 100644 --- a/tests/unit/create/via_global_ref/builtin/pypy/portable_pypy38.json +++ b/tests/unit/create/via_global_ref/builtin/pypy/portable_pypy38.json @@ -21,7 +21,10 @@ "original_executable": "/tmp/pypy3.8-v7.3.8-linux64/bin/pypy", "system_executable": "/tmp/pypy3.8-v7.3.8-linux64/bin/pypy", "has_venv": true, - "path": ["/tmp/pypy3.8-v7.3.8-linux64/lib/pypy3.8", "/tmp/pypy3.8-v7.3.8-linux64/lib/pypy3.8/site-packages"], + "path": [ + "/tmp/pypy3.8-v7.3.8-linux64/lib/pypy3.8", + "/tmp/pypy3.8-v7.3.8-linux64/lib/pypy3.8/site-packages" + ], "file_system_encoding": "utf-8", "stdout_encoding": "UTF-8", "sysconfig_scheme": null, @@ -56,5 +59,6 @@ "system_stdlib": "/tmp/pypy3.8-v7.3.8-linux64/lib/pypy3.8", "system_stdlib_platform": "/tmp/pypy3.8-v7.3.8-linux64/lib/pypy3.8", "max_size": 9223372036854775807, - "_creators": null + "_creators": null, + "free_threaded": false } diff --git a/tests/unit/create/via_global_ref/builtin/testing/path.py b/tests/unit/create/via_global_ref/builtin/testing/path.py index 3b64c6388..06ba921c5 100644 --- a/tests/unit/create/via_global_ref/builtin/testing/path.py +++ b/tests/unit/create/via_global_ref/builtin/testing/path.py @@ -1,6 +1,6 @@ from __future__ import annotations -from abc import ABCMeta, abstractmethod +from abc import ABC, abstractmethod from itertools import chain from operator import attrgetter as attr from pathlib import Path @@ -10,7 +10,7 @@ def is_name(path): return str(path) == path.name -class FakeDataABC(metaclass=ABCMeta): +class FakeDataABC(ABC): """Provides data to mock the `Path`""" @property @@ -47,11 +47,15 @@ class PathMockABC(FakeDataABC, Path): """Mocks the behavior of `Path`""" _flavour = getattr(Path(), "_flavour", None) - if hasattr(_flavour, "altsep"): # Allows to pass some tests for Windows via PosixPath. _flavour.altsep = _flavour.altsep or "\\" + # Python 3.13 renamed _flavour to parser + parser = getattr(Path(), "parser", None) + if hasattr(parser, "altsep"): + parser.altsep = parser.altsep or "\\" + def exists(self): return self.is_file() or self.is_dir() diff --git a/tests/unit/create/via_global_ref/test_build_c_ext.py b/tests/unit/create/via_global_ref/test_build_c_ext.py index db1c16eb0..b00a3eb47 100644 --- a/tests/unit/create/via_global_ref/test_build_c_ext.py +++ b/tests/unit/create/via_global_ref/test_build_c_ext.py @@ -1,5 +1,6 @@ from __future__ import annotations +import logging import os import shutil import subprocess @@ -8,58 +9,77 @@ import pytest +from virtualenv.cache import FileCache from virtualenv.discovery.py_info import PythonInfo from virtualenv.run import cli_run +from virtualenv.run.plugin.creators import CreatorSelector -CURRENT = PythonInfo.current_system() -CREATOR_CLASSES = CURRENT.creators().key_to_class - - -def builtin_shows_marker_missing(): - builtin_classs = CREATOR_CLASSES.get("builtin") - if builtin_classs is None: - return False - host_include_marker = getattr(builtin_classs, "host_include_marker", None) - if host_include_marker is None: - return False - marker = host_include_marker(CURRENT) - return not marker.exists() +logger = logging.getLogger(__name__) +@pytest.mark.slow @pytest.mark.xfail( condition=bool(os.environ.get("CI_RUN")), strict=False, reason="did not manage to setup CI to run with VC 14.1 C++ compiler, but passes locally", ) -@pytest.mark.skipif( - not Path(CURRENT.system_include).exists() and not builtin_shows_marker_missing(), - reason="Building C-Extensions requires header files with host python", -) -@pytest.mark.parametrize("creator", [i for i in CREATOR_CLASSES if i != "builtin"]) -def test_can_build_c_extensions(creator, tmp_path, coverage_env): - env, greet = tmp_path / "env", str(tmp_path / "greet") - shutil.copytree(str(Path(__file__).parent.resolve() / "greet"), greet) - session = cli_run(["--creator", creator, "--seeder", "app-data", str(env), "-vvv"]) - coverage_env() - cmd = [ - str(session.creator.script("pip")), - "install", - "--no-index", - "--no-deps", - "--disable-pip-version-check", - "-vvv", - greet, - ] - process = Popen(cmd) - process.communicate() - assert process.returncode == 0 +def test_can_build_c_extensions(tmp_path, coverage_env, session_app_data): + cache = FileCache(session_app_data.py_info, session_app_data.py_info_clear) + current = PythonInfo.current_system(session_app_data, cache) + creator_classes = CreatorSelector.for_interpreter(current).key_to_class + + logger.warning("system_include: %s", current.system_include) + logger.warning("system_include exists: %s", Path(current.system_include).exists()) + + def builtin_shows_marker_missing(): + builtin_classs = creator_classes.get("builtin") + if builtin_classs is None: + return False + host_include_marker = getattr(builtin_classs, "host_include_marker", None) + if host_include_marker is None: + return False + marker = host_include_marker(current) + logger.warning("builtin marker: %s", marker) + logger.warning("builtin marker exists: %s", marker.exists()) + return not marker.exists() + + system_include = current.system_include + if not Path(system_include).exists() and not builtin_shows_marker_missing(): + pytest.skip("Building C-Extensions requires header files with host python") + + for creator in [i for i in creator_classes if i != "builtin"]: + env, greet = tmp_path / creator / "env", str(tmp_path / creator / "greet") + shutil.copytree(str(Path(__file__).parent.resolve() / "greet"), greet) + session = cli_run(["--creator", creator, "--seeder", "app-data", str(env), "-vvv"]) + coverage_env() + setuptools_index_args = () + if current.version_info >= (3, 12): + # requires to be able to install setuptools as build dependency + setuptools_index_args = ( + "--find-links", + "https://pypi.org/simple/setuptools/", + ) + + cmd = [ + str(session.creator.script("pip")), + "install", + "--no-index", + *setuptools_index_args, + "--no-deps", + "--disable-pip-version-check", + "-vvv", + greet, + ] + process = Popen(cmd) + process.communicate() + assert process.returncode == 0 - process = Popen( - [str(session.creator.exe), "-c", "import greet; greet.greet('World')"], - universal_newlines=True, - stdout=subprocess.PIPE, - encoding="utf-8", - ) - out, _ = process.communicate() - assert process.returncode == 0 - assert out == "Hello World!\n" + process = Popen( + [str(session.creator.exe), "-c", "import greet; greet.greet('World')"], + universal_newlines=True, + stdout=subprocess.PIPE, + encoding="utf-8", + ) + out, _ = process.communicate() + assert process.returncode == 0 + assert out == "Hello World!\n" diff --git a/tests/unit/create/virtualenv-16.7.9-py2.py3-none-any.whl b/tests/unit/create/virtualenv-16.7.9-py2.py3-none-any.whl deleted file mode 100644 index 46331cd04..000000000 Binary files a/tests/unit/create/virtualenv-16.7.9-py2.py3-none-any.whl and /dev/null differ diff --git a/tests/unit/discovery/py_info/test_py_info.py b/tests/unit/discovery/py_info/test_py_info.py index 11e63285b..fed641a98 100644 --- a/tests/unit/discovery/py_info/test_py_info.py +++ b/tests/unit/discovery/py_info/test_py_info.py @@ -14,34 +14,40 @@ import pytest +from tests.unit.discovery.util import MockAppData, MockCache +from virtualenv.create.via_global_ref.builtin.cpython.common import is_macos_brew from virtualenv.discovery import cached_py_info from virtualenv.discovery.py_info import PythonInfo, VersionInfo from virtualenv.discovery.py_spec import PythonSpec from virtualenv.info import IS_PYPY, IS_WIN, fs_supports_symlink -CURRENT = PythonInfo.current_system() +CURRENT = PythonInfo.current_system(MockAppData(), MockCache()) def test_current_as_json(): result = CURRENT._to_json() # noqa: SLF001 parsed = json.loads(result) a, b, c, d, e = sys.version_info + f = sysconfig.get_config_var("Py_GIL_DISABLED") == 1 assert parsed["version_info"] == {"major": a, "minor": b, "micro": c, "releaselevel": d, "serial": e} + assert parsed["free_threaded"] is f -def test_bad_exe_py_info_raise(tmp_path, session_app_data): +def test_bad_exe_py_info_raise(tmp_path): exe = str(tmp_path) + app_data, cache = MockAppData(), MockCache() with pytest.raises(RuntimeError) as context: - PythonInfo.from_exe(exe, session_app_data) + PythonInfo.from_exe(exe, app_data, cache) msg = str(context.value) assert "code" in msg assert exe in msg -def test_bad_exe_py_info_no_raise(tmp_path, caplog, capsys, session_app_data): +def test_bad_exe_py_info_no_raise(tmp_path, caplog, capsys): caplog.set_level(logging.NOTSET) exe = str(tmp_path) - result = PythonInfo.from_exe(exe, session_app_data, raise_on_error=False) + app_data, cache = MockAppData(), MockCache() + result = PythonInfo.from_exe(exe, app_data, cache, raise_on_error=False) assert result is None out, _ = capsys.readouterr() assert not out @@ -59,7 +65,7 @@ def test_bad_exe_py_info_no_raise(tmp_path, caplog, capsys, session_app_data): itertools.chain( [sys.executable], [ - f"{impl}{'.'.join(str(i) for i in ver)}{arch}" + f"{impl}{'.'.join(str(i) for i in ver)}{'t' if CURRENT.free_threaded else ''}{arch}" for impl, ver, arch in itertools.product( ( [CURRENT.implementation] @@ -90,6 +96,14 @@ def test_satisfy_not_arch(): assert matches is False +def test_satisfy_not_threaded(): + parsed_spec = PythonSpec.from_string_spec( + f"{CURRENT.implementation}{CURRENT.version_info.major}{'' if CURRENT.free_threaded else 't'}", + ) + matches = CURRENT.satisfies(parsed_spec, True) + assert matches is False + + def _generate_not_match_current_interpreter_version(): result = [] for i in range(3): @@ -112,43 +126,88 @@ def test_satisfy_not_version(spec): assert matches is False -def test_py_info_cached_error(mocker, tmp_path, session_app_data): +def test_py_info_cached_error(mocker, tmp_path): spy = mocker.spy(cached_py_info, "_run_subprocess") + app_data, cache = MockAppData(), MockCache() with pytest.raises(RuntimeError): - PythonInfo.from_exe(str(tmp_path), session_app_data) + PythonInfo.from_exe(str(tmp_path), app_data, cache) with pytest.raises(RuntimeError): - PythonInfo.from_exe(str(tmp_path), session_app_data) + PythonInfo.from_exe(str(tmp_path), app_data, cache) assert spy.call_count == 1 @pytest.mark.skipif(not fs_supports_symlink(), reason="symlink is not supported") -def test_py_info_cached_symlink_error(mocker, tmp_path, session_app_data): +def test_py_info_cached_symlink_error(mocker, tmp_path): spy = mocker.spy(cached_py_info, "_run_subprocess") + app_data, cache = MockAppData(), MockCache() with pytest.raises(RuntimeError): - PythonInfo.from_exe(str(tmp_path), session_app_data) + PythonInfo.from_exe(str(tmp_path), app_data, cache) symlinked = tmp_path / "a" symlinked.symlink_to(tmp_path) with pytest.raises(RuntimeError): - PythonInfo.from_exe(str(symlinked), session_app_data) + PythonInfo.from_exe(str(symlinked), app_data, cache) assert spy.call_count == 2 -def test_py_info_cache_clear(mocker, session_app_data): +def test_py_info_cache_clear(mocker): spy = mocker.spy(cached_py_info, "_run_subprocess") - result = PythonInfo.from_exe(sys.executable, session_app_data) + app_data, cache = MockAppData(), MockCache() + result = PythonInfo.from_exe(sys.executable, app_data, cache) assert result is not None count = 1 if result.executable == sys.executable else 2 # at least two, one for the venv, one more for the host assert spy.call_count >= count - PythonInfo.clear_cache(session_app_data) - assert PythonInfo.from_exe(sys.executable, session_app_data) is not None + PythonInfo.clear_cache() + assert PythonInfo.from_exe(sys.executable, app_data, cache) is not None assert spy.call_count >= 2 * count -@pytest.mark.xfail(IS_PYPY and IS_WIN and sys.version_info[0:2] == (3, 9), reason="symlink is not supported") +def test_py_info_cache_invalidation_on_py_info_change(mocker): + # 1. Get a PythonInfo object for the current executable, this will cache it. + app_data, cache = MockAppData(), MockCache() + PythonInfo.from_exe(sys.executable, app_data, cache) + + # 2. Spy on _run_subprocess + spy = mocker.spy(cached_py_info, "_run_subprocess") + + # 3. Backup py_info.py + py_info_script = Path(cached_py_info.__file__).parent / "py_info.py" + original_content = py_info_script.read_text(encoding="utf-8") + original_stat = py_info_script.stat() + + try: + # 4. Clear the in-memory cache + mocker.patch.dict(cached_py_info._CACHE, {}, clear=True) # noqa: SLF001 + + # 5. Modify py_info.py to invalidate the cache + py_info_script.write_text(original_content + "\n# a comment", encoding="utf-8") + + # 6. Get the PythonInfo object again + info = PythonInfo.from_exe(sys.executable, app_data, cache) + + # 7. Assert that _run_subprocess was called again + native_difference = 1 if info.system_executable == info.executable else 0 + if is_macos_brew(info): + assert spy.call_count + native_difference in {2, 3} + else: + assert spy.call_count + native_difference == 2 + + finally: + # 8. Restore the original content and timestamp + py_info_script.write_text(original_content, encoding="utf-8") + os.utime(str(py_info_script), (original_stat.st_atime, original_stat.st_mtime)) + + +@pytest.mark.skipif(not fs_supports_symlink(), reason="symlink is not supported") +@pytest.mark.xfail( + # https://doc.pypy.org/en/latest/install.html?highlight=symlink#download-a-pre-built-pypy + IS_PYPY and IS_WIN and sys.version_info[0:2] >= (3, 9), + reason="symlink is not supported", +) @pytest.mark.skipif(not fs_supports_symlink(), reason="symlink is not supported") -def test_py_info_cached_symlink(mocker, tmp_path, session_app_data): +def test_py_info_cached_symlink(mocker, tmp_path): spy = mocker.spy(cached_py_info, "_run_subprocess") - first_result = PythonInfo.from_exe(sys.executable, session_app_data) + app_data, cache = MockAppData(), MockCache() + first_result = PythonInfo.from_exe(sys.executable, app_data, cache) assert first_result is not None count = spy.call_count # at least two, one for the venv, one more for the host @@ -161,7 +220,7 @@ def test_py_info_cached_symlink(mocker, tmp_path, session_app_data): if pyvenv.exists(): (tmp_path / pyvenv.name).write_text(pyvenv.read_text(encoding="utf-8"), encoding="utf-8") new_exe_str = str(new_exe) - second_result = PythonInfo.from_exe(new_exe_str, session_app_data) + second_result = PythonInfo.from_exe(new_exe_str, app_data, cache) assert second_result.executable == new_exe_str assert spy.call_count == count + 1 # no longer needed the host invocation, but the new symlink is must @@ -208,10 +267,10 @@ def test_system_executable_no_exact_match( # noqa: PLR0913 tmp_path, mocker, caplog, - session_app_data, ): """Here we should fallback to other compatible""" caplog.set_level(logging.DEBUG) + app_data, cache = MockAppData(), MockCache() def _make_py_info(of): base = copy.deepcopy(CURRENT) @@ -239,15 +298,15 @@ def _make_py_info(of): mocker.patch.object(target_py_info, "_find_possible_exe_names", return_value=names) mocker.patch.object(target_py_info, "_find_possible_folders", return_value=[str(tmp_path)]) - def func(k, app_data, resolve_to_host, raise_on_error, env): # noqa: ARG001 - return discovered_with_path[k] + def func(exe, app_data, cache, raise_on_error=True, ignore_cache=False, resolve_to_host=True, env=None): # noqa: ARG001, PLR0913 + return discovered_with_path.get(exe) - mocker.patch.object(target_py_info, "from_exe", side_effect=func) + mocker.patch.object(PythonInfo, "from_exe", side_effect=func) target_py_info.real_prefix = str(tmp_path) target_py_info.system_executable = None target_py_info.executable = str(tmp_path) - mapped = target_py_info._resolve_to_system(session_app_data, target_py_info) # noqa: SLF001 + mapped = target_py_info._resolve_to_system(app_data, target_py_info, cache) # noqa: SLF001 assert mapped.system_executable == CURRENT.system_executable found = discovered_with_path[mapped.base_executable] assert found is selected @@ -274,7 +333,8 @@ def test_py_info_ignores_distutils_config(monkeypatch, tmp_path): """ (tmp_path / "setup.cfg").write_text(dedent(raw), encoding="utf-8") monkeypatch.chdir(tmp_path) - py_info = PythonInfo.from_exe(sys.executable) + app_data, cache = MockAppData(), MockCache() + py_info = PythonInfo.from_exe(sys.executable, app_data, cache) distutils = py_info.distutils_install for key, value in distutils.items(): assert not value.startswith(str(tmp_path)), f"{key}={value}" @@ -304,17 +364,18 @@ def test_discover_exe_on_path_non_spec_name_not_match(mocker): @pytest.mark.skipif(IS_PYPY, reason="setuptools distutils patching does not work") def test_py_info_setuptools(): - from setuptools.dist import Distribution + from setuptools.dist import Distribution # noqa: PLC0415 assert Distribution PythonInfo() @pytest.mark.usefixtures("_skip_if_test_in_system") -def test_py_info_to_system_raises(session_app_data, mocker, caplog): +def test_py_info_to_system_raises(mocker, caplog): caplog.set_level(logging.DEBUG) mocker.patch.object(PythonInfo, "_find_possible_folders", return_value=[]) - result = PythonInfo.from_exe(sys.executable, app_data=session_app_data, raise_on_error=False) + app_data, cache = MockAppData(), MockCache() + result = PythonInfo.from_exe(sys.executable, app_data=app_data, cache=cache, raise_on_error=False) assert result is None log = caplog.records[-1] assert log.levelno == logging.INFO @@ -385,7 +446,9 @@ def test_custom_venv_install_scheme_is_prefered(mocker): assert pyinfo.install_path("purelib").replace(os.sep, "/") == f"lib/python{pyver}/site-packages" -@pytest.mark.skipif(not (os.name == "posix" and sys.version_info[:2] >= (3, 11)), reason="POSIX 3.11+ specific") +@pytest.mark.skipif( + IS_PYPY or not (os.name == "posix" and sys.version_info[:2] >= (3, 11)), reason="POSIX 3.11+ specific" +) def test_fallback_existent_system_executable(mocker): current = PythonInfo() # Posix may execute a "python" out of a venv but try to set the base_executable @@ -401,11 +464,11 @@ def test_fallback_existent_system_executable(mocker): mocker.patch.object(sys, "executable", current.executable) # ensure it falls back to an alternate binary name that exists - current._fast_get_system_executable() # noqa: SLF001 - assert os.path.basename(current.system_executable) in [ + system_executable = current._fast_get_system_executable() # noqa: SLF001 + assert os.path.basename(system_executable) in [ f"python{v}" for v in (current.version_info.major, f"{current.version_info.major}.{current.version_info.minor}") ] - assert os.path.exists(current.system_executable) + assert os.path.exists(system_executable) @pytest.mark.skipif(sys.version_info[:2] != (3, 10), reason="3.10 specific") diff --git a/tests/unit/discovery/py_info/test_py_info_exe_based_of.py b/tests/unit/discovery/py_info/test_py_info_exe_based_of.py index c843ca7be..f3846422e 100644 --- a/tests/unit/discovery/py_info/test_py_info_exe_based_of.py +++ b/tests/unit/discovery/py_info/test_py_info_exe_based_of.py @@ -6,15 +6,17 @@ import pytest +from tests.unit.discovery.util import MockAppData, MockCache from virtualenv.discovery.py_info import EXTENSIONS, PythonInfo from virtualenv.info import IS_WIN, fs_is_case_sensitive, fs_supports_symlink -CURRENT = PythonInfo.current() +CURRENT = PythonInfo.current(MockAppData(), MockCache()) -def test_discover_empty_folder(tmp_path, session_app_data): +def test_discover_empty_folder(tmp_path): + app_data, cache = MockAppData(), MockCache() with pytest.raises(RuntimeError): - CURRENT.discover_exe(session_app_data, prefix=str(tmp_path)) + CURRENT.discover_exe(app_data, cache, prefix=str(tmp_path)) BASE = (CURRENT.install_path("scripts"), ".") @@ -26,11 +28,12 @@ def test_discover_empty_folder(tmp_path, session_app_data): @pytest.mark.parametrize("arch", [CURRENT.architecture, ""]) @pytest.mark.parametrize("version", [".".join(str(i) for i in CURRENT.version_info[0:i]) for i in range(3, 0, -1)]) @pytest.mark.parametrize("impl", [CURRENT.implementation, "python"]) -def test_discover_ok(tmp_path, suffix, impl, version, arch, into, caplog, session_app_data): # noqa: PLR0913 +def test_discover_ok(tmp_path, suffix, impl, version, arch, into, caplog): # noqa: PLR0913 caplog.set_level(logging.DEBUG) + app_data, cache = MockAppData(), MockCache() folder = tmp_path / into folder.mkdir(parents=True, exist_ok=True) - name = f"{impl}{version}" + name = f"{impl}{version}{'t' if CURRENT.free_threaded else ''}" if arch: name += f"-{arch}" name += suffix @@ -40,7 +43,7 @@ def test_discover_ok(tmp_path, suffix, impl, version, arch, into, caplog, sessio if pyvenv.exists(): (folder / pyvenv.name).write_text(pyvenv.read_text(encoding="utf-8"), encoding="utf-8") inside_folder = str(tmp_path) - base = CURRENT.discover_exe(session_app_data, inside_folder) + base = CURRENT.discover_exe(app_data, cache, inside_folder) found = base.executable dest_str = str(dest) if not fs_is_case_sensitive(): @@ -53,4 +56,4 @@ def test_discover_ok(tmp_path, suffix, impl, version, arch, into, caplog, sessio dest.rename(dest.parent / (dest.name + "-1")) CURRENT._cache_exe_discovery.clear() # noqa: SLF001 with pytest.raises(RuntimeError): - CURRENT.discover_exe(session_app_data, inside_folder) + CURRENT.discover_exe(app_data, cache, inside_folder) diff --git a/tests/unit/discovery/test_discovery.py b/tests/unit/discovery/test_discovery.py index f5eb17cbb..f517101e2 100644 --- a/tests/unit/discovery/test_discovery.py +++ b/tests/unit/discovery/test_discovery.py @@ -2,30 +2,48 @@ import logging import os +import subprocess import sys from argparse import Namespace from pathlib import Path +from unittest.mock import patch from uuid import uuid4 import pytest -from virtualenv.discovery.builtin import Builtin, get_interpreter +from tests.unit.discovery.util import MockAppData, MockCache +from virtualenv.discovery.builtin import Builtin, LazyPathDump, get_interpreter, get_paths from virtualenv.discovery.py_info import PythonInfo -from virtualenv.info import fs_supports_symlink +from virtualenv.info import IS_WIN, fs_supports_symlink @pytest.mark.skipif(not fs_supports_symlink(), reason="symlink not supported") @pytest.mark.parametrize("case", ["mixed", "lower", "upper"]) -def test_discovery_via_path(monkeypatch, case, tmp_path, caplog, session_app_data): +@pytest.mark.parametrize("specificity", ["more", "less", "none"]) +def test_discovery_via_path(monkeypatch, case, specificity, tmp_path, caplog): caplog.set_level(logging.DEBUG) - current = PythonInfo.current_system(session_app_data) - core = f"somethingVeryCryptic{'.'.join(str(i) for i in current.version_info[0:3])}" + app_data, cache = MockAppData(), MockCache() + current = PythonInfo.current_system(app_data, cache) name = "somethingVeryCryptic" + threaded = "t" if current.free_threaded else "" if case == "lower": name = name.lower() elif case == "upper": name = name.upper() - exe_name = f"{name}{current.version_info.major}{'.exe' if sys.platform == 'win32' else ''}" + if specificity == "more": + # e.g. spec: python3, exe: /bin/python3.12 + core_ver = current.version_info.major + exe_ver = ".".join(str(i) for i in current.version_info[0:2]) + threaded + elif specificity == "less": + # e.g. spec: python3.12.1, exe: /bin/python3 + core_ver = ".".join(str(i) for i in current.version_info[0:3]) + exe_ver = current.version_info.major + elif specificity == "none": + # e.g. spec: python3.12.1, exe: /bin/python + core_ver = ".".join(str(i) for i in current.version_info[0:3]) + exe_ver = "" + core = "" if specificity == "none" else f"{name}{core_ver}{threaded}" + exe_name = f"{name}{exe_ver}{'.exe' if sys.platform == 'win32' else ''}" target = tmp_path / current.install_path("scripts") target.mkdir(parents=True) executable = target / exe_name @@ -35,30 +53,113 @@ def test_discovery_via_path(monkeypatch, case, tmp_path, caplog, session_app_dat (target / pyvenv_cfg.name).write_bytes(pyvenv_cfg.read_bytes()) new_path = os.pathsep.join([str(target), *os.environ.get("PATH", "").split(os.pathsep)]) monkeypatch.setenv("PATH", new_path) - interpreter = get_interpreter(core, []) + interpreter = get_interpreter(core, [], app_data, cache) assert interpreter is not None def test_discovery_via_path_not_found(tmp_path, monkeypatch): monkeypatch.setenv("PATH", str(tmp_path)) - interpreter = get_interpreter(uuid4().hex, []) + app_data, cache = MockAppData(), MockCache() + interpreter = get_interpreter(uuid4().hex, [], app_data, cache) assert interpreter is None -def test_relative_path(session_app_data, monkeypatch): - sys_executable = Path(PythonInfo.current_system(app_data=session_app_data).system_executable) +def test_discovery_via_path_in_nonbrowseable_directory(tmp_path, monkeypatch): + bad_perm = tmp_path / "bad_perm" + bad_perm.mkdir(mode=0o000) + app_data, cache = MockAppData(), MockCache() + # path entry is unreadable + monkeypatch.setenv("PATH", str(bad_perm)) + interpreter = get_interpreter(uuid4().hex, [], app_data, cache) + assert interpreter is None + # path entry parent is unreadable + monkeypatch.setenv("PATH", str(bad_perm / "bin")) + interpreter = get_interpreter(uuid4().hex, [], app_data, cache) + assert interpreter is None + + +def test_relative_path(monkeypatch): + app_data, cache = MockAppData(), MockCache() + sys_executable = Path(PythonInfo.current_system(app_data=app_data, cache=cache).system_executable) cwd = sys_executable.parents[1] monkeypatch.chdir(str(cwd)) relative = str(sys_executable.relative_to(cwd)) - result = get_interpreter(relative, [], session_app_data) + result = get_interpreter(relative, [], app_data, cache) assert result is not None -def test_discovery_fallback_fail(session_app_data, caplog): +def test_uv_python(monkeypatch, tmp_path_factory, mocker): + monkeypatch.delenv("UV_PYTHON_INSTALL_DIR", raising=False) + monkeypatch.delenv("XDG_DATA_HOME", raising=False) + monkeypatch.setenv("PATH", "") + mocker.patch.object(PythonInfo, "satisfies", return_value=False) + + # UV_PYTHON_INSTALL_DIR + uv_python_install_dir = tmp_path_factory.mktemp("uv_python_install_dir") + with patch("virtualenv.discovery.builtin.PathPythonInfo.from_exe") as mock_from_exe, monkeypatch.context() as m: + m.setenv("UV_PYTHON_INSTALL_DIR", str(uv_python_install_dir)) + + app_data, cache = MockAppData(), MockCache() + get_interpreter("python", [], app_data, cache) + mock_from_exe.assert_not_called() + + bin_path = uv_python_install_dir.joinpath("some-py-impl", "bin") + bin_path.mkdir(parents=True) + bin_path.joinpath("python").touch() + get_interpreter("python", [], app_data, cache) + mock_from_exe.assert_called_once() + assert mock_from_exe.call_args[0][0] == str(bin_path / "python") + + # PATH takes precedence + mock_from_exe.reset_mock() + python_exe = "python.exe" if IS_WIN else "python" + dir_in_path = tmp_path_factory.mktemp("path_bin_dir") + dir_in_path.joinpath(python_exe).touch() + m.setenv("PATH", str(dir_in_path)) + get_interpreter("python", []) + mock_from_exe.assert_called_once() + assert mock_from_exe.call_args[0][0] == str(dir_in_path / python_exe) + + # XDG_DATA_HOME + xdg_data_home = tmp_path_factory.mktemp("xdg_data_home") + with patch("virtualenv.discovery.builtin.PathPythonInfo.from_exe") as mock_from_exe, monkeypatch.context() as m: + m.setenv("XDG_DATA_HOME", str(xdg_data_home)) + + app_data, cache = MockAppData(), MockCache() + get_interpreter("python", [], app_data, cache) + mock_from_exe.assert_not_called() + + bin_path = xdg_data_home.joinpath("uv", "python", "some-py-impl", "bin") + bin_path.mkdir(parents=True) + bin_path.joinpath("python").touch() + get_interpreter("python", [], app_data, cache) + mock_from_exe.assert_called_once() + assert mock_from_exe.call_args[0][0] == str(bin_path / "python") + + # User data path + user_data_path = tmp_path_factory.mktemp("user_data_path") + with patch("virtualenv.discovery.builtin.PathPythonInfo.from_exe") as mock_from_exe, monkeypatch.context() as m: + m.setattr("virtualenv.discovery.builtin.user_data_path", lambda x: user_data_path / x) + + app_data, cache = MockAppData(), MockCache() + get_interpreter("python", [], app_data, cache) + mock_from_exe.assert_not_called() + + bin_path = user_data_path.joinpath("uv", "python", "some-py-impl", "bin") + bin_path.mkdir(parents=True) + bin_path.joinpath("python").touch() + get_interpreter("python", [], app_data, cache) + mock_from_exe.assert_called_once() + assert mock_from_exe.call_args[0][0] == str(bin_path / "python") + + +def test_discovery_fallback_fail(caplog): caplog.set_level(logging.DEBUG) + app_data, cache = MockAppData(), MockCache() builtin = Builtin( - Namespace(app_data=session_app_data, try_first_with=[], python=["magic-one", "magic-two"], env=os.environ), + Namespace(app_data=app_data, try_first_with=[], python=["magic-one", "magic-two"], env=os.environ), + cache, ) result = builtin.run() @@ -67,10 +168,12 @@ def test_discovery_fallback_fail(session_app_data, caplog): assert "accepted" not in caplog.text -def test_discovery_fallback_ok(session_app_data, caplog): +def test_discovery_fallback_ok(caplog): caplog.set_level(logging.DEBUG) + app_data, cache = MockAppData(), MockCache() builtin = Builtin( - Namespace(app_data=session_app_data, try_first_with=[], python=["magic-one", sys.executable], env=os.environ), + Namespace(app_data=app_data, try_first_with=[], python=["magic-one", sys.executable], env=os.environ), + cache, ) result = builtin.run() @@ -78,3 +181,170 @@ def test_discovery_fallback_ok(session_app_data, caplog): assert result.executable == sys.executable, caplog.text assert "accepted" in caplog.text + + +@pytest.fixture +def mock_get_interpreter(mocker): + return mocker.patch( + "virtualenv.discovery.builtin.get_interpreter", + lambda key, *args, **kwargs: getattr(mocker.sentinel, key), # noqa: ARG005 + ) + + +@pytest.mark.usefixtures("mock_get_interpreter") +def test_returns_first_python_specified_when_only_env_var_one_is_specified(mocker, monkeypatch): + monkeypatch.setenv("VIRTUALENV_PYTHON", "python_from_env_var") + app_data, cache = MockAppData(), MockCache() + builtin = Builtin( + Namespace(app_data=app_data, try_first_with=[], python=["python_from_env_var"], env=os.environ), + cache, + ) + + result = builtin.run() + + assert result == mocker.sentinel.python_from_env_var + + +@pytest.mark.usefixtures("mock_get_interpreter") +def test_returns_second_python_specified_when_more_than_one_is_specified_and_env_var_is_specified(mocker, monkeypatch): + monkeypatch.setenv("VIRTUALENV_PYTHON", "python_from_env_var") + app_data, cache = MockAppData(), MockCache() + builtin = Builtin( + Namespace( + app_data=app_data, + try_first_with=[], + python=["python_from_env_var", "python_from_cli"], + env=os.environ, + ), + cache, + ) + + result = builtin.run() + + assert result == mocker.sentinel.python_from_cli + + +def test_discovery_absolute_path_with_try_first(tmp_path): + good_env = tmp_path / "good" + bad_env = tmp_path / "bad" + + # Create two real virtual environments + subprocess.check_call([sys.executable, "-m", "virtualenv", str(good_env)]) + subprocess.check_call([sys.executable, "-m", "virtualenv", str(bad_env)]) + + # On Windows, the executable is in Scripts/python.exe + scripts_dir = "Scripts" if IS_WIN else "bin" + exe_name = "python.exe" if IS_WIN else "python" + good_exe = good_env / scripts_dir / exe_name + bad_exe = bad_env / scripts_dir / exe_name + + # The spec is an absolute path, this should be a hard requirement. + # The --try-first-with option should be rejected as it does not match the spec. + app_data, cache = MockAppData(), MockCache() + interpreter = get_interpreter( + str(good_exe), + try_first_with=[str(bad_exe)], + app_data=app_data, + cache=cache, + ) + + assert interpreter is not None + assert Path(interpreter.executable) == good_exe + + +def test_discovery_via_path_with_file(tmp_path, monkeypatch): + a_file = tmp_path / "a_file" + a_file.touch() + monkeypatch.setenv("PATH", str(a_file)) + app_data, cache = MockAppData(), MockCache() + interpreter = get_interpreter(uuid4().hex, [], app_data, cache) + assert interpreter is None + + +def test_absolute_path_does_not_exist(tmp_path): + """ + Test that virtualenv does not fail when an absolute path that does not exist is provided. + """ + # Create a command that uses an absolute path that does not exist + # and a valid python executable. + command = [ + sys.executable, + "-m", + "virtualenv", + "-p", + "/this/path/does/not/exist", + "-p", + sys.executable, + str(tmp_path / "dest"), + ] + + # Run the command + process = subprocess.run( + command, + capture_output=True, + text=True, + check=False, + encoding="utf-8", + ) + + # Check that the command was successful + assert process.returncode == 0, process.stderr + + +def test_absolute_path_does_not_exist_fails(tmp_path): + """ + Test that virtualenv fails when a single absolute path that does not exist is provided. + """ + # Create a command that uses an absolute path that does not exist + command = [ + sys.executable, + "-m", + "virtualenv", + "-p", + "/this/path/does/not/exist", + str(tmp_path / "dest"), + ] + + # Run the command + process = subprocess.run( + command, + capture_output=True, + text=True, + check=False, + encoding="utf-8", + ) + + # Check that the command failed + assert process.returncode != 0, process.stderr + + +def test_get_paths_no_path_env(monkeypatch): + monkeypatch.delenv("PATH", raising=False) + paths = list(get_paths({})) + assert paths + + +def test_lazy_path_dump_debug(monkeypatch, tmp_path): + monkeypatch.setenv("_VIRTUALENV_DEBUG", "1") + a_dir = tmp_path + executable_file = "a_file.exe" if IS_WIN else "a_file" + (a_dir / executable_file).touch(mode=0o755) + (a_dir / "b_file").touch(mode=0o644) + dumper = LazyPathDump(0, a_dir, os.environ) + output = repr(dumper) + assert executable_file in output + assert "b_file" not in output + + +@pytest.mark.usefixtures("mock_get_interpreter") +def test_returns_first_python_specified_when_no_env_var_is_specified(mocker, monkeypatch): + monkeypatch.delenv("VIRTUALENV_PYTHON", raising=False) + app_data, cache = MockAppData(), MockCache() + builtin = Builtin( + Namespace(app_data=app_data, try_first_with=[], python=["python_from_cli"], env=os.environ), + cache, + ) + + result = builtin.run() + + assert result == mocker.sentinel.python_from_cli diff --git a/tests/unit/discovery/test_py_spec.py b/tests/unit/discovery/test_py_spec.py index 765686645..0841019ec 100644 --- a/tests/unit/discovery/test_py_spec.py +++ b/tests/unit/discovery/test_py_spec.py @@ -45,6 +45,16 @@ def test_spec_satisfies_arch(): assert spec_2.satisfies(spec_1) is False +def test_spec_satisfies_free_threaded(): + spec_1 = PythonSpec.from_string_spec("python3.13t") + spec_2 = PythonSpec.from_string_spec("python3.13") + + assert spec_1.satisfies(spec_1) is True + assert spec_1.free_threaded is True + assert spec_2.satisfies(spec_1) is False + assert spec_2.free_threaded is False + + @pytest.mark.parametrize( ("req", "spec"), [("py", "python"), ("jython", "jython"), ("CPython", "cpython")], @@ -66,13 +76,22 @@ def test_spec_satisfies_implementation_nok(): def _version_satisfies_pairs(): target = set() version = tuple(str(i) for i in sys.version_info[0:3]) - for i in range(len(version) + 1): - req = ".".join(version[0:i]) - for j in range(i + 1): - sat = ".".join(version[0:j]) - # can be satisfied in both directions - target.add((req, sat)) - target.add((sat, req)) + for threading in (False, True): + for i in range(len(version) + 1): + req = ".".join(version[0:i]) + for j in range(i + 1): + sat = ".".join(version[0:j]) + # can be satisfied in both directions + if sat: + target.add((req, sat)) + # else: no version => no free-threading info + target.add((sat, req)) + if not threading or not sat or not req: + # free-threading info requires a version + continue + target.add((f"{req}t", f"{sat}t")) + target.add((f"{sat}t", f"{req}t")) + return sorted(target) diff --git a/tests/unit/discovery/util.py b/tests/unit/discovery/util.py new file mode 100644 index 000000000..7908c7df8 --- /dev/null +++ b/tests/unit/discovery/util.py @@ -0,0 +1,51 @@ +from __future__ import annotations + +from contextlib import contextmanager +from typing import TYPE_CHECKING, Any, ContextManager + +if TYPE_CHECKING: + from pathlib import Path + + +class MockAppData: + def __init__(self, readonly: bool = False) -> None: + self.readonly = readonly + self._py_info_clear_called = 0 + self._py_info_map: dict[Path, Any] = {} + + def py_info(self, path: Path) -> Any: + return self._py_info_map.get(path) + + def py_info_clear(self) -> None: + self._py_info_clear_called += 1 + + @contextmanager + def ensure_extracted(self, path: Path, to_folder: Path | None = None) -> ContextManager[Path]: # noqa: ARG002 + yield path + + @contextmanager + def extract(self, path: Path, to_folder: Path | None = None) -> ContextManager[Path]: # noqa: ARG002 + yield path + + def close(self) -> None: + pass + + +class MockCache: + def __init__(self) -> None: + self._cache: dict[Path, Any] = {} + self._clear_called = 0 + + def get(self, path: Path) -> Any: + return self._cache.get(path) + + def set(self, path: Path, data: Any) -> None: + self._cache[path] = data + + def remove(self, path: Path) -> None: + if path in self._cache: + del self._cache[path] + + def clear(self) -> None: + self._clear_called += 1 + self._cache.clear() diff --git a/tests/unit/discovery/windows/conftest.py b/tests/unit/discovery/windows/conftest.py index 8959ee9e2..f75278e06 100644 --- a/tests/unit/discovery/windows/conftest.py +++ b/tests/unit/discovery/windows/conftest.py @@ -6,9 +6,9 @@ import pytest -@pytest.fixture() +@pytest.fixture def _mock_registry(mocker): # noqa: C901 - from virtualenv.discovery.windows.pep514 import winreg + from virtualenv.discovery.windows.pep514 import winreg # noqa: PLC0415 loc, glob = {}, {} mock_value_str = (Path(__file__).parent / "winreg-mock-values.py").read_text(encoding="utf-8") @@ -65,9 +65,9 @@ def _open_key_ex(*args): mocker.patch("os.path.exists", return_value=True) -def _mock_pyinfo(major, minor, arch, exe): +def _mock_pyinfo(major, minor, arch, exe, threaded=False): """Return PythonInfo objects with essential metadata set for the given args""" - from virtualenv.discovery.py_info import PythonInfo, VersionInfo + from virtualenv.discovery.py_info import PythonInfo, VersionInfo # noqa: PLC0415 info = PythonInfo() info.base_prefix = str(Path(exe).parent) @@ -75,28 +75,31 @@ def _mock_pyinfo(major, minor, arch, exe): info.implementation = "CPython" info.architecture = arch info.version_info = VersionInfo(major, minor, 0, "final", 0) + info.free_threaded = threaded return info -@pytest.fixture() +@pytest.fixture def _populate_pyinfo_cache(monkeypatch): """Add metadata to virtualenv.discovery.cached_py_info._CACHE for all (mocked) registry entries""" - import virtualenv.discovery.cached_py_info + import virtualenv.discovery.cached_py_info # noqa: PLC0415 # Data matches _mock_registry fixture + python_core_path = "C:\\Users\\user\\AppData\\Local\\Programs\\Python" interpreters = [ - ("ContinuumAnalytics", 3, 10, 32, "C:\\Users\\user\\Miniconda3\\python.exe", None), - ("ContinuumAnalytics", 3, 10, 64, "C:\\Users\\user\\Miniconda3-64\\python.exe", None), - ("PythonCore", 3, 9, 64, "C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python36\\python.exe", None), - ("PythonCore", 3, 9, 64, "C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python36\\python.exe", None), - ("PythonCore", 3, 5, 64, "C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python35\\python.exe", None), - ("PythonCore", 3, 9, 64, "C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python36\\python.exe", None), - ("PythonCore", 3, 7, 32, "C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python37-32\\python.exe", None), - ("PythonCore", 3, 12, 64, "C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python312\\python.exe", None), - ("PythonCore", 2, 7, 64, "C:\\Python27\\python.exe", None), - ("PythonCore", 3, 4, 64, "C:\\Python34\\python.exe", None), - ("CompanyA", 3, 6, 64, "Z:\\CompanyA\\Python\\3.6\\python.exe", None), + ("ContinuumAnalytics", 3, 10, 32, False, "C:\\Users\\user\\Miniconda3\\python.exe"), + ("ContinuumAnalytics", 3, 10, 64, False, "C:\\Users\\user\\Miniconda3-64\\python.exe"), + ("PythonCore", 3, 9, 64, False, f"{python_core_path}\\Python36\\python.exe"), + ("PythonCore", 3, 9, 64, False, f"{python_core_path}\\Python36\\python.exe"), + ("PythonCore", 3, 5, 64, False, f"{python_core_path}\\Python35\\python.exe"), + ("PythonCore", 3, 9, 64, False, f"{python_core_path}\\Python36\\python.exe"), + ("PythonCore", 3, 7, 32, False, f"{python_core_path}\\Python37-32\\python.exe"), + ("PythonCore", 3, 12, 64, False, f"{python_core_path}\\Python312\\python.exe"), + ("PythonCore", 3, 13, 64, True, f"{python_core_path}\\Python313\\python3.13t.exe"), + ("PythonCore", 2, 7, 64, False, "C:\\Python27\\python.exe"), + ("PythonCore", 3, 4, 64, False, "C:\\Python34\\python.exe"), + ("CompanyA", 3, 6, 64, False, "Z:\\CompanyA\\Python\\3.6\\python.exe"), ] - for _, major, minor, arch, exe, _ in interpreters: - info = _mock_pyinfo(major, minor, arch, exe) + for _, major, minor, arch, threaded, exe in interpreters: + info = _mock_pyinfo(major, minor, arch, exe, threaded) monkeypatch.setitem(virtualenv.discovery.cached_py_info._CACHE, Path(info.executable), info) # noqa: SLF001 diff --git a/tests/unit/discovery/windows/test_windows.py b/tests/unit/discovery/windows/test_windows.py index 38dd546a4..98b849f57 100644 --- a/tests/unit/discovery/windows/test_windows.py +++ b/tests/unit/discovery/windows/test_windows.py @@ -4,6 +4,7 @@ import pytest +from tests.unit.discovery.util import MockAppData, MockCache from virtualenv.discovery.py_spec import PythonSpec @@ -20,16 +21,21 @@ ("python3.12", "C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python312\\python.exe"), ("cpython3.12", "C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python312\\python.exe"), # resolves to highest available version - ("python", "C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python312\\python.exe"), - ("cpython", "C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python312\\python.exe"), + ("python", "C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python313\\python3.13t.exe"), + ("cpython", "C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python313\\python3.13t.exe"), + ("python3", "C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python312\\python.exe"), + ("cpython3", "C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python312\\python.exe"), # Non-standard org name ("python3.6", "Z:\\CompanyA\\Python\\3.6\\python.exe"), ("cpython3.6", "Z:\\CompanyA\\Python\\3.6\\python.exe"), + # free-threaded + ("3t", "C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python313\\python3.13t.exe"), + ("python3.13t", "C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python313\\python3.13t.exe"), ], ) def test_propose_interpreters(string_spec, expected_exe): - from virtualenv.discovery.windows import propose_interpreters + from virtualenv.discovery.windows import propose_interpreters # noqa: PLC0415 spec = PythonSpec.from_string_spec(string_spec) - interpreter = next(propose_interpreters(spec=spec, cache_dir=None, env=None)) + interpreter = next(propose_interpreters(spec, MockAppData(), MockCache(), env=None)) assert interpreter.executable == expected_exe diff --git a/tests/unit/discovery/windows/test_windows_pep514.py b/tests/unit/discovery/windows/test_windows_pep514.py index 0d36840ba..0498352aa 100644 --- a/tests/unit/discovery/windows/test_windows_pep514.py +++ b/tests/unit/discovery/windows/test_windows_pep514.py @@ -9,45 +9,103 @@ @pytest.mark.skipif(sys.platform != "win32", reason="no Windows registry") @pytest.mark.usefixtures("_mock_registry") def test_pep514(): - from virtualenv.discovery.windows.pep514 import discover_pythons + from virtualenv.discovery.windows.pep514 import discover_pythons # noqa: PLC0415 interpreters = list(discover_pythons()) assert interpreters == [ - ("ContinuumAnalytics", 3, 10, 32, "C:\\Users\\user\\Miniconda3\\python.exe", None), - ("ContinuumAnalytics", 3, 10, 64, "C:\\Users\\user\\Miniconda3-64\\python.exe", None), - ("PythonCore", 3, 9, 64, "C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python39\\python.exe", None), - ("PythonCore", 3, 9, 64, "C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python39\\python.exe", None), - ("PythonCore", 3, 8, 64, "C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python38\\python.exe", None), - ("PythonCore", 3, 9, 64, "C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python39\\python.exe", None), - ("PythonCore", 3, 10, 32, "C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python310-32\\python.exe", None), - ("PythonCore", 3, 12, 64, "C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python312\\python.exe", None), - ("CompanyA", 3, 6, 64, "Z:\\CompanyA\\Python\\3.6\\python.exe", None), - ("PythonCore", 2, 7, 64, "C:\\Python27\\python.exe", None), - ("PythonCore", 3, 7, 64, "C:\\Python37\\python.exe", None), + ("ContinuumAnalytics", 3, 10, 32, False, "C:\\Users\\user\\Miniconda3\\python.exe", None), + ("ContinuumAnalytics", 3, 10, 64, False, "C:\\Users\\user\\Miniconda3-64\\python.exe", None), + ( + "PythonCore", + 3, + 9, + 64, + False, + "C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python39\\python.exe", + None, + ), + ( + "PythonCore", + 3, + 9, + 64, + False, + "C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python39\\python.exe", + None, + ), + ( + "PythonCore", + 3, + 8, + 64, + False, + "C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python38\\python.exe", + None, + ), + ( + "PythonCore", + 3, + 9, + 64, + False, + "C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python39\\python.exe", + None, + ), + ( + "PythonCore", + 3, + 10, + 32, + False, + "C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python310-32\\python.exe", + None, + ), + ( + "PythonCore", + 3, + 12, + 64, + False, + "C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python312\\python.exe", + None, + ), + ( + "PythonCore", + 3, + 13, + 64, + True, + "C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python313\\python3.13t.exe", + None, + ), + ("CompanyA", 3, 6, 64, False, "Z:\\CompanyA\\Python\\3.6\\python.exe", None), + ("PythonCore", 2, 7, 64, False, "C:\\Python27\\python.exe", None), + ("PythonCore", 3, 7, 64, False, "C:\\Python37\\python.exe", None), ] @pytest.mark.skipif(sys.platform != "win32", reason="no Windows registry") @pytest.mark.usefixtures("_mock_registry") def test_pep514_run(capsys, caplog): - from virtualenv.discovery.windows import pep514 + from virtualenv.discovery.windows import pep514 # noqa: PLC0415 pep514._run() # noqa: SLF001 out, err = capsys.readouterr() expected = textwrap.dedent( r""" - ('CompanyA', 3, 6, 64, 'Z:\\CompanyA\\Python\\3.6\\python.exe', None) - ('ContinuumAnalytics', 3, 10, 32, 'C:\\Users\\user\\Miniconda3\\python.exe', None) - ('ContinuumAnalytics', 3, 10, 64, 'C:\\Users\\user\\Miniconda3-64\\python.exe', None) - ('PythonCore', 2, 7, 64, 'C:\\Python27\\python.exe', None) - ('PythonCore', 3, 10, 32, 'C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python310-32\\python.exe', None) - ('PythonCore', 3, 12, 64, 'C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python312\\python.exe', None) - ('PythonCore', 3, 7, 64, 'C:\\Python37\\python.exe', None) - ('PythonCore', 3, 8, 64, 'C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python38\\python.exe', None) - ('PythonCore', 3, 9, 64, 'C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python39\\python.exe', None) - ('PythonCore', 3, 9, 64, 'C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python39\\python.exe', None) - ('PythonCore', 3, 9, 64, 'C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python39\\python.exe', None) - """, + ('CompanyA', 3, 6, 64, False, 'Z:\\CompanyA\\Python\\3.6\\python.exe', None) + ('ContinuumAnalytics', 3, 10, 32, False, 'C:\\Users\\user\\Miniconda3\\python.exe', None) + ('ContinuumAnalytics', 3, 10, 64, False, 'C:\\Users\\user\\Miniconda3-64\\python.exe', None) + ('PythonCore', 2, 7, 64, False, 'C:\\Python27\\python.exe', None) + ('PythonCore', 3, 10, 32, False, 'C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python310-32\\python.exe', None) + ('PythonCore', 3, 12, 64, False, 'C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python312\\python.exe', None) + ('PythonCore', 3, 13, 64, True, 'C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python313\\python3.13t.exe', None) + ('PythonCore', 3, 7, 64, False, 'C:\\Python37\\python.exe', None) + ('PythonCore', 3, 8, 64, False, 'C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python38\\python.exe', None) + ('PythonCore', 3, 9, 64, False, 'C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python39\\python.exe', None) + ('PythonCore', 3, 9, 64, False, 'C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python39\\python.exe', None) + ('PythonCore', 3, 9, 64, False, 'C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python39\\python.exe', None) + """, # noqa: E501 ).strip() assert out.strip() == expected assert not err diff --git a/tests/unit/discovery/windows/winreg-mock-values.py b/tests/unit/discovery/windows/winreg-mock-values.py index da76c56b9..fa2619ba7 100644 --- a/tests/unit/discovery/windows/winreg-mock-values.py +++ b/tests/unit/discovery/windows/winreg-mock-values.py @@ -35,6 +35,8 @@ "3.11": 78700656, "3.12\\InstallPath": 78703632, "3.12": 78702608, + "3.13t\\InstallPath": 78703633, + "3.13t": 78702609, "3.X": 78703088, }, 78702960: {"2.7\\InstallPath": 78700912, "2.7": 78703136, "3.7\\InstallPath": 78703648, "3.7": 78704032}, @@ -45,27 +47,27 @@ }, } value_collect = { - 78703568: {"SysVersion": ("3.10", 1), "SysArchitecture": ("32bit", 1)}, + 78703568: {"SysVersion": ("3.10", 1), "SysArchitecture": ("32bit", 1), "DisplayName": ("Python 3.10 (32-bit)", 1)}, 78703200: { "ExecutablePath": ("C:\\Users\\user\\Miniconda3\\python.exe", 1), "ExecutableArguments": OSError(2, "The system cannot find the file specified"), }, - 78702368: {"SysVersion": ("3.10", 1), "SysArchitecture": ("64bit", 1)}, + 78702368: {"SysVersion": ("3.10", 1), "SysArchitecture": ("64bit", 1), "DisplayName": ("Python 3.10 (64-bit)", 1)}, 78703520: { "ExecutablePath": ("C:\\Users\\user\\Miniconda3-64\\python.exe", 1), "ExecutableArguments": OSError(2, "The system cannot find the file specified"), }, - 78700704: {"SysVersion": ("3.9", 1), "SysArchitecture": ("magic", 1)}, + 78700704: {"SysVersion": ("3.9", 1), "SysArchitecture": ("magic", 1), "DisplayName": ("Python 3.9 (wizardry)", 1)}, 78701824: { "ExecutablePath": ("C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python39\\python.exe", 1), "ExecutableArguments": OSError(2, "The system cannot find the file specified"), }, - 78704368: {"SysVersion": ("3.9", 1), "SysArchitecture": (100, 4)}, + 78704368: {"SysVersion": ("3.9", 1), "SysArchitecture": (100, 4), "DisplayName": ("Python 3.9 (64-bit)", 1)}, 78704048: { "ExecutablePath": ("C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python39\\python.exe", 1), "ExecutableArguments": OSError(2, "The system cannot find the file specified"), }, - 78703024: {"SysVersion": ("3.9", 1), "SysArchitecture": ("64bit", 1)}, + 78703024: {"SysVersion": ("3.9", 1), "SysArchitecture": ("64bit", 1), "DisplayName": ("Python 3.9 (64-bit)", 1)}, 78701936: { "ExecutablePath": OSError(2, "The system cannot find the file specified"), None: OSError(2, "The system cannot find the file specified"), @@ -73,17 +75,18 @@ 78701792: { "SysVersion": OSError(2, "The system cannot find the file specified"), "SysArchitecture": OSError(2, "The system cannot find the file specified"), + "DisplayName": OSError(2, "The system cannot find the file specified"), }, 78703792: { "ExecutablePath": ("C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python38\\python.exe", 1), "ExecutableArguments": OSError(2, "The system cannot find the file specified"), }, - 78703424: {"SysVersion": ("3.9", 1), "SysArchitecture": ("64bit", 1)}, + 78703424: {"SysVersion": ("3.9", 1), "SysArchitecture": ("64bit", 1), "DisplayName": ("Python 3.9 (64-bit)", 1)}, 78701888: { "ExecutablePath": ("C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python39\\python.exe", 1), "ExecutableArguments": OSError(2, "The system cannot find the file specified"), }, - 78704512: {"SysVersion": ("3.10", 1), "SysArchitecture": ("32bit", 1)}, + 78704512: {"SysVersion": ("3.10", 1), "SysArchitecture": ("32bit", 1), "DisplayName": ("Python 3.10 (32-bit)", 1)}, 78703600: { "ExecutablePath": ("C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python310-32\\python.exe", 1), "ExecutableArguments": OSError(2, "The system cannot find the file specified"), @@ -91,16 +94,31 @@ 78700656: { "SysVersion": OSError(2, "The system cannot find the file specified"), "SysArchitecture": OSError(2, "The system cannot find the file specified"), + "DisplayName": OSError(2, "The system cannot find the file specified"), + }, + 78702608: { + "SysVersion": ("magic", 1), + "SysArchitecture": ("64bit", 1), + "DisplayName": ("Python 3.12 (wizard edition)", 1), }, - 78702608: {"SysVersion": ("magic", 1), "SysArchitecture": ("64bit", 1)}, 78703632: { "ExecutablePath": ("C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python312\\python.exe", 1), "ExecutableArguments": OSError(2, "The system cannot find the file specified"), }, + 78702609: { + "SysVersion": ("3.13", 1), + "SysArchitecture": ("64bit", 1), + "DisplayName": ("Python 3.13 (64-bit, freethreaded)", 1), + }, + 78703633: { + "ExecutablePath": ("C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python313\\python3.13t.exe", 1), + "ExecutableArguments": OSError(2, "The system cannot find the file specified"), + }, 78703088: {"SysVersion": (2778, 11)}, 78703136: { "SysVersion": OSError(2, "The system cannot find the file specified"), "SysArchitecture": OSError(2, "The system cannot find the file specified"), + "DisplayName": OSError(2, "The system cannot find the file specified"), }, 78700912: { "ExecutablePath": OSError(2, "The system cannot find the file specified"), @@ -110,6 +128,7 @@ 78704032: { "SysVersion": OSError(2, "The system cannot find the file specified"), "SysArchitecture": OSError(2, "The system cannot find the file specified"), + "DisplayName": OSError(2, "The system cannot find the file specified"), }, 78703648: { "ExecutablePath": OSError(2, "The system cannot find the file specified"), @@ -120,7 +139,11 @@ "ExecutablePath": ("Z:\\CompanyA\\Python\\3.6\\python.exe", 1), "ExecutableArguments": OSError(2, "The system cannot find the file specified"), }, - 88820000: {"SysVersion": ("3.6", 1), "SysArchitecture": ("64bit", 1)}, + 88820000: { + "SysVersion": ("3.6", 1), + "SysArchitecture": ("64bit", 1), + "DisplayName": OSError(2, "The system cannot find the file specified"), + }, } enum_collect = { 78701856: [ @@ -139,6 +162,7 @@ "3.10-32", "3.11", "3.12", + "3.13t", "3.X", OSError(22, "No more data is available", None, 259, None), ], diff --git a/tests/unit/seed/embed/test_base_embed.py b/tests/unit/seed/embed/test_base_embed.py index 255e03177..4f4fadac9 100644 --- a/tests/unit/seed/embed/test_base_embed.py +++ b/tests/unit/seed/embed/test_base_embed.py @@ -20,11 +20,46 @@ def test_download_cli_flag(args, download, tmp_path): assert session.seeder.download is download +@pytest.mark.skipif(sys.version_info[:2] == (3, 8), reason="We still bundle wheel for Python 3.8") +@pytest.mark.parametrize("flag", ["--no-wheel", "--wheel=none", "--wheel=embed", "--wheel=bundle"]) +def test_wheel_cli_flags_do_nothing(tmp_path, flag): + session = session_via_cli([flag, str(tmp_path)]) + if sys.version_info[:2] >= (3, 12): + expected = {"pip": "bundle"} + else: + expected = {"pip": "bundle", "setuptools": "bundle"} + assert session.seeder.distribution_to_versions() == expected + + +@pytest.mark.skipif(sys.version_info[:2] == (3, 8), reason="We still bundle wheel for Python 3.8") +@pytest.mark.parametrize("flag", ["--no-wheel", "--wheel=none", "--wheel=embed", "--wheel=bundle"]) +def test_wheel_cli_flags_warn(tmp_path, flag, capsys): + session_via_cli([flag, str(tmp_path)]) + out, err = capsys.readouterr() + assert "The --no-wheel and --wheel options are deprecated." in out + err + + +@pytest.mark.skipif(sys.version_info[:2] == (3, 8), reason="We still bundle wheel for Python 3.8") +def test_unused_wheel_cli_flags_dont_warn(tmp_path, capsys): + session_via_cli([str(tmp_path)]) + out, err = capsys.readouterr() + assert "The --no-wheel and --wheel options are deprecated." not in out + err + + +@pytest.mark.skipif(sys.version_info[:2] != (3, 8), reason="We only bundle wheel for Python 3.8") +@pytest.mark.parametrize("flag", ["--no-wheel", "--wheel=none", "--wheel=embed", "--wheel=bundle"]) +def test_wheel_cli_flags_dont_warn_on_38(tmp_path, flag, capsys): + session_via_cli([flag, str(tmp_path)]) + out, err = capsys.readouterr() + assert "The --no-wheel and --wheel options are deprecated." not in out + err + + def test_embed_wheel_versions(tmp_path: Path) -> None: session = session_via_cli([str(tmp_path)]) - expected = ( - {"pip": "bundle"} - if sys.version_info[:2] >= (3, 12) - else {"pip": "bundle", "setuptools": "bundle", "wheel": "bundle"} - ) + if sys.version_info[:2] >= (3, 12): + expected = {"pip": "bundle"} + elif sys.version_info[:2] >= (3, 9): + expected = {"pip": "bundle", "setuptools": "bundle"} + else: + expected = {"pip": "bundle", "setuptools": "bundle", "wheel": "bundle"} assert session.seeder.distribution_to_versions() == expected diff --git a/tests/unit/seed/embed/test_bootstrap_link_via_app_data.py b/tests/unit/seed/embed/test_bootstrap_link_via_app_data.py index 7db52e1f1..b37b7cdbb 100644 --- a/tests/unit/seed/embed/test_bootstrap_link_via_app_data.py +++ b/tests/unit/seed/embed/test_bootstrap_link_via_app_data.py @@ -10,6 +10,7 @@ import pytest +from virtualenv.cache import FileCache from virtualenv.discovery import cached_py_info from virtualenv.discovery.py_info import PythonInfo from virtualenv.info import fs_supports_symlink @@ -23,10 +24,18 @@ from pytest_mock import MockerFixture -@pytest.mark.slow() +@pytest.mark.slow @pytest.mark.parametrize("copies", [False, True] if fs_supports_symlink() else [True]) -def test_seed_link_via_app_data(tmp_path, coverage_env, current_fastest, copies): - current = PythonInfo.current_system() +def test_seed_link_via_app_data( # noqa: PLR0913, PLR0915 + tmp_path, + coverage_env, + current_fastest, + copies, + for_py_version, + session_app_data, +): + cache = FileCache(session_app_data.py_info, session_app_data.py_info_clear) + current = PythonInfo.current_system(session_app_data, cache) bundle_ver = BUNDLE_SUPPORT[current.version_release_str] create_cmd = [ str(tmp_path / "en v"), # space in the name to ensure generated scripts work when path has space @@ -45,6 +54,8 @@ def test_seed_link_via_app_data(tmp_path, coverage_env, current_fastest, copies) current_fastest, "-vv", ] + if for_py_version == "3.8": + create_cmd += ["--wheel", bundle_ver["wheel"].split("-")[1]] if not copies: create_cmd.append("--symlink-app-data") result = cli_run(create_cmd) @@ -109,7 +120,7 @@ def test_seed_link_via_app_data(tmp_path, coverage_env, current_fastest, copies) # Windows does not allow removing a executable while running it, so when uninstalling pip we need to do it via # python -m pip - remove_cmd = [str(result.creator.exe), "-m", "pip"] + remove_cmd[1:] + remove_cmd = [str(result.creator.exe), "-m", "pip", *remove_cmd[1:]] process = Popen([*remove_cmd, "pip", "wheel"]) _, __ = process.communicate() assert not process.returncode @@ -140,13 +151,14 @@ def read_only_dir(d): os.chmod(name, os.stat(name).st_mode | write) -@pytest.fixture() +@pytest.fixture def read_only_app_data(temp_app_data): temp_app_data.mkdir() with read_only_dir(temp_app_data): yield temp_app_data +@pytest.mark.slow @pytest.mark.skipif(sys.platform == "win32", reason="Windows only applies R/O to files") @pytest.mark.usefixtures("read_only_app_data") def test_base_bootstrap_link_via_app_data_not_writable(tmp_path, current_fastest): @@ -155,6 +167,7 @@ def test_base_bootstrap_link_via_app_data_not_writable(tmp_path, current_fastest assert result +@pytest.mark.slow @pytest.mark.skipif(sys.platform == "win32", reason="Windows only applies R/O to files") def test_populated_read_only_cache_and_symlinked_app_data(tmp_path, current_fastest, temp_app_data): dest = tmp_path / "venv" @@ -180,6 +193,7 @@ def test_populated_read_only_cache_and_symlinked_app_data(tmp_path, current_fast check_call((str(dest.joinpath("bin/python")), "-c", "import pip")) +@pytest.mark.slow @pytest.mark.skipif(sys.platform == "win32", reason="Windows only applies R/O to files") def test_populated_read_only_cache_and_copied_app_data(tmp_path, current_fastest, temp_app_data): dest = tmp_path / "venv" @@ -204,14 +218,21 @@ def test_populated_read_only_cache_and_copied_app_data(tmp_path, current_fastest assert cli_run(["--read-only-app-data", *cmd]) -@pytest.mark.slow() +@pytest.mark.slow @pytest.mark.parametrize("pkg", ["pip", "setuptools", "wheel"]) @pytest.mark.usefixtures("session_app_data", "current_fastest", "coverage_env") -def test_base_bootstrap_link_via_app_data_no(tmp_path, pkg): - create_cmd = [str(tmp_path), "--seeder", "app-data", f"--no-{pkg}", "--wheel", "bundle", "--setuptools", "bundle"] +def test_base_bootstrap_link_via_app_data_no(tmp_path, pkg, for_py_version): + if for_py_version != "3.8" and pkg == "wheel": + msg = "wheel isn't installed on Python > 3.8" + raise pytest.skip(msg) + create_cmd = [str(tmp_path), "--seeder", "app-data", f"--no-{pkg}", "--setuptools", "bundle"] + if for_py_version == "3.8": + create_cmd += ["--wheel", "bundle"] result = cli_run(create_cmd) assert not (result.creator.purelib / pkg).exists() for key in {"pip", "setuptools", "wheel"} - {pkg}: + if for_py_version != "3.8" and key == "wheel": + continue assert (result.creator.purelib / key).exists() @@ -227,7 +248,7 @@ def test_app_data_parallel_fail(tmp_path: Path, mocker: MockerFixture) -> None: exceptions = _run_parallel_threads(tmp_path) assert len(exceptions) == 2 for exception in exceptions: - assert exception.startswith("failed to build image wheel because:\nTraceback") + assert exception.startswith("failed to build image pip because:\nTraceback") assert "RuntimeError" in exception, exception @@ -236,7 +257,10 @@ def _run_parallel_threads(tmp_path): def _run(name): try: - cli_run(["--seeder", "app-data", str(tmp_path / name), "--no-pip", "--no-setuptools", "--wheel", "bundle"]) + cmd = ["--seeder", "app-data", str(tmp_path / name), "--no-setuptools"] + if sys.version_info[:2] == (3, 8): + cmd.append("--no-wheel") + cli_run(cmd) except Exception as exception: # noqa: BLE001 as_str = str(exception) exceptions.append(as_str) diff --git a/tests/unit/seed/embed/test_pip_invoke.py b/tests/unit/seed/embed/test_pip_invoke.py index b12d85a5d..41f4395d2 100644 --- a/tests/unit/seed/embed/test_pip_invoke.py +++ b/tests/unit/seed/embed/test_pip_invoke.py @@ -12,7 +12,7 @@ from virtualenv.seed.wheels.embed import BUNDLE_FOLDER, BUNDLE_SUPPORT -@pytest.mark.slow() +@pytest.mark.slow @pytest.mark.parametrize("no", ["pip", "setuptools", "wheel", ""]) def test_base_bootstrap_via_pip_invoke(tmp_path, coverage_env, mocker, current_fastest, no): # noqa: C901 extra_search_dir = tmp_path / "extra" @@ -25,7 +25,7 @@ def test_base_bootstrap_via_pip_invoke(tmp_path, coverage_env, mocker, current_f def _load_embed_wheel(app_data, distribution, for_py_version, version): # noqa: ARG001 return load_embed_wheel(app_data, distribution, old_ver, version) - old_ver = "3.7" + old_ver = "3.8" old = BUNDLE_SUPPORT[old_ver] mocker.patch("virtualenv.seed.wheels.bundle.load_embed_wheel", side_effect=_load_embed_wheel) @@ -39,7 +39,7 @@ def _execute(cmd, env): else: expected.add(extra_search_dir) expected_list = list( - itertools.chain.from_iterable(["--find-links", str(e)] for e in sorted(expected, key=lambda x: str(x))), + itertools.chain.from_iterable(["--find-links", str(e)] for e in sorted(expected, key=str)), ) found = cmd[-len(expected_list) :] if expected_list else [] assert "--no-index" not in cmd @@ -49,7 +49,9 @@ def _execute(cmd, env): original = PipInvoke._execute # noqa: SLF001 run = mocker.patch.object(PipInvoke, "_execute", side_effect=_execute) - versions = {"pip": "embed", "setuptools": "bundle", "wheel": new["wheel"].split("-")[1]} + versions = {"pip": "embed", "setuptools": "bundle"} + if sys.version_info[:2] == (3, 8): + versions["wheel"] = new["wheel"].split("-")[1] create_cmd = [ "--seeder", @@ -86,4 +88,6 @@ def _execute(cmd, env): for key in ("pip", "setuptools", "wheel"): if key == no: continue + if sys.version_info[:2] >= (3, 9) and key == "wheel": + continue assert locals()[key] in files_post_first_create diff --git a/tests/unit/seed/wheels/test_acquire.py b/tests/unit/seed/wheels/test_acquire.py index caa6c1b84..27b013d38 100644 --- a/tests/unit/seed/wheels/test_acquire.py +++ b/tests/unit/seed/wheels/test_acquire.py @@ -5,7 +5,7 @@ from datetime import datetime, timezone from pathlib import Path from subprocess import CalledProcessError -from typing import TYPE_CHECKING, Callable +from typing import TYPE_CHECKING import pytest @@ -16,6 +16,7 @@ from virtualenv.seed.wheels.util import Wheel, discover_wheels if TYPE_CHECKING: + from collections.abc import Callable from unittest.mock import MagicMock from pytest_mock import MockerFixture @@ -88,7 +89,19 @@ def test_download_fails(mocker, for_py_version, session_app_data): ] == exc.cmd -@pytest.fixture() +def test_download_wheel_python_io_encoding(mocker, for_py_version, session_app_data): + mock_popen = mocker.patch("virtualenv.seed.wheels.acquire.Popen") + mock_popen.return_value.communicate.return_value = "Saved a-b-c.whl", "" + mock_popen.return_value.returncode = 0 + mocker.patch("pathlib.Path.absolute", return_value=Path("a-b-c.whl")) + + download_wheel("pip", "==1", for_py_version, [], session_app_data, "folder", os.environ.copy()) + + env = mock_popen.call_args[1]["env"] + assert env["PYTHONIOENCODING"] == "utf-8" + + +@pytest.fixture def downloaded_wheel(mocker): wheel = Wheel.from_path(Path("setuptools-0.0.0-py2.py3-none-any.whl")) return wheel, mocker.patch("virtualenv.seed.wheels.acquire.download_wheel", return_value=wheel) @@ -127,7 +140,7 @@ def test_get_wheel_download_cached( time_freeze: Callable[[datetime], None], ) -> None: time_freeze(datetime.now(tz=timezone.utc)) - from virtualenv.app_data.via_disk_folder import JSONStoreDisk + from virtualenv.app_data.via_disk_folder import JSONStoreDisk # noqa: PLC0415 app_data = AppDataDiskFolder(folder=str(tmp_path)) expected = downloaded_wheel[0] diff --git a/tests/unit/seed/wheels/test_periodic_update.py b/tests/unit/seed/wheels/test_periodic_update.py index 85756d93b..e11fe5e1a 100644 --- a/tests/unit/seed/wheels/test_periodic_update.py +++ b/tests/unit/seed/wheels/test_periodic_update.py @@ -35,7 +35,7 @@ @pytest.fixture(autouse=True) def _clear_pypi_info_cache(): - from virtualenv.seed.wheels.periodic_update import _PYPI_CACHE + from virtualenv.seed.wheels.periodic_update import _PYPI_CACHE # noqa: PLC0415 _PYPI_CACHE.clear() @@ -74,7 +74,7 @@ def _do_update( # noqa: PLR0913 packages[args[1]["distribution"]].append(args[1]["for_py_version"]) packages = {key: sorted(value) for key, value in packages.items()} versions = sorted(BUNDLE_SUPPORT.keys()) - expected = {"setuptools": versions, "wheel": versions, "pip": versions} + expected = {"setuptools": versions, "wheel": ["3.8"], "pip": versions} assert packages == expected @@ -101,8 +101,6 @@ def test_pick_periodic_update(tmp_path, mocker, for_py_version): "--no-pip", "--setuptools", "bundle", - "--wheel", - "bundle", ], ) @@ -280,6 +278,7 @@ def test_trigger_update_no_debug(for_py_version, session_app_data, tmp_path, moc monkeypatch.delenv("_VIRTUALENV_PERIODIC_UPDATE_INLINE", raising=False) current = get_embed_wheel("setuptools", for_py_version) process = mocker.MagicMock() + process.pid = 123 process.communicate.return_value = None, None Popen = mocker.patch("virtualenv.seed.wheels.periodic_update.Popen", return_value=process) # noqa: N806 @@ -328,6 +327,7 @@ def test_trigger_update_debug(for_py_version, session_app_data, tmp_path, mocker current = get_embed_wheel("pip", for_py_version) process = mocker.MagicMock() + process.pid = 123 process.communicate.return_value = None, None Popen = mocker.patch("virtualenv.seed.wheels.periodic_update.Popen", return_value=process) # noqa: N806 diff --git a/tests/unit/seed/wheels/test_wheels_util.py b/tests/unit/seed/wheels/test_wheels_util.py index 87edab2d4..4d4ec9b9b 100644 --- a/tests/unit/seed/wheels/test_wheels_util.py +++ b/tests/unit/seed/wheels/test_wheels_util.py @@ -29,3 +29,8 @@ def test_wheel_not_support(): def test_wheel_repr(): wheel = get_embed_wheel("setuptools", MAX) assert str(wheel.path) in repr(wheel) + + +def test_unknown_distribution(): + wheel = get_embed_wheel("unknown", MAX) + assert wheel is None diff --git a/tests/unit/test_file_limit.py b/tests/unit/test_file_limit.py new file mode 100644 index 000000000..0c66561f8 --- /dev/null +++ b/tests/unit/test_file_limit.py @@ -0,0 +1,53 @@ +from __future__ import annotations + +import errno +import os +import sys + +import pytest + +from virtualenv.info import IMPLEMENTATION +from virtualenv.run import cli_run + + +@pytest.mark.skipif(sys.platform == "win32", reason="resource module not available on Windows") +def test_too_many_open_files(tmp_path): + """ + Test that we get a specific error message when we have too many open files. + """ + import resource # noqa: PLC0415 + + soft_limit, hard_limit = resource.getrlimit(resource.RLIMIT_NOFILE) + + # Lower the soft limit to a small number to trigger the error + try: + resource.setrlimit(resource.RLIMIT_NOFILE, (32, hard_limit)) + except ValueError: + pytest.skip("could not lower the soft limit for open files") + except AttributeError as exc: # pypy, graalpy + if "module 'resource' has no attribute 'setrlimit'" in str(exc): + pytest.skip(f"{IMPLEMENTATION} does not support resource.setrlimit") + + # Keep some file descriptors open to make it easier to trigger the error + fds = [] + try: + # JIT implementations use more file descriptors up front so we can run out early + try: + fds.extend(os.open(os.devnull, os.O_RDONLY) for _ in range(20)) + except OSError as jit_exceptions: # pypy, graalpy + assert jit_exceptions.errno == errno.EMFILE # noqa: PT017 + assert "Too many open files" in str(jit_exceptions) # noqa: PT017 + + expected_exceptions = SystemExit, OSError, RuntimeError + with pytest.raises(expected_exceptions) as too_many_open_files_exc: + cli_run([str(tmp_path / "venv")]) + + if isinstance(too_many_open_files_exc, SystemExit): + assert too_many_open_files_exc.code != 0 + else: + assert "Too many open files" in str(too_many_open_files_exc.value) + + finally: + for fd in fds: + os.close(fd) + resource.setrlimit(resource.RLIMIT_NOFILE, (soft_limit, hard_limit)) diff --git a/tests/unit/test_run.py b/tests/unit/test_run.py index a048e60a6..b61731d57 100644 --- a/tests/unit/test_run.py +++ b/tests/unit/test_run.py @@ -27,7 +27,7 @@ def test_version(capsys): assert not err assert __version__ in content - import virtualenv + import virtualenv # noqa: PLC0415 assert virtualenv.__file__ in content diff --git a/tox.ini b/tox.ini index 5e9101031..d8aefed7c 100644 --- a/tox.ini +++ b/tox.ini @@ -1,18 +1,20 @@ [tox] requires = - tox>=4.2 + tox>=4.28 env_list = fix - py312 - py311 - py310 - py39 - py38 - py37 pypy3 + 3.13 + 3.12 + 3.11 + 3.10 + 3.9 + 3.8 + graalpy coverage readme docs + 3.13t skip_missing_interpreters = true [testenv] @@ -31,31 +33,34 @@ set_env = PYTHONWARNDEFAULTENCODING = 1 _COVERAGE_SRC = {envsitepackagesdir}/virtualenv commands = - coverage erase - coverage run -m pytest {posargs:--junitxml {toxworkdir}/junit.{envname}.xml tests --int} - coverage combine - coverage report --skip-covered --show-missing - coverage xml -o {toxworkdir}/coverage.{envname}.xml - coverage html -d {envtmpdir}/htmlcov --show-contexts --title virtualenv-{envname}-coverage + !graalpy: coverage erase + !graalpy: coverage run -m pytest {posargs:--junitxml "{toxworkdir}/junit.{envname}.xml" tests --int} + !graalpy: coverage combine + !graalpy: coverage report --skip-covered --show-missing + !graalpy: coverage xml -o "{toxworkdir}/coverage.{envname}.xml" + !graalpy: coverage html -d {envtmpdir}/htmlcov --show-contexts --title virtualenv-{envname}-coverage + graalpy: pytest {posargs:--junitxml "{toxworkdir}/junit.{envname}.xml" tests --skip-slow} +uv_seed = true [testenv:fix] description = format the code base to adhere to our styles, and complain about what we cannot do automatically skip_install = true deps = - pre-commit>=3.3.3 + pre-commit-uv>=4.1.4 commands = pre-commit run --all-files --show-diff-on-failure [testenv:readme] -description = check that the long description is valid (need for PyPI) +description = check that the long description is valid skip_install = true deps = - build>=0.10 - twine>=4.0.2 -extras = + check-wheel-contents>=0.6.2 + twine>=6.1 + uv>=0.8 commands = - python -m build -o {envtmpdir} --wheel --sdist . - twine check {envtmpdir}/* + uv build --sdist --wheel --out-dir {envtmpdir} . + twine check {envtmpdir}{/}* + check-wheel-contents --no-config {envtmpdir} [testenv:docs] description = build documentation @@ -65,23 +70,27 @@ commands = sphinx-build -d "{envtmpdir}/doctree" docs "{toxworkdir}/docs_out" --color -b html {posargs:-W} python -c 'import pathlib; print("documentation available under file://\{0\}".format(pathlib.Path(r"{toxworkdir}") / "docs_out" / "index.html"))' +[testenv:3.13t] +base_python = {env:TOX_BASEPYTHON} + [testenv:upgrade] description = upgrade pip/wheels/setuptools to latest skip_install = true deps = - black>=23.7 + ruff>=0.12.4 pass_env = UPGRADE_ADVISORY change_dir = {toxinidir}/tasks commands = - python upgrade_wheels.py + - python upgrade_wheels.py +uv_seed = true [testenv:release] description = do a release, required posarg of the version number deps = - gitpython>=3.1.32 - packaging>=23.1 - towncrier>=23.6 + gitpython>=3.1.44 + packaging>=25 + towncrier>=24.8 change_dir = {toxinidir}/tasks commands = python release.py --version {posargs} @@ -93,13 +102,14 @@ extras = docs test commands = - python -m pip list --format=columns + uv pip tree python -c 'import sys; print(sys.executable)' [testenv:zipapp] description = generate a zipapp skip_install = true deps = - packaging>=23.1 + packaging>=25 commands = python tasks/make_zipapp.py +uv_seed = true diff --git a/whitelist.txt b/whitelist.txt deleted file mode 100644 index be9ad5c0a..000000000 --- a/whitelist.txt +++ /dev/null @@ -1,230 +0,0 @@ -1st -2nd -32key -64key -abi -addoption -addsitedir -altsep -appauthor -appname -arcname -argtypes -arm64 -attrgetter -autoclass -autodoc -autosectionlabel -autouse -b64 -bak -buf -bzrignore -calc -caplog -capsys -cfg -changelog -chdir -checkwindow -cigam -classs -codecs -codesign -colspec -commonprefix -conda -confs -confstr -copy2 -copytree -cov -cpython -cpython2 -cpython3 -cpython3windows -crc -crc32 -csh -cshell -cust -defpath -delenv -deque -devnull -distlib -distro -dll -docname -doctree -doctreedir -docutils -downloader -dword -dylib -endian -enquote -epilog -exe -executables -extlinks -extractall -filelock -fillvalue -filterby -fixturenames -fixup -fromlist -fs -gc -getattribute -geteuid -getfilesystemencoding -getinfo -getitem -getoption -getplugin -getsitepackages -gevent -groupby -hgignore -hkey -img -impls -iread -irgrp -iroth -ish -iterdir -iwgrp -iwoth -iwusr -ixgrp -ixoth -ixusr -joinpath -kwonly -ld -levelno -libffi -libpypy -libpypy3 -libs -lpcwstr -lpwstr -lstat -makefile -maxint -methodcaller -mh -mktemp -modifyitems -morecols -namelist -nfat -nodot -noinspection -nok -nonwrappers -normcase -notset -nul -nushell -onerror -pardir -pathlist -pep514 -pep8 -platformdirs -platlib -pluginmanager -popleft -pos -posix -powershell -prefered -preimport -pseudorandom -pth -purelib -py27 -pyc -pyd -pydoc -pyenv -pygments -pyinfo -pypy -pypy2 -pypy3 -pypy37 -pypy38 -python2mac -python3mac -pythonmac -pythonpath -pythonw -pythonx -pyvenv -pyver -rawtext -readlink -readouterr -reentrant -refid -refspec -releaselevel -repo -restype -rfind -rglob -rmdir -rpartition -rst -scr -sdist -setenv -shlex -simplefilter -sitepackages -skipif -sphinxarg -srcs -stdlib -strerror -stringify -subdirectory -subdirs -subinclude -submodules -symlinked -symlinks -sysconfig -t1 -t2 -tbody -tcl -tempdir -testpth -tgroup -tk -topdown -towncrier -tox -truediv -unsecure -usefixtures -v37 -vcs -venv -venvlauncher -virtualenv -virtualenvs -whl -winapi -winreg -wintypes -wow64 -xfail -zipapp -zipimporter