diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 07c20d8d8..6eabb999f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,18 +6,18 @@ on: - master pull_request: schedule: - # Run at 18:00 UTC every day - - cron: '0 18 * * *' + # Run multiple times a day as the successfull cached links are not checked every time. + - cron: '0 */8 * * *' jobs: ci: if: github.repository == 'rust-lang/rustc-dev-guide' runs-on: ubuntu-latest env: - MDBOOK_VERSION: 0.4.21 - MDBOOK_LINKCHECK2_VERSION: 0.8.1 + MDBOOK_VERSION: 0.4.48 + MDBOOK_LINKCHECK2_VERSION: 0.9.1 MDBOOK_MERMAID_VERSION: 0.12.6 - MDBOOK_TOC_VERSION: 0.11.2 + MDBOOK_OUTPUT__LINKCHECK__FOLLOW_WEB_LINKS: ${{ github.event_name != 'pull_request' }} DEPLOY_DIR: book/html BASE_SHA: ${{ github.event.pull_request.base.sha }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -33,14 +33,17 @@ jobs: with: path: | ~/.cargo/bin - key: ${{ runner.os }}-${{ env.MDBOOK_VERSION }}--${{ env.MDBOOK_LINKCHECK2_VERSION }}--${{ env.MDBOOK_TOC_VERSION }}--${{ env.MDBOOK_MERMAID_VERSION }} + key: ${{ runner.os }}-${{ env.MDBOOK_VERSION }}--${{ env.MDBOOK_LINKCHECK2_VERSION }}--${{ env.MDBOOK_MERMAID_VERSION }} - - name: Cache linkcheck - uses: actions/cache@v4 + - name: Restore cached Linkcheck + if: github.event_name == 'schedule' + id: cache-linkcheck-restore + uses: actions/cache/restore@v4 with: - path: | - ~/book/linkcheck - key: ${{ runner.os }}-${{ hashFiles('./book/linkcheck') }} + path: book/linkcheck/cache.json + key: linkcheck--${{ env.MDBOOK_LINKCHECK2_VERSION }}--${{ github.run_id }} + restore-keys: | + linkcheck--${{ env.MDBOOK_LINKCHECK2_VERSION }}-- - name: Install latest nightly Rust toolchain if: steps.mdbook-cache.outputs.cache-hit != 'true' @@ -53,12 +56,19 @@ jobs: run: | cargo install mdbook --version ${{ env.MDBOOK_VERSION }} cargo install mdbook-linkcheck2 --version ${{ env.MDBOOK_LINKCHECK2_VERSION }} - cargo install mdbook-toc --version ${{ env.MDBOOK_TOC_VERSION }} cargo install mdbook-mermaid --version ${{ env.MDBOOK_MERMAID_VERSION }} - name: Check build run: ENABLE_LINKCHECK=1 mdbook build + - name: Save cached Linkcheck + id: cache-linkcheck-save + if: ${{ !cancelled() && github.event_name == 'schedule' }} + uses: actions/cache/save@v4 + with: + path: book/linkcheck/cache.json + key: linkcheck--${{ env.MDBOOK_LINKCHECK2_VERSION }}--${{ github.run_id }} + - name: Deploy to gh-pages if: github.event_name == 'push' run: | diff --git a/.github/workflows/rustc-pull.yml b/.github/workflows/rustc-pull.yml new file mode 100644 index 000000000..5ff311896 --- /dev/null +++ b/.github/workflows/rustc-pull.yml @@ -0,0 +1,21 @@ +name: rustc-pull + +on: + workflow_dispatch: + schedule: + # Run at 04:00 UTC every Monday + - cron: '0 4 * * 1' + +jobs: + pull: + if: github.repository == 'rust-lang/rustc-dev-guide' + uses: rust-lang/josh-sync/.github/workflows/rustc-pull.yml@main + with: + github-app-id: ${{ vars.APP_CLIENT_ID }} + zulip-stream-id: 196385 + zulip-bot-email: "rustc-dev-guide-gha-notif-bot@rust-lang.zulipchat.com" + pr-base-branch: master + branch-name: rustc-pull + secrets: + zulip-api-token: ${{ secrets.ZULIP_API_TOKEN }} + github-app-secret: ${{ secrets.APP_PRIVATE_KEY }} diff --git a/.mailmap b/.mailmap index 1a1f6ffb6..907495ed1 100644 --- a/.mailmap +++ b/.mailmap @@ -3,3 +3,4 @@ Jynn Nelson Jynn Nelson Jynn Nelson Jynn Nelson +Tshepang Mbambo diff --git a/README.md b/README.md index 7ce7d4ddf..1ad895aed 100644 --- a/README.md +++ b/README.md @@ -43,13 +43,13 @@ rustdocs][rustdocs]. To build a local static HTML site, install [`mdbook`](https://github.com/rust-lang/mdBook) with: ``` -> cargo install mdbook mdbook-linkcheck2 mdbook-toc mdbook-mermaid +cargo install mdbook mdbook-linkcheck2 mdbook-mermaid ``` and execute the following command in the root of the repository: ``` -> mdbook build --open +mdbook build --open ``` The build files are found in the `book/html` directory. @@ -61,30 +61,17 @@ checking is **not** run by default locally, though it is in CI. To enable it locally, set the environment variable `ENABLE_LINKCHECK=1` like in the following example. -```console -$ ENABLE_LINKCHECK=1 mdbook serve +``` +ENABLE_LINKCHECK=1 mdbook serve ``` ### Table of Contents -We use `mdbook-toc` to auto-generate TOCs for long sections. You can invoke the preprocessor by -including the `` marker at the place where you want the TOC. +Each page has a TOC that is automatically generated by `pagetoc.js`. +There is an associated `pagetoc.css`, for styling. ## Synchronizing josh subtree with rustc -This repository is linked to `rust-lang/rust` as a [josh](https://josh-project.github.io/josh/intro.html) subtree. You can use the following commands to synchronize the subtree in both directions. - -### Pull changes from `rust-lang/rust` into this repository -1) Checkout a new branch that will be used to create a PR into `rust-lang/rustc-dev-guide` -2) Run the pull command - ``` - $ cargo run --manifest-path josh-sync/Cargo.toml rustc-pull - ``` -3) Push the branch to your fork and create a PR into `rustc-dev-guide` - -### Push changes from this repository into `rust-lang/rust` -1) Run the push command to create a branch named `` in a `rustc` fork under the `` account - ``` - $ cargo run --manifest-path josh-sync/Cargo.toml rustc-push - ``` -2) Create a PR from `` into `rust-lang/rust` +This repository is linked to `rust-lang/rust` as a [josh](https://josh-project.github.io/josh/intro.html) subtree. You can use the [rustc-josh-sync](https://github.com/rust-lang/josh-sync) tool to perform synchronization. + +You can find a guide on how to perform the synchronization [here](./src/external-repos.md#synchronizing-a-josh-subtree). diff --git a/book.toml b/book.toml index 2f67aecf6..daf237ed9 100644 --- a/book.toml +++ b/book.toml @@ -1,22 +1,23 @@ [book] title = "Rust Compiler Development Guide" -author = "The Rust Project Developers" +authors = ["The Rust Project Developers"] description = "A guide to developing the Rust compiler (rustc)" [build] create-missing = false -[preprocessor.toc] -command = "mdbook-toc" -renderer = ["html"] - [preprocessor.mermaid] command = "mdbook-mermaid" [output.html] git-repository-url = "https://github.com/rust-lang/rustc-dev-guide" edit-url-template = "https://github.com/rust-lang/rustc-dev-guide/edit/master/{path}" -additional-js = ["mermaid.min.js", "mermaid-init.js"] +additional-js = [ + "mermaid.min.js", + "mermaid-init.js", + "pagetoc.js", +] +additional-css = ["pagetoc.css"] [output.html.search] use-boolean-and = true @@ -52,7 +53,9 @@ exclude = [ # 500 is returned for HEAD request "code\\.visualstudio\\.com/docs/editor/tasks", ] -cache-timeout = 86400 +# The scheduled CI runs every day and so we need to reuse a part of the cache +# in order to face "Server returned 429 Too Many Requests" errors for github.com. +cache-timeout = 90000 warning-policy = "error" [output.html.redirect] @@ -60,5 +63,7 @@ warning-policy = "error" "/diagnostics/sessiondiagnostic.html" = "diagnostic-structs.html" "/diagnostics/diagnostic-codes.html" = "error-codes.html" "/miri.html" = "const-eval/interpret.html" -"/tests/integration.html" = "ecosystem.html" +"/tests/fuchsia.html" = "ecosystem-test-jobs/fuchsia.html" "/tests/headers.html" = "directives.html" +"/tests/integration.html" = "ecosystem.html" +"/tests/rust-for-linux.html" = "ecosystem-test-jobs/rust-for-linux.html" diff --git a/ci/date-check/Cargo.lock b/ci/date-check/Cargo.lock index 6326b2daf..710754733 100644 --- a/ci/date-check/Cargo.lock +++ b/ci/date-check/Cargo.lock @@ -1,6 +1,6 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. -version = 3 +version = 4 [[package]] name = "aho-corasick" @@ -28,21 +28,24 @@ dependencies = [ [[package]] name = "autocfg" -version = "1.3.0" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" +checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" [[package]] name = "bumpalo" -version = "3.16.0" +version = "3.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" +checksum = "1628fb46dfa0b37568d12e5edd512553eccf6a22a78e8bde00bb4aed84d5bdbf" [[package]] name = "cc" -version = "1.0.106" +version = "1.2.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "066fce287b1d4eafef758e89e09d724a24808a9196fe9756b8ca90e86d0719a2" +checksum = "32db95edf998450acc7881c932f94cd9b05c87b4b2599e8bab064753da4acfd1" +dependencies = [ + "shlex", +] [[package]] name = "cfg-if" @@ -52,27 +55,27 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "chrono" -version = "0.4.38" +version = "0.4.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a21f936df1771bf62b77f047b726c4625ff2e8aa607c01ec06e5a05bd8463401" +checksum = "c469d952047f47f91b68d1cba3f10d63c11d73e4636f24f08daf0278abf01c4d" dependencies = [ "android-tzdata", "iana-time-zone", "js-sys", "num-traits", "wasm-bindgen", - "windows-targets", + "windows-link", ] [[package]] name = "core-foundation-sys" -version = "0.8.6" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" [[package]] name = "date-check" -version = "0.1.0" +version = "0.0.0" dependencies = [ "chrono", "glob", @@ -81,20 +84,21 @@ dependencies = [ [[package]] name = "glob" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" +checksum = "a8d1add55171497b4705a648c6b583acafb01d58050a51727785f0b2c8e0a2b2" [[package]] name = "iana-time-zone" -version = "0.1.60" +version = "0.1.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7ffbb5a1b541ea2561f8c41c087286cc091e21e556a4f09a8f6cbf17b69b141" +checksum = "b0c919e5debc312ad217002b8048a17b7d83f80703865bbfcfebb0458b0b27d8" dependencies = [ "android_system_properties", "core-foundation-sys", "iana-time-zone-haiku", "js-sys", + "log", "wasm-bindgen", "windows-core", ] @@ -110,24 +114,25 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.69" +version = "0.3.77" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29c15563dc2726973df627357ce0c9ddddbea194836909d655df6a75d2cf296d" +checksum = "1cfaf33c695fc6e08064efbc1f72ec937429614f25eef83af942d0e227c3a28f" dependencies = [ + "once_cell", "wasm-bindgen", ] [[package]] name = "libc" -version = "0.2.155" +version = "0.2.172" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c" +checksum = "d750af042f7ef4f724306de029d18836c26c1765a54a6a3f094cbd23a7267ffa" [[package]] name = "log" -version = "0.4.22" +version = "0.4.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" +checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94" [[package]] name = "memchr" @@ -146,33 +151,33 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.19.0" +version = "1.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" +checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" [[package]] name = "proc-macro2" -version = "1.0.86" +version = "1.0.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77" +checksum = "02b3e5e68a3a1a02aad3ec490a98007cbc13c37cbe84a3cd7b8e406d76e7f778" dependencies = [ "unicode-ident", ] [[package]] name = "quote" -version = "1.0.36" +version = "1.0.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7" +checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d" dependencies = [ "proc-macro2", ] [[package]] name = "regex" -version = "1.10.5" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b91213439dad192326a0d7c6ee3955910425f441d7038e0d6933b0aec5c4517f" +checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" dependencies = [ "aho-corasick", "memchr", @@ -182,9 +187,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.7" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38caf58cc5ef2fed281f89292ef23f6365465ed9a41b7a7754eb4e26496c92df" +checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908" dependencies = [ "aho-corasick", "memchr", @@ -193,15 +198,27 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.8.4" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b" +checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" + +[[package]] +name = "rustversion" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eded382c5f5f786b989652c49544c4877d9f015cc22e145a5ea8ea66c2921cd2" + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" [[package]] name = "syn" -version = "2.0.70" +version = "2.0.101" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f0209b68b3613b093e0ec905354eccaedcfe83b8cb37cbdeae64026c3064c16" +checksum = "8ce2b7fc941b3a24138a0a7cf8e858bfc6a992e7978a068a5c760deb0ed43caf" dependencies = [ "proc-macro2", "quote", @@ -210,29 +227,30 @@ dependencies = [ [[package]] name = "unicode-ident" -version = "1.0.12" +version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" +checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512" [[package]] name = "wasm-bindgen" -version = "0.2.92" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4be2531df63900aeb2bca0daaaddec08491ee64ceecbee5076636a3b026795a8" +checksum = "1edc8929d7499fc4e8f0be2262a241556cfc54a0bea223790e71446f2aab1ef5" dependencies = [ "cfg-if", + "once_cell", + "rustversion", "wasm-bindgen-macro", ] [[package]] name = "wasm-bindgen-backend" -version = "0.2.92" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "614d787b966d3989fa7bb98a654e369c762374fd3213d212cfc0251257e747da" +checksum = "2f0a0651a5c2bc21487bde11ee802ccaf4c51935d0d3d42a6101f98161700bc6" dependencies = [ "bumpalo", "log", - "once_cell", "proc-macro2", "quote", "syn", @@ -241,9 +259,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.92" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1f8823de937b71b9460c0c34e25f3da88250760bec0ebac694b49997550d726" +checksum = "7fe63fc6d09ed3792bd0897b314f53de8e16568c2b3f7982f468c0bf9bd0b407" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -251,9 +269,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.92" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" +checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" dependencies = [ "proc-macro2", "quote", @@ -264,79 +282,68 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.92" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96" +checksum = "1a05d73b933a847d6cccdda8f838a22ff101ad9bf93e33684f39c1f5f0eece3d" +dependencies = [ + "unicode-ident", +] [[package]] name = "windows-core" -version = "0.52.0" +version = "0.61.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" +checksum = "4763c1de310c86d75a878046489e2e5ba02c649d185f21c67d4cf8a56d098980" dependencies = [ - "windows-targets", + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", ] [[package]] -name = "windows-targets" -version = "0.52.6" +name = "windows-implement" +version = "0.60.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +checksum = "a47fddd13af08290e67f4acabf4b459f647552718f683a7b415d290ac744a836" dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_gnullvm", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", + "proc-macro2", + "quote", + "syn", ] [[package]] -name = "windows_aarch64_gnullvm" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" - -[[package]] -name = "windows_aarch64_msvc" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" - -[[package]] -name = "windows_i686_gnu" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" - -[[package]] -name = "windows_i686_gnullvm" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" - -[[package]] -name = "windows_i686_msvc" -version = "0.52.6" +name = "windows-interface" +version = "0.59.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" +checksum = "bd9211b69f8dcdfa817bfd14bf1c97c9188afa36f4750130fcdf3f400eca9fa8" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] [[package]] -name = "windows_x86_64_gnu" -version = "0.52.6" +name = "windows-link" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" +checksum = "76840935b766e1b0a05c0066835fb9ec80071d4c09a16f6bd5f7e655e3c14c38" [[package]] -name = "windows_x86_64_gnullvm" -version = "0.52.6" +name = "windows-result" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" +checksum = "c64fd11a4fd95df68efcfee5f44a294fe71b8bc6a91993e2791938abcc712252" +dependencies = [ + "windows-link", +] [[package]] -name = "windows_x86_64_msvc" -version = "0.52.6" +name = "windows-strings" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +checksum = "7a2ba9642430ee452d5a7aa78d72907ebe8cfda358e8cb7918a2050581322f97" +dependencies = [ + "windows-link", +] diff --git a/ci/date-check/Cargo.toml b/ci/date-check/Cargo.toml index 472529511..f49e6d0db 100644 --- a/ci/date-check/Cargo.toml +++ b/ci/date-check/Cargo.toml @@ -1,10 +1,6 @@ [package] name = "date-check" -version = "0.1.0" -authors = ["Noah Lev "] -edition = "2021" - -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html +edition = "2024" [dependencies] glob = "0.3" diff --git a/ci/date-check/src/main.rs b/ci/date-check/src/main.rs index 5ab3e6c8b..0a32f4e9b 100644 --- a/ci/date-check/src/main.rs +++ b/ci/date-check/src/main.rs @@ -1,11 +1,8 @@ -use std::{ - collections::BTreeMap, - convert::TryInto as _, - env, fmt, fs, - path::{Path, PathBuf}, - process, - str::FromStr, -}; +use std::collections::BTreeMap; +use std::convert::TryInto as _; +use std::path::{Path, PathBuf}; +use std::str::FromStr; +use std::{env, fmt, fs, process}; use chrono::{Datelike as _, Month, TimeZone as _, Utc}; use glob::glob; @@ -19,19 +16,13 @@ struct Date { impl Date { fn months_since(self, other: Date) -> Option { - let self_chrono = Utc - .with_ymd_and_hms(self.year.try_into().unwrap(), self.month, 1, 0, 0, 0) - .unwrap(); - let other_chrono = Utc - .with_ymd_and_hms(other.year.try_into().unwrap(), other.month, 1, 0, 0, 0) - .unwrap(); + let self_chrono = + Utc.with_ymd_and_hms(self.year.try_into().unwrap(), self.month, 1, 0, 0, 0).unwrap(); + let other_chrono = + Utc.with_ymd_and_hms(other.year.try_into().unwrap(), other.month, 1, 0, 0, 0).unwrap(); let duration_since = self_chrono.signed_duration_since(other_chrono); let months_since = duration_since.num_days() / 30; - if months_since < 0 { - None - } else { - Some(months_since.try_into().unwrap()) - } + if months_since < 0 { None } else { Some(months_since.try_into().unwrap()) } } } @@ -66,26 +57,18 @@ fn collect_dates_from_file(date_regex: &Regex, text: &str) -> Vec<(usize, Date)> date_regex .captures_iter(text) .filter_map(|cap| { - if let (Some(month), Some(year), None, None) | (None, None, Some(month), Some(year)) = ( - cap.name("m1"), - cap.name("y1"), - cap.name("m2"), - cap.name("y2"), - ) { + if let (Some(month), Some(year), None, None) | (None, None, Some(month), Some(year)) = + (cap.name("m1"), cap.name("y1"), cap.name("m2"), cap.name("y2")) + { let year = year.as_str().parse().expect("year"); - let month = Month::from_str(month.as_str()) - .expect("month") - .number_from_month(); + let month = Month::from_str(month.as_str()).expect("month").number_from_month(); Some((cap.get(0).expect("all").range(), Date { year, month })) } else { None } }) .map(|(byte_range, date)| { - line += text[end_of_last_cap..byte_range.end] - .chars() - .filter(|c| *c == '\n') - .count(); + line += text[end_of_last_cap..byte_range.end].chars().filter(|c| *c == '\n').count(); end_of_last_cap = byte_range.end; (line, date) }) @@ -131,17 +114,14 @@ fn filter_dates( fn main() { let mut args = env::args(); if args.len() == 1 { - eprintln!("error: expected root Markdown directory as CLI argument"); + eprintln!("error: expected root of Markdown directory as CLI argument"); process::exit(1); } let root_dir = args.nth(1).unwrap(); let root_dir_path = Path::new(&root_dir); let glob_pat = format!("{}/**/*.md", root_dir); let today_chrono = Utc::now().date_naive(); - let current_month = Date { - year: today_chrono.year_ce().1, - month: today_chrono.month(), - }; + let current_month = Date { year: today_chrono.year_ce().1, month: today_chrono.month() }; let dates_by_file = collect_dates(glob(&glob_pat).unwrap().map(Result::unwrap)); let dates_by_file: BTreeMap<_, _> = @@ -173,10 +153,7 @@ fn main() { println!(); for (path, dates) in dates_by_file { - println!( - "- {}", - path.strip_prefix(&root_dir_path).unwrap_or(&path).display(), - ); + println!("- {}", path.strip_prefix(&root_dir_path).unwrap_or(&path).display(),); for (line, date) in dates { println!(" - [ ] line {}: {}", line, date); } @@ -191,14 +168,8 @@ mod tests { #[test] fn test_months_since() { - let date1 = Date { - year: 2020, - month: 3, - }; - let date2 = Date { - year: 2021, - month: 1, - }; + let date1 = Date { year: 2020, month: 3 }; + let date2 = Date { year: 2021, month: 1 }; assert_eq!(date2.months_since(date1), Some(10)); } @@ -273,83 +244,17 @@ Test8 assert_eq!( collect_dates_from_file(&make_date_regex(), text), vec![ - ( - 3, - Date { - year: 2021, - month: 1, - } - ), - ( - 6, - Date { - year: 2021, - month: 2, - } - ), - ( - 9, - Date { - year: 2021, - month: 3, - } - ), - ( - 11, - Date { - year: 2021, - month: 4, - } - ), - ( - 17, - Date { - year: 2021, - month: 5, - } - ), - ( - 20, - Date { - year: 2021, - month: 1, - } - ), - ( - 23, - Date { - year: 2021, - month: 2, - } - ), - ( - 26, - Date { - year: 2021, - month: 3, - } - ), - ( - 28, - Date { - year: 2021, - month: 4, - } - ), - ( - 34, - Date { - year: 2021, - month: 5, - } - ), - ( - 38, - Date { - year: 2021, - month: 6, - } - ), + (3, Date { year: 2021, month: 1 }), + (6, Date { year: 2021, month: 2 }), + (9, Date { year: 2021, month: 3 }), + (11, Date { year: 2021, month: 4 }), + (17, Date { year: 2021, month: 5 }), + (20, Date { year: 2021, month: 1 }), + (23, Date { year: 2021, month: 2 }), + (26, Date { year: 2021, month: 3 }), + (28, Date { year: 2021, month: 4 }), + (34, Date { year: 2021, month: 5 }), + (38, Date { year: 2021, month: 6 }), ], ); } diff --git a/examples/README b/examples/README index ca49dd74d..05e446737 100644 --- a/examples/README +++ b/examples/README @@ -4,7 +4,10 @@ For each example to compile, you will need to first run the following: To create an executable: - rustc rustc-driver-example.rs + rustup run nightly rustc rustc-driver-example.rs + +You might need to be more specific about the exact nightly version. See the comments at the top of +the examples for the version they were written for. To run an executable: diff --git a/examples/rustc-driver-example.rs b/examples/rustc-driver-example.rs index 576bbcea9..db6ac1857 100644 --- a/examples/rustc-driver-example.rs +++ b/examples/rustc-driver-example.rs @@ -1,3 +1,5 @@ +// Tested with nightly-2025-03-28 + #![feature(rustc_private)] extern crate rustc_ast; @@ -15,11 +17,11 @@ extern crate rustc_span; use std::io; use std::path::Path; +use std::sync::Arc; use rustc_ast_pretty::pprust::item_to_string; -use rustc_data_structures::sync::Lrc; -use rustc_driver::{Compilation, RunCompiler}; -use rustc_interface::interface::Compiler; +use rustc_driver::{Compilation, run_compiler}; +use rustc_interface::interface::{Compiler, Config}; use rustc_middle::ty::TyCtxt; struct MyFileLoader; @@ -32,9 +34,9 @@ impl rustc_span::source_map::FileLoader for MyFileLoader { fn read_file(&self, path: &Path) -> io::Result { if path == Path::new("main.rs") { Ok(r#" +static MESSAGE: &str = "Hello, World!"; fn main() { - let message = "Hello, World!"; - println!("{message}"); + println!("{MESSAGE}"); } "# .to_string()) @@ -43,7 +45,7 @@ fn main() { } } - fn read_binary_file(&self, _path: &Path) -> io::Result> { + fn read_binary_file(&self, _path: &Path) -> io::Result> { Err(io::Error::other("oops")) } } @@ -51,10 +53,14 @@ fn main() { struct MyCallbacks; impl rustc_driver::Callbacks for MyCallbacks { + fn config(&mut self, config: &mut Config) { + config.file_loader = Some(Box::new(MyFileLoader)); + } + fn after_crate_root_parsing( &mut self, _compiler: &Compiler, - krate: &rustc_ast::Crate, + krate: &mut rustc_ast::Crate, ) -> Compilation { for item in &krate.items { println!("{}", item_to_string(&item)); @@ -65,14 +71,12 @@ impl rustc_driver::Callbacks for MyCallbacks { fn after_analysis(&mut self, _compiler: &Compiler, tcx: TyCtxt<'_>) -> Compilation { // Analyze the program and inspect the types of definitions. - for id in tcx.hir().items() { - let hir = tcx.hir(); - let item = hir.item(id); + for id in tcx.hir_free_items() { + let item = &tcx.hir_item(id); match item.kind { - rustc_hir::ItemKind::Static(_, _, _) | rustc_hir::ItemKind::Fn(_, _, _) => { - let name = item.ident; + rustc_hir::ItemKind::Static(ident, ..) | rustc_hir::ItemKind::Fn { ident, .. } => { let ty = tcx.type_of(item.hir_id().owner.def_id); - println!("{name:?}:\t{ty:?}") + println!("{ident:?}:\t{ty:?}") } _ => (), } @@ -83,10 +87,13 @@ impl rustc_driver::Callbacks for MyCallbacks { } fn main() { - match RunCompiler::new(&["main.rs".to_string()], &mut MyCallbacks) { - mut compiler => { - compiler.set_file_loader(Some(Box::new(MyFileLoader))); - compiler.run(); - } - } + run_compiler( + &[ + // The first argument, which in practice contains the name of the binary being executed + // (i.e. "rustc") is ignored by rustc. + "ignored".to_string(), + "main.rs".to_string(), + ], + &mut MyCallbacks, + ); } diff --git a/examples/rustc-driver-interacting-with-the-ast.rs b/examples/rustc-driver-interacting-with-the-ast.rs index 90a85d5db..c0d7f977d 100644 --- a/examples/rustc-driver-interacting-with-the-ast.rs +++ b/examples/rustc-driver-interacting-with-the-ast.rs @@ -1,3 +1,5 @@ +// Tested with nightly-2025-03-28 + #![feature(rustc_private)] extern crate rustc_ast; @@ -15,11 +17,11 @@ extern crate rustc_span; use std::io; use std::path::Path; +use std::sync::Arc; use rustc_ast_pretty::pprust::item_to_string; -use rustc_data_structures::sync::Lrc; -use rustc_driver::{Compilation, RunCompiler}; -use rustc_interface::interface::Compiler; +use rustc_driver::{Compilation, run_compiler}; +use rustc_interface::interface::{Compiler, Config}; use rustc_middle::ty::TyCtxt; struct MyFileLoader; @@ -43,7 +45,7 @@ fn main() { } } - fn read_binary_file(&self, _path: &Path) -> io::Result> { + fn read_binary_file(&self, _path: &Path) -> io::Result> { Err(io::Error::other("oops")) } } @@ -51,10 +53,14 @@ fn main() { struct MyCallbacks; impl rustc_driver::Callbacks for MyCallbacks { + fn config(&mut self, config: &mut Config) { + config.file_loader = Some(Box::new(MyFileLoader)); + } + fn after_crate_root_parsing( &mut self, _compiler: &Compiler, - krate: &rustc_ast::Crate, + krate: &mut rustc_ast::Crate, ) -> Compilation { for item in &krate.items { println!("{}", item_to_string(&item)); @@ -64,14 +70,12 @@ impl rustc_driver::Callbacks for MyCallbacks { } fn after_analysis(&mut self, _compiler: &Compiler, tcx: TyCtxt<'_>) -> Compilation { - // Every compilation contains a single crate. - let hir_krate = tcx.hir(); // Iterate over the top-level items in the crate, looking for the main function. - for id in hir_krate.items() { - let item = hir_krate.item(id); + for id in tcx.hir_free_items() { + let item = &tcx.hir_item(id); // Use pattern-matching to find a specific node inside the main function. - if let rustc_hir::ItemKind::Fn(_, _, body_id) = item.kind { - let expr = &tcx.hir().body(body_id).value; + if let rustc_hir::ItemKind::Fn { body, .. } = item.kind { + let expr = &tcx.hir_body(body).value; if let rustc_hir::ExprKind::Block(block, _) = expr.kind { if let rustc_hir::StmtKind::Let(let_stmt) = block.stmts[0].kind { if let Some(expr) = let_stmt.init { @@ -90,10 +94,13 @@ impl rustc_driver::Callbacks for MyCallbacks { } fn main() { - match RunCompiler::new(&["main.rs".to_string()], &mut MyCallbacks) { - mut compiler => { - compiler.set_file_loader(Some(Box::new(MyFileLoader))); - compiler.run(); - } - } + run_compiler( + &[ + // The first argument, which in practice contains the name of the binary being executed + // (i.e. "rustc") is ignored by rustc. + "ignored".to_string(), + "main.rs".to_string(), + ], + &mut MyCallbacks, + ); } diff --git a/examples/rustc-interface-example.rs b/examples/rustc-interface-example.rs index 30f48ea52..360f70c8e 100644 --- a/examples/rustc-interface-example.rs +++ b/examples/rustc-interface-example.rs @@ -1,3 +1,5 @@ +// Tested with nightly-2025-03-28 + #![feature(rustc_private)] extern crate rustc_driver; @@ -9,8 +11,6 @@ extern crate rustc_interface; extern crate rustc_session; extern crate rustc_span; -use std::sync::Arc; - use rustc_errors::registry; use rustc_hash::FxHashMap; use rustc_session::config; @@ -56,7 +56,7 @@ fn main() { expanded_args: Vec::new(), ice_file: None, hash_untracked_state: None, - using_internal_features: Arc::default(), + using_internal_features: &rustc_driver::USING_INTERNAL_FEATURES, }; rustc_interface::run_compiler(config, |compiler| { // Parse the program and print the syntax tree. @@ -64,14 +64,13 @@ fn main() { println!("{krate:?}"); // Analyze the program and inspect the types of definitions. rustc_interface::create_and_enter_global_ctxt(&compiler, krate, |tcx| { - for id in tcx.hir().items() { - let hir = tcx.hir(); - let item = hir.item(id); + for id in tcx.hir_free_items() { + let item = tcx.hir_item(id); match item.kind { - rustc_hir::ItemKind::Static(_, _, _) | rustc_hir::ItemKind::Fn(_, _, _) => { - let name = item.ident; + rustc_hir::ItemKind::Static(ident, ..) + | rustc_hir::ItemKind::Fn { ident, .. } => { let ty = tcx.type_of(item.hir_id().owner.def_id); - println!("{name:?}:\t{ty:?}") + println!("{ident:?}:\t{ty:?}") } _ => (), } diff --git a/examples/rustc-interface-getting-diagnostics.rs b/examples/rustc-interface-getting-diagnostics.rs index be37dd867..2512ba3c3 100644 --- a/examples/rustc-interface-getting-diagnostics.rs +++ b/examples/rustc-interface-getting-diagnostics.rs @@ -1,3 +1,5 @@ +// Tested with nightly-2025-03-28 + #![feature(rustc_private)] extern crate rustc_data_structures; @@ -10,15 +12,15 @@ extern crate rustc_interface; extern crate rustc_session; extern crate rustc_span; +use std::sync::{Arc, Mutex}; + use rustc_errors::emitter::Emitter; use rustc_errors::registry::{self, Registry}; use rustc_errors::translation::Translate; -use rustc_errors::{DiagCtxt, DiagInner, FluentBundle}; +use rustc_errors::{DiagInner, FluentBundle}; use rustc_session::config; use rustc_span::source_map::SourceMap; -use std::sync::{Arc, Mutex}; - struct DebugEmitter { source_map: Arc, diagnostics: Arc>>, @@ -67,10 +69,10 @@ fn main() { locale_resources: rustc_driver::DEFAULT_LOCALE_RESOURCES.to_owned(), lint_caps: rustc_hash::FxHashMap::default(), psess_created: Some(Box::new(|parse_sess| { - parse_sess.set_dcx(DiagCtxt::new(Box::new(DebugEmitter { + parse_sess.dcx().set_emitter(Box::new(DebugEmitter { source_map: parse_sess.clone_source_map(), diagnostics, - }))); + })); })), register_lints: None, override_queries: None, @@ -79,13 +81,15 @@ fn main() { expanded_args: Vec::new(), ice_file: None, hash_untracked_state: None, - using_internal_features: Arc::default(), + using_internal_features: &rustc_driver::USING_INTERNAL_FEATURES, }; rustc_interface::run_compiler(config, |compiler| { let krate = rustc_interface::passes::parse(&compiler.sess); rustc_interface::create_and_enter_global_ctxt(&compiler, krate, |tcx| { - // Run the analysis phase on the local crate to trigger the type error. - let _ = tcx.analysis(()); + // Iterate all the items defined and perform type checking. + tcx.par_hir_body_owners(|item_def_id| { + tcx.ensure_ok().typeck(item_def_id); + }); }); // If the compiler has encountered errors when this closure returns, it will abort (!) the program. // We avoid this by resetting the error count before returning diff --git a/josh-sync.toml b/josh-sync.toml new file mode 100644 index 000000000..7882051e2 --- /dev/null +++ b/josh-sync.toml @@ -0,0 +1,3 @@ +org = "rust-lang" +repo = "rustc-dev-guide" +path = "src/doc/rustc-dev-guide" diff --git a/josh-sync/Cargo.lock b/josh-sync/Cargo.lock deleted file mode 100644 index 844518628..000000000 --- a/josh-sync/Cargo.lock +++ /dev/null @@ -1,430 +0,0 @@ -# This file is automatically @generated by Cargo. -# It is not intended for manual editing. -version = 4 - -[[package]] -name = "anstream" -version = "0.6.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8acc5369981196006228e28809f761875c0327210a891e941f4c683b3a99529b" -dependencies = [ - "anstyle", - "anstyle-parse", - "anstyle-query", - "anstyle-wincon", - "colorchoice", - "is_terminal_polyfill", - "utf8parse", -] - -[[package]] -name = "anstyle" -version = "1.0.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9" - -[[package]] -name = "anstyle-parse" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b2d16507662817a6a20a9ea92df6652ee4f94f914589377d69f3b21bc5798a9" -dependencies = [ - "utf8parse", -] - -[[package]] -name = "anstyle-query" -version = "1.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79947af37f4177cfead1110013d678905c37501914fba0efea834c3fe9a8d60c" -dependencies = [ - "windows-sys 0.59.0", -] - -[[package]] -name = "anstyle-wincon" -version = "3.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2109dbce0e72be3ec00bed26e6a7479ca384ad226efdd66db8fa2e3a38c83125" -dependencies = [ - "anstyle", - "windows-sys 0.59.0", -] - -[[package]] -name = "anyhow" -version = "1.0.95" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34ac096ce696dc2fcabef30516bb13c0a68a11d30131d3df6f04711467681b04" - -[[package]] -name = "bitflags" -version = "2.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" - -[[package]] -name = "cfg-if" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" - -[[package]] -name = "clap" -version = "4.5.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3135e7ec2ef7b10c6ed8950f0f792ed96ee093fa088608f1c76e569722700c84" -dependencies = [ - "clap_builder", - "clap_derive", -] - -[[package]] -name = "clap_builder" -version = "4.5.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30582fc632330df2bd26877bde0c1f4470d57c582bbc070376afcd04d8cb4838" -dependencies = [ - "anstream", - "anstyle", - "clap_lex", - "strsim", -] - -[[package]] -name = "clap_derive" -version = "4.5.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ac6a0c7b1a9e9a5186361f67dfa1b88213572f427fb9ab038efb2bd8c582dab" -dependencies = [ - "heck", - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "clap_lex" -version = "0.7.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f46ad14479a25103f283c0f10005961cf086d8dc42205bb44c46ac563475dca6" - -[[package]] -name = "colorchoice" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" - -[[package]] -name = "directories" -version = "5.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a49173b84e034382284f27f1af4dcbbd231ffa358c0fe316541a7337f376a35" -dependencies = [ - "dirs-sys", -] - -[[package]] -name = "dirs-sys" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "520f05a5cbd335fae5a99ff7a6ab8627577660ee5cfd6a94a6a929b52ff0321c" -dependencies = [ - "libc", - "option-ext", - "redox_users", - "windows-sys 0.48.0", -] - -[[package]] -name = "getrandom" -version = "0.2.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" -dependencies = [ - "cfg-if", - "libc", - "wasi", -] - -[[package]] -name = "heck" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" - -[[package]] -name = "is_terminal_polyfill" -version = "1.70.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" - -[[package]] -name = "josh-sync" -version = "0.1.0" -dependencies = [ - "anyhow", - "clap", - "directories", - "xshell", -] - -[[package]] -name = "libc" -version = "0.2.169" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a" - -[[package]] -name = "libredox" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" -dependencies = [ - "bitflags", - "libc", -] - -[[package]] -name = "option-ext" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" - -[[package]] -name = "proc-macro2" -version = "1.0.92" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37d3544b3f2748c54e147655edb5025752e2303145b5aefb3c3ea2c78b973bb0" -dependencies = [ - "unicode-ident", -] - -[[package]] -name = "quote" -version = "1.0.38" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e4dccaaaf89514f546c693ddc140f729f958c247918a13380cccc6078391acc" -dependencies = [ - "proc-macro2", -] - -[[package]] -name = "redox_users" -version = "0.4.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba009ff324d1fc1b900bd1fdb31564febe58a8ccc8a6fdbb93b543d33b13ca43" -dependencies = [ - "getrandom", - "libredox", - "thiserror", -] - -[[package]] -name = "strsim" -version = "0.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" - -[[package]] -name = "syn" -version = "2.0.93" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c786062daee0d6db1132800e623df74274a0a87322d8e183338e01b3d98d058" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - -[[package]] -name = "thiserror" -version = "1.0.69" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" -dependencies = [ - "thiserror-impl", -] - -[[package]] -name = "thiserror-impl" -version = "1.0.69" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "unicode-ident" -version = "1.0.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adb9e6ca4f869e1180728b7950e35922a7fc6397f7b641499e8f3ef06e50dc83" - -[[package]] -name = "utf8parse" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" - -[[package]] -name = "wasi" -version = "0.11.0+wasi-snapshot-preview1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" - -[[package]] -name = "windows-sys" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" -dependencies = [ - "windows-targets 0.48.5", -] - -[[package]] -name = "windows-sys" -version = "0.59.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" -dependencies = [ - "windows-targets 0.52.6", -] - -[[package]] -name = "windows-targets" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" -dependencies = [ - "windows_aarch64_gnullvm 0.48.5", - "windows_aarch64_msvc 0.48.5", - "windows_i686_gnu 0.48.5", - "windows_i686_msvc 0.48.5", - "windows_x86_64_gnu 0.48.5", - "windows_x86_64_gnullvm 0.48.5", - "windows_x86_64_msvc 0.48.5", -] - -[[package]] -name = "windows-targets" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" -dependencies = [ - "windows_aarch64_gnullvm 0.52.6", - "windows_aarch64_msvc 0.52.6", - "windows_i686_gnu 0.52.6", - "windows_i686_gnullvm", - "windows_i686_msvc 0.52.6", - "windows_x86_64_gnu 0.52.6", - "windows_x86_64_gnullvm 0.52.6", - "windows_x86_64_msvc 0.52.6", -] - -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" - -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" - -[[package]] -name = "windows_aarch64_msvc" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" - -[[package]] -name = "windows_aarch64_msvc" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" - -[[package]] -name = "windows_i686_gnu" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" - -[[package]] -name = "windows_i686_gnu" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" - -[[package]] -name = "windows_i686_gnullvm" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" - -[[package]] -name = "windows_i686_msvc" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" - -[[package]] -name = "windows_i686_msvc" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" - -[[package]] -name = "windows_x86_64_gnu" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" - -[[package]] -name = "windows_x86_64_gnu" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" - -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" - -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" - -[[package]] -name = "windows_x86_64_msvc" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" - -[[package]] -name = "windows_x86_64_msvc" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" - -[[package]] -name = "xshell" -version = "0.2.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e7290c623014758632efe00737145b6867b66292c42167f2ec381eb566a373d" -dependencies = [ - "xshell-macros", -] - -[[package]] -name = "xshell-macros" -version = "0.2.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32ac00cd3f8ec9c1d33fb3e7958a82df6989c42d747bd326c822b1d625283547" diff --git a/josh-sync/Cargo.toml b/josh-sync/Cargo.toml deleted file mode 100644 index 81d0d1ebd..000000000 --- a/josh-sync/Cargo.toml +++ /dev/null @@ -1,10 +0,0 @@ -[package] -name = "josh-sync" -version = "0.1.0" -edition = "2021" - -[dependencies] -anyhow = "1.0.95" -clap = { version = "4.5.21", features = ["derive"] } -directories = "5" -xshell = "0.2.6" diff --git a/josh-sync/README.md b/josh-sync/README.md deleted file mode 100644 index a3dd876e8..000000000 --- a/josh-sync/README.md +++ /dev/null @@ -1,4 +0,0 @@ -# Git josh sync -This utility serves for syncing the josh git subtree to and from the rust-lang/rust repository. - -See CLI help for usage. diff --git a/josh-sync/src/main.rs b/josh-sync/src/main.rs deleted file mode 100644 index 84613ad86..000000000 --- a/josh-sync/src/main.rs +++ /dev/null @@ -1,32 +0,0 @@ -use clap::Parser; -use crate::sync::GitSync; - -mod sync; - -#[derive(clap::Parser)] -enum Args { - /// Pull changes from the main `rustc` repository. - /// This creates new commits that should be then merged into `rustc-dev-guide`. - RustcPull, - /// Push changes from `rustc-dev-guide` to the given `branch` of a `rustc` fork under the given - /// GitHub `username`. - /// The pushed branch should then be merged into the `rustc` repository. - RustcPush { - branch: String, - github_username: String - } -} - -fn main() -> anyhow::Result<()> { - let args = Args::parse(); - let sync = GitSync::from_current_dir()?; - match args { - Args::RustcPull => { - sync.rustc_pull(None)?; - } - Args::RustcPush { github_username, branch } => { - sync.rustc_push(github_username, branch)?; - } - } - Ok(()) -} diff --git a/josh-sync/src/sync.rs b/josh-sync/src/sync.rs deleted file mode 100644 index da21a4c9a..000000000 --- a/josh-sync/src/sync.rs +++ /dev/null @@ -1,234 +0,0 @@ -use std::ops::Not; -use std::path::PathBuf; -use std::{env, net, process}; -use std::io::Write; -use std::time::Duration; -use anyhow::{anyhow, bail, Context}; -use xshell::{cmd, Shell}; - -/// Used for rustc syncs. -const JOSH_FILTER: &str = ":/src/doc/rustc-dev-guide"; -const JOSH_PORT: u16 = 42042; -const UPSTREAM_REPO: &str = "rust-lang/rust"; - -pub struct GitSync { - dir: PathBuf, -} - -/// This code was adapted from the miri repository -/// (https://github.com/rust-lang/miri/blob/6a68a79f38064c3bc30617cca4bdbfb2c336b140/miri-script/src/commands.rs#L236). -impl GitSync { - pub fn from_current_dir() -> anyhow::Result { - Ok(Self { - dir: std::env::current_dir()? - }) - } - - pub fn rustc_pull(&self, commit: Option) -> anyhow::Result<()> { - let sh = Shell::new()?; - sh.change_dir(&self.dir); - let commit = commit.map(Ok).unwrap_or_else(|| { - let rust_repo_head = - cmd!(sh, "git ls-remote https://github.com/{UPSTREAM_REPO}/ HEAD").read()?; - rust_repo_head - .split_whitespace() - .next() - .map(|front| front.trim().to_owned()) - .ok_or_else(|| anyhow!("Could not obtain Rust repo HEAD from remote.")) - })?; - // Make sure the repo is clean. - if cmd!(sh, "git status --untracked-files=no --porcelain").read()?.is_empty().not() { - bail!("working directory must be clean before performing rustc pull"); - } - // Make sure josh is running. - let josh = Self::start_josh()?; - let josh_url = - format!("http://localhost:{JOSH_PORT}/{UPSTREAM_REPO}.git@{commit}{JOSH_FILTER}.git"); - - // Update rust-version file. As a separate commit, since making it part of - // the merge has confused the heck out of josh in the past. - // We pass `--no-verify` to avoid running git hooks. - // We do this before the merge so that if there are merge conflicts, we have - // the right rust-version file while resolving them. - sh.write_file("rust-version", format!("{commit}\n"))?; - const PREPARING_COMMIT_MESSAGE: &str = "Preparing for merge from rustc"; - cmd!(sh, "git commit rust-version --no-verify -m {PREPARING_COMMIT_MESSAGE}") - .run() - .context("FAILED to commit rust-version file, something went wrong")?; - - // Fetch given rustc commit. - cmd!(sh, "git fetch {josh_url}") - .run() - .inspect_err(|_| { - // Try to un-do the previous `git commit`, to leave the repo in the state we found it. - cmd!(sh, "git reset --hard HEAD^") - .run() - .expect("FAILED to clean up again after failed `git fetch`, sorry for that"); - }) - .context("FAILED to fetch new commits, something went wrong (committing the rust-version file has been undone)")?; - - // This should not add any new root commits. So count those before and after merging. - let num_roots = || -> anyhow::Result { - Ok(cmd!(sh, "git rev-list HEAD --max-parents=0 --count") - .read() - .context("failed to determine the number of root commits")? - .parse::()?) - }; - let num_roots_before = num_roots()?; - - // Merge the fetched commit. - const MERGE_COMMIT_MESSAGE: &str = "Merge from rustc"; - cmd!(sh, "git merge FETCH_HEAD --no-verify --no-ff -m {MERGE_COMMIT_MESSAGE}") - .run() - .context("FAILED to merge new commits, something went wrong")?; - - // Check that the number of roots did not increase. - if num_roots()? != num_roots_before { - bail!("Josh created a new root commit. This is probably not the history you want."); - } - - drop(josh); - Ok(()) - } - - pub fn rustc_push(&self, github_user: String, branch: String) -> anyhow::Result<()> { - let sh = Shell::new()?; - sh.change_dir(&self.dir); - let base = sh.read_file("rust-version")?.trim().to_owned(); - // Make sure the repo is clean. - if cmd!(sh, "git status --untracked-files=no --porcelain").read()?.is_empty().not() { - bail!("working directory must be clean before running `rustc-push`"); - } - // Make sure josh is running. - let josh = Self::start_josh()?; - let josh_url = - format!("http://localhost:{JOSH_PORT}/{github_user}/rust.git{JOSH_FILTER}.git"); - - // Find a repo we can do our preparation in. - if let Ok(rustc_git) = env::var("RUSTC_GIT") { - // If rustc_git is `Some`, we'll use an existing fork for the branch updates. - sh.change_dir(rustc_git); - } else { - // Otherwise, do this in the local repo. - println!( - "This will pull a copy of the rust-lang/rust history into this checkout, growing it by about 1GB." - ); - print!( - "To avoid that, abort now and set the `RUSTC_GIT` environment variable to an existing rustc checkout. Proceed? [y/N] " - ); - std::io::stdout().flush()?; - let mut answer = String::new(); - std::io::stdin().read_line(&mut answer)?; - if answer.trim().to_lowercase() != "y" { - std::process::exit(1); - } - }; - // Prepare the branch. Pushing works much better if we use as base exactly - // the commit that we pulled from last time, so we use the `rust-version` - // file to find out which commit that would be. - println!("Preparing {github_user}/rust (base: {base})..."); - if cmd!(sh, "git fetch https://github.com/{github_user}/rust {branch}") - .ignore_stderr() - .read() - .is_ok() - { - println!( - "The branch '{branch}' seems to already exist in 'https://github.com/{github_user}/rust'. Please delete it and try again." - ); - std::process::exit(1); - } - cmd!(sh, "git fetch https://github.com/{UPSTREAM_REPO} {base}").run()?; - cmd!(sh, "git push https://github.com/{github_user}/rust {base}:refs/heads/{branch}") - .ignore_stdout() - .ignore_stderr() // silence the "create GitHub PR" message - .run()?; - println!(); - - // Do the actual push. - sh.change_dir(&self.dir); - println!("Pushing changes..."); - cmd!(sh, "git push {josh_url} HEAD:{branch}").run()?; - println!(); - - // Do a round-trip check to make sure the push worked as expected. - cmd!(sh, "git fetch {josh_url} {branch}").ignore_stderr().read()?; - let head = cmd!(sh, "git rev-parse HEAD").read()?; - let fetch_head = cmd!(sh, "git rev-parse FETCH_HEAD").read()?; - if head != fetch_head { - bail!( - "Josh created a non-roundtrip push! Do NOT merge this into rustc!\n\ - Expected {head}, got {fetch_head}." - ); - } - println!( - "Confirmed that the push round-trips back to rustc-dev-guide properly. Please create a rustc PR:" - ); - println!( - // Open PR with `subtree update` title to silence the `no-merges` triagebot check - " https://github.com/{UPSTREAM_REPO}/compare/{github_user}:{branch}?quick_pull=1&title=Rustc+dev+guide+subtree+update&body=r?+@ghost" - ); - - drop(josh); - Ok(()) - } - - fn start_josh() -> anyhow::Result { - // Determine cache directory. - let local_dir = { - let user_dirs = - directories::ProjectDirs::from("org", "rust-lang", "rustc-dev-guide-josh").unwrap(); - user_dirs.cache_dir().to_owned() - }; - - // Start josh, silencing its output. - let mut cmd = process::Command::new("josh-proxy"); - cmd.arg("--local").arg(local_dir); - cmd.arg("--remote").arg("https://github.com"); - cmd.arg("--port").arg(JOSH_PORT.to_string()); - cmd.arg("--no-background"); - cmd.stdout(process::Stdio::null()); - cmd.stderr(process::Stdio::null()); - let josh = cmd.spawn().context("failed to start josh-proxy, make sure it is installed")?; - - // Create a wrapper that stops it on drop. - struct Josh(process::Child); - impl Drop for Josh { - fn drop(&mut self) { - #[cfg(unix)] - { - // Try to gracefully shut it down. - process::Command::new("kill") - .args(["-s", "INT", &self.0.id().to_string()]) - .output() - .expect("failed to SIGINT josh-proxy"); - // Sadly there is no "wait with timeout"... so we just give it some time to finish. - std::thread::sleep(Duration::from_millis(100)); - // Now hopefully it is gone. - if self.0.try_wait().expect("failed to wait for josh-proxy").is_some() { - return; - } - } - // If that didn't work (or we're not on Unix), kill it hard. - eprintln!( - "I have to kill josh-proxy the hard way, let's hope this does not break anything." - ); - self.0.kill().expect("failed to SIGKILL josh-proxy"); - } - } - - // Wait until the port is open. We try every 10ms until 1s passed. - for _ in 0..100 { - // This will generally fail immediately when the port is still closed. - let josh_ready = net::TcpStream::connect_timeout( - &net::SocketAddr::from(([127, 0, 0, 1], JOSH_PORT)), - Duration::from_millis(1), - ); - if josh_ready.is_ok() { - return Ok(Josh(josh)); - } - // Not ready yet. - std::thread::sleep(Duration::from_millis(10)); - } - bail!("Even after waiting for 1s, josh-proxy is still not available.") - } -} diff --git a/pagetoc.css b/pagetoc.css new file mode 100644 index 000000000..fa709194f --- /dev/null +++ b/pagetoc.css @@ -0,0 +1,84 @@ +/* Inspired by https://github.com/JorelAli/mdBook-pagetoc/tree/98ee241 (under WTFPL) */ + +:root { + --toc-width: 270px; + --center-content-toc-shift: calc(-1 * var(--toc-width) / 2); +} + +.nav-chapters { + /* adjust width of buttons that bring to the previous or the next page */ + min-width: 50px; +} + +@media only screen { + @media (max-width: 1179px) { + .sidebar-hidden #sidetoc { + display: none; + } + } + + @media (max-width: 1439px) { + .sidebar-visible #sidetoc { + display: none; + } + } + + @media (1180px <= width <= 1439px) { + .sidebar-hidden main { + position: relative; + left: var(--center-content-toc-shift); + } + } + + @media (1440px <= width <= 1700px) { + .sidebar-visible main { + position: relative; + left: var(--center-content-toc-shift); + } + } + + #sidetoc { + margin-left: calc(100% + 20px); + } + #pagetoc { + position: fixed; + /* adjust TOC width */ + width: var(--toc-width); + height: calc(100vh - var(--menu-bar-height) - 0.67em * 4); + overflow: auto; + } + #pagetoc a { + border-left: 1px solid var(--sidebar-bg); + color: var(--fg); + display: block; + padding-bottom: 5px; + padding-top: 5px; + padding-left: 10px; + text-align: left; + text-decoration: none; + } + #pagetoc a:hover, + #pagetoc a.active { + background: var(--sidebar-bg); + color: var(--sidebar-active) !important; + } + #pagetoc .active { + background: var(--sidebar-bg); + color: var(--sidebar-active); + } + #pagetoc .pagetoc-H2 { + padding-left: 20px; + } + #pagetoc .pagetoc-H3 { + padding-left: 40px; + } + #pagetoc .pagetoc-H4 { + padding-left: 60px; + } +} + +@media print { + #sidetoc { + display: none; + } +} diff --git a/pagetoc.js b/pagetoc.js new file mode 100644 index 000000000..927a5b107 --- /dev/null +++ b/pagetoc.js @@ -0,0 +1,104 @@ +// Inspired by https://github.com/JorelAli/mdBook-pagetoc/tree/98ee241 (under WTFPL) + +let activeHref = location.href; +function updatePageToc(elem = undefined) { + let selectedPageTocElem = elem; + const pagetoc = document.getElementById("pagetoc"); + + function getRect(element) { + return element.getBoundingClientRect(); + } + + function overflowTop(container, element) { + return getRect(container).top - getRect(element).top; + } + + function overflowBottom(container, element) { + return getRect(container).bottom - getRect(element).bottom; + } + + // We've not selected a heading to highlight, and the URL needs updating + // so we need to find a heading based on the URL + if (selectedPageTocElem === undefined && location.href !== activeHref) { + activeHref = location.href; + for (const pageTocElement of pagetoc.children) { + if (pageTocElement.href === activeHref) { + selectedPageTocElem = pageTocElement; + } + } + } + + // We still don't have a selected heading, let's try and find the most + // suitable heading based on the scroll position + if (selectedPageTocElem === undefined) { + const margin = window.innerHeight / 3; + + const headers = document.getElementsByClassName("header"); + for (let i = 0; i < headers.length; i++) { + const header = headers[i]; + if (selectedPageTocElem === undefined && getRect(header).top >= 0) { + if (getRect(header).top < margin) { + selectedPageTocElem = header; + } else { + selectedPageTocElem = headers[Math.max(0, i - 1)]; + } + } + // a very long last section's heading is over the screen + if (selectedPageTocElem === undefined && i === headers.length - 1) { + selectedPageTocElem = header; + } + } + } + + // Remove the active flag from all pagetoc elements + for (const pageTocElement of pagetoc.children) { + pageTocElement.classList.remove("active"); + } + + // If we have a selected heading, set it to active and scroll to it + if (selectedPageTocElem !== undefined) { + for (const pageTocElement of pagetoc.children) { + if (selectedPageTocElem.href.localeCompare(pageTocElement.href) === 0) { + pageTocElement.classList.add("active"); + if (overflowTop(pagetoc, pageTocElement) > 0) { + pagetoc.scrollTop = pageTocElement.offsetTop; + } + if (overflowBottom(pagetoc, pageTocElement) < 0) { + pagetoc.scrollTop -= overflowBottom(pagetoc, pageTocElement); + } + } + } + } +} + +if (document.getElementById("sidetoc") === null && + document.getElementsByClassName("header").length > 0) { + // The sidetoc element doesn't exist yet, let's create it + + // Create the empty sidetoc and pagetoc elements + const sidetoc = document.createElement("div"); + const pagetoc = document.createElement("div"); + sidetoc.id = "sidetoc"; + pagetoc.id = "pagetoc"; + sidetoc.appendChild(pagetoc); + + // And append them to the current DOM + const main = document.querySelector('main'); + main.insertBefore(sidetoc, main.firstChild); + + // Populate sidebar on load + window.addEventListener("load", () => { + for (const header of document.getElementsByClassName("header")) { + const link = document.createElement("a"); + link.innerHTML = header.innerHTML; + link.href = header.hash; + link.classList.add("pagetoc-" + header.parentElement.tagName); + document.getElementById("pagetoc").appendChild(link); + link.onclick = () => updatePageToc(link); + } + updatePageToc(); + }); + + // Update page table of contents selected heading on scroll + window.addEventListener("scroll", () => updatePageToc()); +} diff --git a/rust-version b/rust-version index 876c32dcf..6ec700b9b 100644 --- a/rust-version +++ b/rust-version @@ -1 +1 @@ -dcfa38fe234de9304169afc6638e81d0dd222c06 +6bcdcc73bd11568fd85f5a38b58e1eda054ad1cd diff --git a/rustfmt.toml b/rustfmt.toml new file mode 100644 index 000000000..b285329c7 --- /dev/null +++ b/rustfmt.toml @@ -0,0 +1,7 @@ +# matches that of rust-lang/rust +style_edition = "2024" +use_small_heuristics = "Max" +merge_derives = false +group_imports = "StdExternalCrate" +imports_granularity = "Module" +use_field_init_shorthand = true diff --git a/src/SUMMARY.md b/src/SUMMARY.md index a8fddbc75..9ded467d5 100644 --- a/src/SUMMARY.md +++ b/src/SUMMARY.md @@ -1,6 +1,7 @@ # Summary [Getting Started](./getting-started.md) + [About this guide](./about-this-guide.md) --- @@ -10,9 +11,9 @@ - [How to build and run the compiler](./building/how-to-build-and-run.md) - [Quickstart](./building/quickstart.md) - [Prerequisites](./building/prerequisites.md) - - [Suggested Workflows](./building/suggested.md) + - [Suggested workflows](./building/suggested.md) - [Distribution artifacts](./building/build-install-distribution-artifacts.md) - - [Building Documentation](./building/compiler-documenting.md) + - [Building documentation](./building/compiler-documenting.md) - [Rustdoc overview](./rustdoc.md) - [Adding a new target](./building/new-target.md) - [Optimized build](./building/optimized-build.md) @@ -28,10 +29,12 @@ - [Minicore](./tests/minicore.md) - [Ecosystem testing](./tests/ecosystem.md) - [Crater](./tests/crater.md) - - [Fuchsia](./tests/fuchsia.md) - - [Rust for Linux](./tests/rust-for-linux.md) + - [Fuchsia](./tests/ecosystem-test-jobs/fuchsia.md) + - [Rust for Linux](./tests/ecosystem-test-jobs/rust-for-linux.md) + - [Codegen backend testing](./tests/codegen-backend-tests/intro.md) + - [Cranelift codegen backend](./tests/codegen-backend-tests/cg_clif.md) + - [GCC codegen backend](./tests/codegen-backend-tests/cg_gcc.md) - [Performance testing](./tests/perf.md) - - [Suggest tests tool](./tests/suggest-tests.md) - [Misc info](./tests/misc.md) - [Debugging the compiler](./compiler-debugging.md) - [Using the tracing/logging instrumentation](./tracing.md) @@ -39,34 +42,34 @@ - [with the linux perf tool](./profiling/with_perf.md) - [with Windows Performance Analyzer](./profiling/wpa_profiling.md) - [with the Rust benchmark suite](./profiling/with_rustc_perf.md) -- [crates.io Dependencies](./crates-io.md) +- [crates.io dependencies](./crates-io.md) # Contributing to Rust -- [Contribution Procedures](./contributing.md) +- [Contribution procedures](./contributing.md) - [About the compiler team](./compiler-team.md) - [Using Git](./git.md) - [Mastering @rustbot](./rustbot.md) - [Walkthrough: a typical contribution](./walkthrough.md) - [Implementing new language features](./implementing_new_features.md) - [Stability attributes](./stability.md) -- [Stabilizing Features](./stabilization_guide.md) +- [Stabilizing language features](./stabilization_guide.md) + - [Stabilization report template](./stabilization_report_template.md) - [Feature Gates](./feature-gates.md) - [Coding conventions](./conventions.md) -- [Procedures for Breaking Changes](./bug-fix-procedure.md) +- [Procedures for breaking changes](./bug-fix-procedure.md) - [Using external repositories](./external-repos.md) - [Fuzzing](./fuzzing.md) - [Notification groups](notification-groups/about.md) - [Apple](notification-groups/apple.md) - [ARM](notification-groups/arm.md) - - [Cleanup Crew](notification-groups/cleanup-crew.md) - [Emscripten](notification-groups/emscripten.md) - - [LLVM](notification-groups/llvm.md) + - [Fuchsia](notification-groups/fuchsia.md) - [RISC-V](notification-groups/risc-v.md) + - [Rust for Linux](notification-groups/rust-for-linux.md) - [WASI](notification-groups/wasi.md) - [WebAssembly](notification-groups/wasm.md) - [Windows](notification-groups/windows.md) - - [Rust for Linux](notification-groups/rust-for-linux.md) - [Licenses](./licenses.md) - [Editions](guides/editions.md) @@ -75,6 +78,9 @@ - [Prologue](./building/bootstrapping/intro.md) - [What Bootstrapping does](./building/bootstrapping/what-bootstrapping-does.md) - [How Bootstrap does it](./building/bootstrapping/how-bootstrap-does-it.md) +- [Writing tools in Bootstrap](./building/bootstrapping/writing-tools-in-bootstrap.md) +- [Debugging bootstrap](./building/bootstrapping/debugging-bootstrap.md) +- [cfg(bootstrap) in dependencies](./building/bootstrapping/bootstrap-in-dependencies.md) # High-level Compiler Architecture @@ -82,33 +88,44 @@ - [Overview of the compiler](./overview.md) - [The compiler source code](./compiler-src.md) - [Queries: demand-driven compilation](./query.md) - - [The Query Evaluation Model in Detail](./queries/query-evaluation-model-in-detail.md) + - [The Query Evaluation Model in detail](./queries/query-evaluation-model-in-detail.md) - [Incremental compilation](./queries/incremental-compilation.md) - - [Incremental compilation In Detail](./queries/incremental-compilation-in-detail.md) - - [Debugging and Testing](./incrcomp-debugging.md) + - [Incremental compilation in detail](./queries/incremental-compilation-in-detail.md) + - [Debugging and testing](./incrcomp-debugging.md) - [Salsa](./queries/salsa.md) -- [Memory Management in Rustc](./memory.md) -- [Serialization in Rustc](./serialization.md) -- [Parallel Compilation](./parallel-rustc.md) +- [Memory management in rustc](./memory.md) +- [Serialization in rustc](./serialization.md) +- [Parallel compilation](./parallel-rustc.md) - [Rustdoc internals](./rustdoc-internals.md) - [Search](./rustdoc-internals/search.md) + - [The `rustdoc` test suite](./rustdoc-internals/rustdoc-test-suite.md) + - [The `rustdoc-gui` test suite](./rustdoc-internals/rustdoc-gui-test-suite.md) + - [The `rustdoc-json` test suite](./rustdoc-internals/rustdoc-json-test-suite.md) +- [GPU offload internals](./offload/internals.md) + - [Installation](./offload/installation.md) +- [Autodiff internals](./autodiff/internals.md) + - [Installation](./autodiff/installation.md) + - [How to debug](./autodiff/debugging.md) + - [Autodiff flags](./autodiff/flags.md) + - [Current limitations](./autodiff/limitations.md) # Source Code Representation - [Prologue](./part-3-intro.md) - [Syntax and the AST](./syntax-intro.md) - - [Lexing and Parsing](./the-parser.md) + - [Lexing and parsing](./the-parser.md) - [Macro expansion](./macro-expansion.md) - [Name resolution](./name-resolution.md) - [Attributes](./attributes.md) - - [`#[test]` Implementation](./test-implementation.md) - - [Panic Implementation](./panic-implementation.md) - - [AST Validation](./ast-validation.md) - - [Feature Gate Checking](./feature-gate-ck.md) + - [`#[test]` implementation](./test-implementation.md) + - [Panic implementation](./panic-implementation.md) + - [AST validation](./ast-validation.md) + - [Feature gate checking](./feature-gate-ck.md) - [Lang Items](./lang-items.md) - [The HIR (High-level IR)](./hir.md) - - [Lowering AST to HIR](./ast-lowering.md) - - [Debugging](./hir-debugging.md) + - [Lowering AST to HIR](./hir/lowering.md) + - [Ambig/Unambig Types and Consts](./hir/ambig-unambig-ty-and-consts.md) + - [Debugging](./hir/debugging.md) - [The THIR (Typed High-level IR)](./thir.md) - [The MIR (Mid-level IR)](./mir/index.md) - [MIR construction](./mir/construction.md) @@ -120,9 +137,10 @@ - [Command-line arguments](./cli.md) - [rustc_driver and rustc_interface](./rustc-driver/intro.md) + - [Remarks on perma-unstable features](./rustc-driver/remarks-on-perma-unstable-features.md) - [Example: Type checking](./rustc-driver/interacting-with-the-ast.md) - [Example: Getting diagnostics](./rustc-driver/getting-diagnostics.md) -- [Errors and Lints](diagnostics.md) +- [Errors and lints](diagnostics.md) - [Diagnostic and subdiagnostic structs](./diagnostics/diagnostic-structs.md) - [Translation](./diagnostics/translation.md) - [`LintStore`](./diagnostics/lintstore.md) @@ -142,10 +160,8 @@ - [ADTs and Generic Arguments](./ty_module/generic_arguments.md) - [Parameter types/consts/regions](./ty_module/param_ty_const_regions.md) - [`TypeFolder` and `TypeFoldable`](./ty-fold.md) -- [Parameter Environments](./param_env/param_env_summary.md) - - [What is it?](./param_env/param_env_what_is_it.md) - - [How are `ParamEnv`'s constructed internally](./param_env/param_env_construction_internals.md) - - [Which `ParamEnv` do I use?](./param_env/param_env_acquisition.md) +- [Aliases and Normalization](./normalization.md) +- [Typing/Param Envs](./typing_parameter_envs.md) - [Type inference](./type-inference.md) - [Trait solving](./traits/resolution.md) - [Higher-ranked trait bounds](./traits/hrtb.md) @@ -160,25 +176,25 @@ - [Next-gen trait solving](./solve/trait-solving.md) - [Invariants of the type system](./solve/invariants.md) - [The solver](./solve/the-solver.md) + - [Candidate preference](./solve/candidate-preference.md) - [Canonicalization](./solve/canonicalization.md) - [Coinduction](./solve/coinduction.md) - [Caching](./solve/caching.md) - [Proof trees](./solve/proof-trees.md) - - [Normalization](./solve/normalization.md) - [Opaque types](./solve/opaque-types.md) - [Significant changes and quirks](./solve/significant-changes.md) - [`Unsize` and `CoerceUnsized` traits](./traits/unsize.md) - [Type checking](./type-checking.md) - - [Method Lookup](./method-lookup.md) + - [Method lookup](./method-lookup.md) - [Variance](./variance.md) - - [Coherence Checking](./coherence.md) - - [Opaque Types](./opaque-types-type-alias-impl-trait.md) + - [Coherence checking](./coherence.md) + - [Opaque types](./opaque-types-type-alias-impl-trait.md) - [Inference details](./opaque-types-impl-trait-inference.md) - [Return Position Impl Trait In Trait](./return-position-impl-trait-in-trait.md) - [Region inference restrictions][opaque-infer] -- [Effect checking](./effects.md) -- [Pattern and Exhaustiveness Checking](./pat-exhaustive-checking.md) -- [Unsafety Checking](./unsafety-checking.md) +- [Const condition checking](./effects.md) +- [Pattern and exhaustiveness checking](./pat-exhaustive-checking.md) +- [Unsafety checking](./unsafety-checking.md) - [MIR dataflow](./mir/dataflow.md) - [Drop elaboration](./mir/drop-elaboration.md) - [The borrow checker](./borrow_check.md) @@ -197,7 +213,7 @@ - [Closure capture inference](./closure.md) - [Async closures/"coroutine-closures"](coroutine-closures.md) -# MIR to Binaries +# MIR to binaries - [Prologue](./part-5-intro.md) - [MIR optimizations](./mir/optimizations.md) @@ -206,23 +222,27 @@ - [Interpreter](./const-eval/interpret.md) - [Monomorphization](./backend/monomorph.md) - [Lowering MIR](./backend/lowering-mir.md) -- [Code Generation](./backend/codegen.md) +- [Code generation](./backend/codegen.md) - [Updating LLVM](./backend/updating-llvm.md) - [Debugging LLVM](./backend/debugging.md) - [Backend Agnostic Codegen](./backend/backend-agnostic.md) - - [Implicit Caller Location](./backend/implicit-caller-location.md) -- [Libraries and Metadata](./backend/libs-and-metadata.md) -- [Profile-guided Optimization](./profile-guided-optimization.md) -- [LLVM Source-Based Code Coverage](./llvm-coverage-instrumentation.md) -- [Sanitizers Support](./sanitizers.md) + - [Implicit caller location](./backend/implicit-caller-location.md) +- [Libraries and metadata](./backend/libs-and-metadata.md) +- [Profile-guided optimization](./profile-guided-optimization.md) +- [LLVM source-based code coverage](./llvm-coverage-instrumentation.md) +- [Sanitizers support](./sanitizers.md) - [Debugging support in the Rust compiler](./debugging-support-in-rustc.md) --- [Appendix A: Background topics](./appendix/background.md) + [Appendix B: Glossary](./appendix/glossary.md) + [Appendix C: Code Index](./appendix/code-index.md) + [Appendix D: Compiler Lecture Series](./appendix/compiler-lecture.md) + [Appendix E: Bibliography](./appendix/bibliography.md) [Appendix Z: HumorRust](./appendix/humorust.md) diff --git a/src/about-this-guide.md b/src/about-this-guide.md index 793bfa9e6..f39577249 100644 --- a/src/about-this-guide.md +++ b/src/about-this-guide.md @@ -3,33 +3,41 @@ This guide is meant to help document how rustc – the Rust compiler – works, as well as to help new contributors get involved in rustc development. -There are seven parts to this guide: +There are several parts to this guide: -1. [Building `rustc`][p1]: +1. [Building and debugging `rustc`][p1]: Contains information that should be useful no matter how you are contributing, about building, debugging, profiling, etc. -2. [Contributing to `rustc`][p2]: +1. [Contributing to Rust][p2]: Contains information that should be useful no matter how you are contributing, about procedures for contribution, using git and Github, stabilizing features, etc. -3. [High-Level Compiler Architecture][p3]: +1. [Bootstrapping][p3]: + Describes how the Rust compiler builds itself using previous versions, including + an introduction to the bootstrap process and debugging methods. +1. [High-level Compiler Architecture][p4]: Discusses the high-level architecture of the compiler and stages of the compile process. -4. [Source Code Representation][p4]: +1. [Source Code Representation][p5]: Describes the process of taking raw source code from the user and transforming it into various forms that the compiler can work with easily. -5. [Analysis][p5]: - discusses the analyses that the compiler uses to check various properties of the code +1. [Supporting Infrastructure][p6]: + Covers command-line argument conventions, compiler entry points like rustc_driver and + rustc_interface, and the design and implementation of errors and lints. +1. [Analysis][p7]: + Discusses the analyses that the compiler uses to check various properties of the code and inform later stages of the compile process (e.g., type checking). -6. [From MIR to Binaries][p6]: How linked executable machine code is generated. -7. [Appendices][p7] at the end with useful reference information. +1. [MIR to Binaries][p8]: How linked executable machine code is generated. +1. [Appendices][p9] at the end with useful reference information. There are a few of these with different information, including a glossary. [p1]: ./building/how-to-build-and-run.html [p2]: ./contributing.md -[p3]: ./part-2-intro.md -[p4]: ./part-3-intro.md -[p5]: ./part-4-intro.md -[p6]: ./part-5-intro.md -[p7]: ./appendix/background.md +[p3]: ./building/bootstrapping/intro.md +[p4]: ./part-2-intro.md +[p5]: ./part-3-intro.md +[p6]: ./cli.md +[p7]: ./part-4-intro.md +[p8]: ./part-5-intro.md +[p9]: ./appendix/background.md ### Constant change @@ -66,13 +74,11 @@ You might also find the following sites useful: of the team procedures, active working groups, and the team calendar. - [std-dev-guide] -- a similar guide for developing the standard library. - [The t-compiler zulip][z] -- `#contribute` and `#wg-rustup` on [Discord](https://discord.gg/rust-lang). - The [Rust Internals forum][rif], a place to ask questions and discuss Rust's internals - The [Rust reference][rr], even though it doesn't specifically talk about Rust's internals, is a great resource nonetheless - Although out of date, [Tom Lee's great blog article][tlgba] is very helpful -- [rustaceans.org][ro] is helpful, but mostly dedicated to IRC - The [Rust Compiler Testing Docs][rctd] - For [@bors], [this cheat sheet][cheatsheet] is helpful - Google is always helpful when programming. diff --git a/src/appendix/bibliography.md b/src/appendix/bibliography.md index 8f6810cbc..93426b645 100644 --- a/src/appendix/bibliography.md +++ b/src/appendix/bibliography.md @@ -82,7 +82,7 @@ Rust, as well as publications about Rust. * [Ownership is Theft: Experiences Building an Embedded OS in Rust - Amit Levy, et. al.](https://amitlevy.com/papers/tock-plos2015.pdf) * [You can't spell trust without Rust](https://faultlore.com/blah/papers/thesis.pdf). Aria Beingessner's master's thesis. * [Rust-Bio: a fast and safe bioinformatics library](https://rust-bio.github.io/). Johannes Köster -* [Safe, Correct, and Fast Low-Level Networking](https://octarineparrot.com/assets/msci_paper.pdf). Robert Clipsham's master's thesis. +* [Safe, Correct, and Fast Low-Level Networking](https://csperkins.org/research/thesis-msci-clipsham.pdf). Robert Clipsham's master's thesis. * [Formalizing Rust traits](https://open.library.ubc.ca/cIRcle/collections/ubctheses/24/items/1.0220521). Jonatan Milewski's master's thesis. * [Rust as a Language for High Performance GC Implementation](https://dl.acm.org/doi/pdf/10.1145/3241624.2926707) * [Simple Verification of Rust Programs via Functional Purification](https://github.com/Kha/electrolysis). Sebastian Ullrich's master's thesis. diff --git a/src/appendix/code-index.md b/src/appendix/code-index.md index c33887d72..65fbf752d 100644 --- a/src/appendix/code-index.md +++ b/src/appendix/code-index.md @@ -14,17 +14,16 @@ Item | Kind | Short description | Chapter | `Diag` | struct | A struct for a compiler diagnostic, such as an error or lint | [Emitting Diagnostics] | [compiler/rustc_errors/src/diagnostic.rs](https://doc.rust-lang.org/nightly/nightly-rustc/rustc_errors/struct.Diag.html) `DocContext` | struct | A state container used by rustdoc when crawling through a crate to gather its documentation | [Rustdoc] | [src/librustdoc/core.rs](https://github.com/rust-lang/rust/blob/master/src/librustdoc/core.rs) `HirId` | struct | One of four types of HIR node identifiers | [Identifiers in the HIR] | [compiler/rustc_hir/src/hir_id.rs](https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir/hir_id/struct.HirId.html) +`Lexer` | struct | This is the lexer used during parsing. It consumes characters from the raw source code being compiled and produces a series of tokens for use by the rest of the parser | [The parser] | [compiler/rustc_parse/src/lexer/mod.rs](https://doc.rust-lang.org/nightly/nightly-rustc/rustc_parse/lexer/struct.Lexer.html) `NodeId` | struct | One of four types of HIR node identifiers. Being phased out | [Identifiers in the HIR] | [compiler/rustc_ast/src/ast.rs](https://doc.rust-lang.org/nightly/nightly-rustc/rustc_ast/node_id/struct.NodeId.html) `P` | struct | An owned immutable smart pointer. By contrast, `&T` is not owned, and `Box` is not immutable. | None | [compiler/rustc_ast/src/ptr.rs](https://doc.rust-lang.org/nightly/nightly-rustc/rustc_ast/ptr/struct.P.html) `ParamEnv` | struct | Information about generic parameters or `Self`, useful for working with associated or generic items | [Parameter Environment] | [compiler/rustc_middle/src/ty/mod.rs](https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/struct.ParamEnv.html) `ParseSess` | struct | This struct contains information about a parsing session | [The parser] | [compiler/rustc_session/src/parse/parse.rs](https://doc.rust-lang.org/nightly/nightly-rustc/rustc_session/parse/struct.ParseSess.html) -`Query` | struct | Represents the result of query to the `Compiler` interface and allows stealing, borrowing, and returning the results of compiler passes. | [The Rustc Driver and Interface] | [compiler/rustc_interface/src/queries.rs](https://doc.rust-lang.org/nightly/nightly-rustc/rustc_interface/queries/struct.Query.html) `Rib` | struct | Represents a single scope of names | [Name resolution] | [compiler/rustc_resolve/src/lib.rs](https://doc.rust-lang.org/nightly/nightly-rustc/rustc_resolve/late/struct.Rib.html) `Session` | struct | The data associated with a compilation session | [The parser], [The Rustc Driver and Interface] | [compiler/rustc_session/src/session.rs](https://doc.rust-lang.org/nightly/nightly-rustc/rustc_session/struct.Session.html) `SourceFile` | struct | Part of the `SourceMap`. Maps AST nodes to their source code for a single source file. Was previously called FileMap | [The parser] | [compiler/rustc_span/src/lib.rs](https://doc.rust-lang.org/nightly/nightly-rustc/rustc_span/struct.SourceFile.html) `SourceMap` | struct | Maps AST nodes to their source code. It is composed of `SourceFile`s. Was previously called CodeMap | [The parser] | [compiler/rustc_span/src/source_map.rs](https://doc.rust-lang.org/nightly/nightly-rustc/rustc_span/source_map/struct.SourceMap.html) `Span` | struct | A location in the user's source code, used for error reporting primarily | [Emitting Diagnostics] | [compiler/rustc_span/src/span_encoding.rs](https://doc.rust-lang.org/nightly/nightly-rustc/rustc_span/struct.Span.html) -`StringReader` | struct | This is the lexer used during parsing. It consumes characters from the raw source code being compiled and produces a series of tokens for use by the rest of the parser | [The parser] | [compiler/rustc_parse/src/lexer/mod.rs](https://doc.rust-lang.org/nightly/nightly-rustc/rustc_parse/lexer/struct.StringReader.html) `rustc_ast::token_stream::TokenStream` | struct | An abstract sequence of tokens, organized into `TokenTree`s | [The parser], [Macro expansion] | [compiler/rustc_ast/src/tokenstream.rs](https://doc.rust-lang.org/nightly/nightly-rustc/rustc_ast/tokenstream/struct.TokenStream.html) `TraitDef` | struct | This struct contains a trait's definition with type information | [The `ty` modules] | [compiler/rustc_middle/src/ty/trait_def.rs](https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/trait_def/struct.TraitDef.html) `TraitRef` | struct | The combination of a trait and its input types (e.g. `P0: Trait`) | [Trait Solving: Goals and Clauses] | [compiler/rustc_middle/src/ty/sty.rs](https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/type.TraitRef.html) @@ -41,5 +40,5 @@ Item | Kind | Short description | Chapter | [Emitting Diagnostics]: ../diagnostics.html [Macro expansion]: ../macro-expansion.html [Name resolution]: ../name-resolution.html -[Parameter Environment]: ../param_env/param_env_summary.html +[Parameter Environment]: ../typing_parameter_envs.html [Trait Solving: Goals and Clauses]: ../traits/goals-and-clauses.html#domain-goals diff --git a/src/appendix/compiler-lecture.md b/src/appendix/compiler-lecture.md index dabd2f087..90c4097cc 100644 --- a/src/appendix/compiler-lecture.md +++ b/src/appendix/compiler-lecture.md @@ -46,3 +46,4 @@ These are videos where various experts explain different parts of the compiler: ## Code Generation - [January 2019: Cranelift](https://www.youtube.com/watch?v=9OIA7DTFQWU) +- [December 2024: LLVM Developers' Meeting - Rust ❤️ LLVM](https://www.youtube.com/watch?v=Kqz-umsAnk8) \ No newline at end of file diff --git a/src/appendix/glossary.md b/src/appendix/glossary.md index bf3475a98..1837b59e8 100644 --- a/src/appendix/glossary.md +++ b/src/appendix/glossary.md @@ -31,7 +31,6 @@ Term | Meaning generics | The list of generic parameters defined on an item. There are three kinds of generic parameters: Type, lifetime and const parameters. HIR | The _high-level [IR](#ir)_, created by lowering and desugaring the AST. ([see more](../hir.md)) `HirId` | Identifies a particular node in the HIR by combining a def-id with an "intra-definition offset". See [the HIR chapter for more](../hir.md#identifiers-in-the-hir). -HIR map | The HIR map, accessible via `tcx.hir()`, allows you to quickly navigate the HIR and convert between various forms of identifiers. ICE | Short for _internal compiler error_, this is when the compiler crashes. ICH | Short for _incremental compilation hash_, these are used as fingerprints for things such as HIR and crate metadata, to check if changes have been made. This is useful in incremental compilation to see if part of a crate has changed and should be recompiled. `infcx` | The type inference context (`InferCtxt`). (see `rustc_middle::infer`) @@ -69,7 +68,7 @@ Term | Meaning rib | A data structure in the name resolver that keeps track of a single scope for names. ([see more](../name-resolution.md)) RPIT | A return-position `impl Trait`. ([see the reference](https://doc.rust-lang.org/reference/types/impl-trait.html#abstract-return-types)). RPITIT | A return-position `impl Trait` in trait. Unlike RPIT, this is desugared to a generic associated type (GAT). Introduced in [RFC 3425](https://rust-lang.github.io/rfcs/3425-return-position-impl-trait-in-traits.html). ([see more](../return-position-impl-trait-in-trait.md)) -scrutinee | A scrutinee is the expression that is matched on in `match` expressions and similar pattern matching constructs. For example, in `match x { A => 1, B => 2 }`, the expression `x` is the scrutinee. +scrutinee | A scrutinee is the expression that is matched on in `match` expressions and similar pattern matching constructs. For example, in `match x { A => 1, B => 2 }`, the expression `x` is the scrutinee. `sess` | The compiler _session_, which stores global data used throughout compilation side tables | Because the [AST](#ast) and HIR are immutable once created, we often carry extra information about them in the form of hashtables, indexed by the id of a particular node. sigil | Like a keyword but composed entirely of non-alphanumeric tokens. For example, `&` is a sigil for references. diff --git a/src/appendix/humorust.md b/src/appendix/humorust.md index 6df3b212a..8681512ed 100644 --- a/src/appendix/humorust.md +++ b/src/appendix/humorust.md @@ -3,7 +3,7 @@ What's a project without a sense of humor? And frankly some of these are enlightening? -- [Weird exprs test](https://github.com/rust-lang/rust/blob/master/tests/ui/weird-exprs.rs) +- [Weird exprs test](https://github.com/rust-lang/rust/blob/master/tests/ui/expr/weird-exprs.rs) - [Ferris Rap](https://fitzgen.com/2018/12/13/rust-raps.html) - [The Genesis of Generic Germination](https://github.com/rust-lang/rust/pull/53645#issue-210543221) - [The Bastion of the Turbofish test](https://github.com/rust-lang/rust/blob/79d8a0fcefa5134db2a94739b1d18daa01fc6e9f/src/test/ui/bastion-of-the-turbofish.rs) diff --git a/src/asm.md b/src/asm.md index eec9d448b..b5857d546 100644 --- a/src/asm.md +++ b/src/asm.md @@ -1,7 +1,5 @@ # Inline assembly - - ## Overview Inline assembly in rustc mostly revolves around taking an `asm!` macro invocation and plumbing it @@ -155,9 +153,9 @@ can't know ahead of time whether a function will require a frame/base pointer. Various tests for inline assembly are available: -- `tests/assembly/asm` +- `tests/assembly-llvm/asm` - `tests/ui/asm` -- `tests/codegen/asm-*` +- `tests/codegen-llvm/asm-*` Every architecture supported by inline assembly must have exhaustive tests in -`tests/assembly/asm` which test all combinations of register classes and types. +`tests/assembly-llvm/asm` which test all combinations of register classes and types. diff --git a/src/ast-validation.md b/src/ast-validation.md index fa0f1d954..8f10bbecf 100644 --- a/src/ast-validation.md +++ b/src/ast-validation.md @@ -1,4 +1,4 @@ -# AST Validation +# AST validation _AST validation_ is a separate AST pass that visits each item in the tree and performs simple checks. This pass diff --git a/src/autodiff/debugging.md b/src/autodiff/debugging.md new file mode 100644 index 000000000..bd46a66fa --- /dev/null +++ b/src/autodiff/debugging.md @@ -0,0 +1,113 @@ +# Reporting backend crashes + +If after a compilation failure you are greeted by a large amount of llvm-ir code, then our enzyme backend likely failed to compile your code. These cases are harder to debug, so your help is highly appreciated. Please also keep in mind that release builds are usually much more likely to work at the moment. + +The final goal here is to reproduce your bug in the enzyme [compiler explorer](https://enzyme.mit.edu/explorer/), in order to create a bug report in the [Enzyme](https://github.com/enzymead/enzyme/issues) repository. + +We have an `autodiff` flag which you can pass to `rustflags` to help with this. it will print the whole llvm-ir module, along with some `__enzyme_fwddiff` or `__enzyme_autodiff` calls. A potential workflow on linux could look like: + +## Controlling llvm-ir generation + +Before generating the llvm-ir, keep in mind two techniques that can help ensure the relevant rust code is visible for debugging: + +- **`std::hint::black_box`**: wrap rust variables or expressions in `std::hint::black_box()` to prevent rust and llvm from optimizing them away. This is useful when you need to inspect or manually manipulate specific values in the llvm-ir. +- **`extern "rust"` or `extern "c"`**: if you want to see how a specific function declaration is lowered to llvm-ir, you can declare it as `extern "rust"` or `extern "c"`. You can also look for existing `__enzyme_autodiff` or similar declarations within the generated module for examples. + +## 1) Generate an llvm-ir reproducer + +```sh +rustflags="-z autodiff=enable,printmodbefore" cargo +enzyme build --release &> out.ll +``` + +This also captures a few warnings and info messages above and below your module. open out.ll and remove every line above `; moduleid = `. Now look at the end of the file and remove everything that's not part of llvm-ir, i.e. remove errors and warnings. The last line of your llvm-ir should now start with `! = `, i.e. `!40831 = !{i32 0, i32 1037508, i32 1037538, i32 1037559}` or `!43760 = !dilocation(line: 297, column: 5, scope: !43746)`. + +The actual numbers will depend on your code. + +## 2) Check your llvm-ir reproducer + +To confirm that your previous step worked, we will use llvm's `opt` tool. find your path to the opt binary, with a path similar to `/rust/build//build/bin/opt`. also find `llvmenzyme-19.` path, similar to `/rust/build/target-tripple/enzyme/build/enzyme/llvmenzyme-19`. Please keep in mind that llvm frequently updates it's llvm backend, so the version number might be higher (20, 21, ...). Once you have both, run the following command: + +```sh + out.ll -load-pass-plugin=/path/to/llvmenzyme-19.so -passes="enzyme" -s +``` + +If the previous step succeeded, you are going to see the same error that you saw when compiling your rust code with cargo. + +If you fail to get the same error, please open an issue in the rust repository. If you succeed, congrats! the file is still huge, so let's automatically minimize it. + +## 3) Minimize your llvm-ir reproducer + +First find your `llvm-extract` binary, it's in the same folder as your opt binary. then run: + +```sh + -s --func= --recursive --rfunc="enzyme_autodiff*" --rfunc="enzyme_fwddiff*" --rfunc= out.ll -o mwe.ll +``` + +This command creates `mwe.ll`, a minimal working example. + +Please adjust the name passed with the last `--func` flag. You can either apply the `#[no_mangle]` attribute to the function you differentiate, then you can replace it with the rust name. otherwise you will need to look up the mangled function name. To do that, open `out.ll` and search for `__enzyme_fwddiff` or `__enzyme_autodiff`. the first string in that function call is the name of your function. example: + +```llvm-ir +define double @enzyme_opt_helper_0(ptr %0, i64 %1, double %2) { + %4 = call double (...) @__enzyme_fwddiff(ptr @_zn2ad3_f217h3b3b1800bd39fde3e, metadata !"enzyme_const", ptr %0, metadata !"enzyme_const", i64 %1, metadata !"enzyme_dup", double %2, double %2) + ret double %4 +} +``` + +Here, `_zn2ad3_f217h3b3b1800bd39fde3e` is the correct name. make sure to not copy the leading `@`. redo step 2) by running the `opt` command again, but this time passing `mwe.ll` as the input file instead of `out.ll`. Check if this minimized example still reproduces the crash. + +## 4) (Optional) Minimize your llvm-ir reproducer further. + +After the previous step you should have an `mwe.ll` file with ~5k loc. let's try to get it down to 50. find your `llvm-reduce` binary next to `opt` and `llvm-extract`. Copy the first line of your error message, an example could be: + +```sh +opt: /home/manuel/prog/rust/src/llvm-project/llvm/lib/ir/instructions.cpp:686: void llvm::callinst::init(llvm::functiontype*, llvm::value*, llvm::arrayref, llvm::arrayref >, const llvm::twine&): assertion `(args.size() == fty->getnumparams() || (fty->isvararg() && args.size() > fty->getnumparams())) && "calling a function with bad signature!"' failed. +``` + +If you just get a `segfault` there is no sensible error message and not much to do automatically, so continue to 5). +otherwise, create a `script.sh` file containing + +```sh +#!/bin/bash + $1 -load-pass-plugin=/path/to/llvmenzyme-19.so -passes="enzyme" \ + |& grep "/some/path.cpp:686: void llvm::callinst::init" +``` + +Experiment a bit with which error message you pass to grep. it should be long enough to make sure that the error is unique. However, for longer errors including `(` or `)` you will need to escape them correctly which can become annoying. Run + +```sh + --test=script.sh mwe.ll +``` + +If you see `input isn't interesting! verify interesting-ness test`, you got the error message in script.sh wrong, you need to make sure that grep matches your actual error. If all works out, you will see a lot of iterations, ending with a new `reduced.ll` file. Verify with `opt` that you still get the same error. + +### Advanced debugging: manual llvm-ir investigation + +Once you have a minimized reproducer (`mwe.ll` or `reduced.ll`), you can delve deeper: + +- **manual editing:** try manually rewriting the llvm-ir. for certain issues, like those involving indirect calls, you might investigate enzyme-specific intrinsics like `__enzyme_virtualreverse`. Understanding how to use these might require consulting enzyme's documentation or source code. +- **enzyme test cases:** look for relevant test cases within the [enzyme repository](https://github.com/enzymead/enzyme/tree/main/enzyme/test) that might demonstrate the correct usage of features or intrinsics related to your problem. + +## 5) Report your bug. + +Afterwards, you should be able to copy and paste your `mwe.ll` (or `reduced.ll`) example into our [compiler explorer](https://enzyme.mit.edu/explorer/). + +- Select `llvm ir` as language and `opt 20` as compiler. +- Replace the field to the right of your compiler with `-passes="enzyme"`, if it is not already set. +- Hopefully, you will see once again your now familiar error. +- Please use the share button to copy links to them. +- Please create an issue on [https://github.com/enzymead/enzyme/issues](https://github.com/enzymead/enzyme/issues) and share `mwe.ll` and (if you have it) `reduced.ll`, as well as links to the compiler explorer. Please feel free to also add your rust code or a link to it. + +#### Documenting findings + +some enzyme errors, like `"attempting to call an indirect active function whose runtime value is inactive"`, have historically caused confusion. If you investigate such an issue, even if you don't find a complete solution, please consider documenting your findings. If the insights are general to enzyme and not specific to its rust usage, contributing them to the main [enzyme documentation](https://github.com/enzymead/www) is often the best first step. You can also mention your findings in the relevant enzyme github issue or propose updates to these docs if appropriate. This helps prevent others from starting from scratch. + +With a clear reproducer and documentation, hopefully an enzyme developer will be able to fix your bug. Once that happens, the enzyme submodule inside the rust compiler will be updated, which should allow you to differentiate your rust code. Thanks for helping us to improve rust-ad. + +# Minimize rust code + +Beyond having a minimal llvm-ir reproducer, it is also helpful to have a minimal rust reproducer without dependencies. This allows us to add it as a test case to ci once we fix it, which avoids regressions for the future. + +There are a few solutions to help you with minimizing the rust reproducer. This is probably the most simple automated approach: [cargo-minimize](https://github.com/nilstrieb/cargo-minimize). + +Otherwise we have various alternatives, including [`treereduce`](https://github.com/langston-barrett/treereduce), [`halfempty`](https://github.com/googleprojectzero/halfempty), or [`picireny`](https://github.com/renatahodovan/picireny), potentially also [`creduce`](https://github.com/csmith-project/creduce). diff --git a/src/autodiff/flags.md b/src/autodiff/flags.md new file mode 100644 index 000000000..efbb9ea34 --- /dev/null +++ b/src/autodiff/flags.md @@ -0,0 +1,45 @@ +# Supported `RUSTFLAGS` + +To support you while debugging or profiling, we have added support for an experimental `-Z autodiff` rustc flag (which can be passed to cargo via `RUSTFLAGS`), which allow changing the behaviour of Enzyme, without recompiling rustc. We currently support the following values for `autodiff`. + +### Debug Flags + +```text +PrintTA // Print TypeAnalysis information +PrintTAFn // Print TypeAnalysis information for a specific function +PrintAA // Print ActivityAnalysis information +Print // Print differentiated functions while they are being generated and optimized +PrintPerf // Print AD related Performance warnings +PrintModBefore // Print the whole LLVM-IR module directly before running AD +PrintModAfter // Print the whole LLVM-IR module after running AD, before optimizations +PrintModFinal // Print the whole LLVM-IR module after running optimizations and AD +LooseTypes // Risk incorrect derivatives instead of aborting when missing Type Info +``` + +
+ +`LooseTypes` is often helpful to get rid of Enzyme errors stating `Can not deduce type of ` and to be able to run some code. But please keep in mind that this flag absolutely has the chance to cause incorrect gradients. Even worse, the gradients might be correct for certain input values, but not for others. So please create issues about such bugs and only use this flag temporarily while you wait for your bug to be fixed. + +
+ +### Benchmark flags + +For performance experiments and benchmarking we also support + +```text +NoPostopt // We won't optimize the LLVM-IR Module after AD +RuntimeActivity // Enables the runtime activity feature from Enzyme +Inline // Instructs Enzyme to maximize inlining as far as possible, beyond LLVM's default +``` + +You can combine multiple `autodiff` values using a comma as separator: + +```bash +RUSTFLAGS="-Z autodiff=Enable,LooseTypes,PrintPerf" cargo +enzyme build +``` + +Using `-Zautodiff=Enable` will allow using autodiff and update your normal rustc compilation pipeline: + +1. Run your selected compilation pipeline. If you selected a release build, we will disable vectorization and loop unrolling. +2. Differentiate your functions. +3. Run your selected compilation pipeline again on the whole module. This time we do not disable vectorization or loop unrolling. diff --git a/src/autodiff/installation.md b/src/autodiff/installation.md new file mode 100644 index 000000000..ddbb3a054 --- /dev/null +++ b/src/autodiff/installation.md @@ -0,0 +1,88 @@ +# Installation + +In the near future, `std::autodiff` should become available in nightly builds for users. As a contributor however, you will still need to build rustc from source. Please be aware that the msvc target is not supported at the moment, all other tier 1 targets should work. Please open an issue if you encounter any problems on a supported tier 1 target, or if you successfully build this project on a tier2/tier3 target. + +## Build instructions + +First you need to clone and configure the Rust repository: +```bash +git clone git@github.com:rust-lang/rust +cd rust +./configure --enable-llvm-link-shared --enable-llvm-plugins --enable-llvm-enzyme --release-channel=nightly --enable-llvm-assertions --enable-clang --enable-lld --enable-option-checking --enable-ninja --disable-docs +``` + +Afterwards you can build rustc using: +```bash +./x build --stage 1 library +``` + +Afterwards rustc toolchain link will allow you to use it through cargo: +``` +rustup toolchain link enzyme build/host/stage1 +rustup toolchain install nightly # enables -Z unstable-options +``` + +You can then run our test cases: + +```bash +./x test --stage 1 tests/codegen-llvm/autodiff +./x test --stage 1 tests/pretty/autodiff +./x test --stage 1 tests/ui/autodiff +./x test --stage 1 tests/ui/feature-gates/feature-gate-autodiff.rs +``` + +Autodiff is still experimental, so if you want to use it in your own projects, you will need to add `lto="fat"` to your Cargo.toml +and use `RUSTFLAGS="-Zautodiff=Enable" cargo +enzyme` instead of `cargo` or `cargo +nightly`. + +## Compiler Explorer and dist builds + +Our compiler explorer instance can be updated to a newer rustc in a similar way. First, prepare a docker instance. +```bash +docker run -it ubuntu:22.04 +export CC=clang CXX=clang++ +apt update +apt install wget vim python3 git curl libssl-dev pkg-config lld ninja-build cmake clang build-essential +``` +Then build rustc in a slightly altered way: +```bash +git clone https://github.com/rust-lang/rust +cd rust +./configure --enable-llvm-link-shared --enable-llvm-plugins --enable-llvm-enzyme --release-channel=nightly --enable-llvm-assertions --enable-clang --enable-lld --enable-option-checking --enable-ninja --disable-docs +./x dist +``` +We then copy the tarball to our host. The dockerid is the newest entry under `docker ps -a`. +```bash +docker cp :/rust/build/dist/rust-nightly-x86_64-unknown-linux-gnu.tar.gz rust-nightly-x86_64-unknown-linux-gnu.tar.gz +``` +Afterwards we can create a new (pre-release) tag on the EnzymeAD/rust repository and make a PR against the EnzymeAD/enzyme-explorer repository to update the tag. +Remember to ping `tgymnich` on the PR to run his update script. Note: We should archive EnzymeAD/rust and update the instructions here. The explorer should soon +be able to get the rustc toolchain from the official rust servers. + + +## Build instruction for Enzyme itself + +Following the Rust build instruction above will build LLVMEnzyme, LLDEnzyme, and ClangEnzyme along with the Rust compiler. +We recommend that approach, if you just want to use any of them and have no experience with cmake. +However, if you prefer to just build Enzyme without Rust, then these instructions might help. + +```bash +git clone git@github.com:llvm/llvm-project +cd llvm-project +mkdir build +cd build +cmake -G Ninja ../llvm -DLLVM_TARGETS_TO_BUILD="host" -DLLVM_ENABLE_ASSERTIONS=ON -DLLVM_ENABLE_PROJECTS="clang;lld" -DLLVM_ENABLE_RUNTIMES="openmp" -DLLVM_ENABLE_PLUGINS=ON -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=. +ninja +ninja install +``` +This gives you a working LLVM build, now we can continue with building Enzyme. +Leave the `llvm-project` folder, and execute the following commands: +```bash +git clone git@github.com:EnzymeAD/Enzyme +cd Enzyme/enzyme +mkdir build +cd build +cmake .. -G Ninja -DLLVM_DIR=/llvm-project/build/lib/cmake/llvm/ -DLLVM_EXTERNAL_LIT=/llvm-project/llvm/utils/lit/lit.py -DCMAKE_BUILD_TYPE=Release -DCMAKE_EXPORT_COMPILE_COMMANDS=YES -DBUILD_SHARED_LIBS=ON +ninja +``` +This will build Enzyme, and you can find it in `Enzyme/enzyme/build/lib/Enzyme.so`. (Endings might differ based on your OS). + diff --git a/src/autodiff/internals.md b/src/autodiff/internals.md new file mode 100644 index 000000000..c1b31a0e4 --- /dev/null +++ b/src/autodiff/internals.md @@ -0,0 +1,27 @@ +The `std::autodiff` module in Rust allows differentiable programming: + +```rust +#![feature(autodiff)] +use std::autodiff::*; + +// f(x) = x * x, f'(x) = 2.0 * x +// bar therefore returns (x * x, 2.0 * x) +#[autodiff_reverse(bar, Active, Active)] +fn foo(x: f32) -> f32 { x * x } + +fn main() { + assert_eq!(bar(3.0, 1.0), (9.0, 6.0)); + assert_eq!(bar(4.0, 1.0), (16.0, 8.0)); +} +``` + +The detailed documentation for the `std::autodiff` module is available at [std::autodiff](https://doc.rust-lang.org/std/autodiff/index.html). + +Differentiable programing is used in various fields like numerical computing, [solid mechanics][ratel], [computational chemistry][molpipx], [fluid dynamics][waterlily] or for Neural Network training via Backpropagation, [ODE solver][diffsol], [differentiable rendering][libigl], [quantum computing][catalyst], and climate simulations. + +[ratel]: https://gitlab.com/micromorph/ratel +[molpipx]: https://arxiv.org/abs/2411.17011v +[waterlily]: https://github.com/WaterLily-jl/WaterLily.jl +[diffsol]: https://github.com/martinjrobins/diffsol +[libigl]: https://github.com/alecjacobson/libigl-enzyme-example?tab=readme-ov-file#run +[catalyst]: https://github.com/PennyLaneAI/catalyst diff --git a/src/autodiff/limitations.md b/src/autodiff/limitations.md new file mode 100644 index 000000000..90afbd51f --- /dev/null +++ b/src/autodiff/limitations.md @@ -0,0 +1,27 @@ +# Current limitations + +## Safety and Soundness + +Enzyme currently assumes that the user passes shadow arguments (`dx`, `dy`, ...) of appropriate size. Under Reverse Mode, we additionally assume that shadow arguments are mutable. In Reverse Mode we adjust the outermost pointer or reference to be mutable. Therefore `&f32` will receive the shadow type `&mut f32`. However, we do not check length for other types than slices (e.g. enums, Vec). We also do not enforce mutability of inner references, but will warn if we recognize them. We do intend to add additional checks over time. + +## ABI adjustments + +In some cases, a function parameter might get lowered in a way that we currently don't handle correctly, leading to a compile time type mismatch in the `rustc_codegen_llvm` backend. Here are some [examples](https://github.com/EnzymeAD/rust/issues/105). + +## Compile Times + +Enzyme will often achieve excellent runtime performance, but might increase your compile time by a large factor. For Rust, we already have made significant improvements and have a list of further improvements planed - please reach out if you have time to help here. + +### Type Analysis + +Most of the times, Type Analysis (TA) is the reason of large (>5x) compile time increases when using Enzyme. This poster explains why we need to run Type Analysis in the bottom left part: [Poster Link](https://c.wsmoses.com/posters/Enzyme-llvmdev.pdf). + +We intend to increase the number of locations where we pass down Type information based on Rust types, which in turn will reduce the number of locations where Enzyme has to run Type Analysis, which will help compile times. + +### Duplicated Optimizations + +The key reason for Enzyme offering often excellent performance is that Enzyme differentiates already optimized LLVM-IR. However, we also (have to) run LLVM's optimization pipeline after differentiating, to make sure that the code which Enzyme generates is optimized properly. As a result you should have excellent runtime performance (please fill an issue if not), but at a compile time cost for running optimizations twice. + +### Fat-LTO + +The usage of `#[autodiff(...)]` currently requires compiling your project with Fat-LTO. We technically only need LTO if the function being differentiated calls functions in other compilation units. Therefore, other solutions are possible, but this is the most simple one to get started. diff --git a/src/backend/backend-agnostic.md b/src/backend/backend-agnostic.md index 3521c00f5..2fdda4eda 100644 --- a/src/backend/backend-agnostic.md +++ b/src/backend/backend-agnostic.md @@ -1,7 +1,5 @@ # Backend Agnostic Codegen - - [`rustc_codegen_ssa`] provides an abstract interface for all backends to implement, namely LLVM, [Cranelift], and [GCC]. @@ -45,7 +43,7 @@ heavily on other parts of the crate. The separation of the code must not affect the logic of the code nor its performance. For these reasons, the separation process involves two transformations that -have to be done at the same time for the resulting code to compile : +have to be done at the same time for the resulting code to compile: 1. replace all the LLVM-specific types by generics inside function signatures and structure definitions; diff --git a/src/backend/debugging.md b/src/backend/debugging.md index 275a1777a..4f8712dfa 100644 --- a/src/backend/debugging.md +++ b/src/backend/debugging.md @@ -38,7 +38,7 @@ which means that LLVM assertion failures can show up as compiler crashes (not ICEs but "real" crashes) and other sorts of weird behavior. If you are encountering these, it is a good idea to try using a compiler with LLVM assertions enabled - either an "alt" nightly or a compiler you build yourself -by setting `[llvm] assertions=true` in your config.toml - and see whether +by setting `[llvm] assertions=true` in your bootstrap.toml - and see whether anything turns up. The rustc build process builds the LLVM tools into @@ -56,7 +56,7 @@ These tools include: By default, the Rust build system does not check for changes to the LLVM source code or its build configuration settings. So, if you need to rebuild the LLVM that is linked -into `rustc`, first delete the file `llvm-finished-building`, which should be located +into `rustc`, first delete the file `.llvm-stamp`, which should be located in `build//llvm/`. The default rustc compilation pipeline has multiple codegen units, which is @@ -160,7 +160,7 @@ from `./build//llvm/bin/` with the LLVM IR emitted by rustc. When investigating the implementation of LLVM itself, you should be aware of its [internal debug infrastructure][llvm-debug]. This is provided in LLVM Debug builds, which you enable for rustc -LLVM builds by changing this setting in the config.toml: +LLVM builds by changing this setting in the bootstrap.toml: ``` [llvm] # Indicates whether the LLVM assertions are enabled or not diff --git a/src/backend/implicit-caller-location.md b/src/backend/implicit-caller-location.md index 17158497d..9ca4bcab0 100644 --- a/src/backend/implicit-caller-location.md +++ b/src/backend/implicit-caller-location.md @@ -1,6 +1,4 @@ -# Implicit Caller Location - - +# Implicit caller location Approved in [RFC 2091], this feature enables the accurate reporting of caller location during panics initiated from functions like `Option::unwrap`, `Result::expect`, and `Index::index`. This feature @@ -8,7 +6,7 @@ adds the [`#[track_caller]`][attr-reference] attribute for functions, the [`caller_location`][intrinsic] intrinsic, and the stabilization-friendly [`core::panic::Location::caller`][wrapper] wrapper. -## Motivating Example +## Motivating example Take this example program: @@ -39,7 +37,7 @@ These error messages are achieved through a combination of changes to `panic!` i of `core::panic::Location::caller` and a number of `#[track_caller]` annotations in the standard library which propagate caller information. -## Reading Caller Location +## Reading caller location Previously, `panic!` made use of the `file!()`, `line!()`, and `column!()` macros to construct a [`Location`] pointing to where the panic occurred. These macros couldn't be given an overridden @@ -51,7 +49,7 @@ was expanded. This function is itself annotated with `#[track_caller]` and wraps [`caller_location`][intrinsic] compiler intrinsic implemented by rustc. This intrinsic is easiest explained in terms of how it works in a `const` context. -## Caller Location in `const` +## Caller location in `const` There are two main phases to returning the caller location in a const context: walking up the stack to find the right location and allocating a const value to return. @@ -138,7 +136,7 @@ fn main() { } ``` -### Dynamic Dispatch +### Dynamic dispatch In codegen contexts we have to modify the callee ABI to pass this information down the stack, but the attribute expressly does *not* modify the type of the function. The ABI change must be @@ -156,7 +154,7 @@ probably the best we can do without modifying fully-stabilized type signatures. > whether we'll be called in a const context (safe to ignore shim) or in a codegen context (unsafe > to ignore shim). Even if we did know, the results from const and codegen contexts must agree. -## The Attribute +## The attribute The `#[track_caller]` attribute is checked alongside other codegen attributes to ensure the function: diff --git a/src/backend/libs-and-metadata.md b/src/backend/libs-and-metadata.md index b0823b9a5..aa1d64470 100644 --- a/src/backend/libs-and-metadata.md +++ b/src/backend/libs-and-metadata.md @@ -1,4 +1,4 @@ -# Libraries and Metadata +# Libraries and metadata When the compiler sees a reference to an external crate, it needs to load some information about that crate. This chapter gives an overview of that process, @@ -28,8 +28,8 @@ format is specific to `rustc`, and may change over time. This file contains: [`-C embed-bitcode=no`][embed-bitcode] CLI option to improve compile times and reduce disk space if LTO is not needed. * `rustc` [metadata], in a file named `lib.rmeta`. -* A symbol table, which is generally a list of symbols with offsets to the - object file that contain that symbol. This is pretty standard for archive +* A symbol table, which is essentially a list of symbols with offsets to the + object files that contain that symbol. This is pretty standard for archive files. [archive file]: https://en.wikipedia.org/wiki/Ar_(Unix) @@ -42,16 +42,15 @@ format is specific to `rustc`, and may change over time. This file contains: ### dylib A `dylib` is a platform-specific shared library. It includes the `rustc` -[metadata] in a special link section called `.rustc` in a compressed format. +[metadata] in a special link section called `.rustc`. ### rmeta -An `rmeta` file is custom binary format that contains the [metadata] for the -crate. This file can be used for fast "checks" of a project by skipping all -code generation (as is done with `cargo check`), collecting enough information -for documentation (as is done with `cargo doc`), or for -[pipelining](#pipelining). This file is created if the -[`--emit=metadata`][emit] CLI option is used. +An `rmeta` file is a custom binary format that contains the [metadata] for the +crate. This file can be used for fast "checks" of a project by skipping all code +generation (as is done with `cargo check`), collecting enough information for +documentation (as is done with `cargo doc`), or for [pipelining](#pipelining). +This file is created if the [`--emit=metadata`][emit] CLI option is used. `rmeta` files do not support linking, since they do not contain compiled object files. @@ -60,8 +59,8 @@ object files. ## Metadata -The metadata contains a wide swath of different elements. This guide will not -go into detail of every field it contains. You are encouraged to browse the +The metadata contains a wide swath of different elements. This guide will not go +into detail about every field it contains. You are encouraged to browse the [`CrateRoot`] definition to get a sense of the different elements it contains. Everything about metadata encoding and decoding is in the [`rustc_metadata`] package. @@ -110,7 +109,7 @@ See [`compute_hir_hash`] for where the hash is actually computed. [SVH]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_data_structures/svh/struct.Svh.html [incremental compilation]: ../queries/incremental-compilation.md -[`compute_hir_hash`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_ast_lowering/struct.LoweringContext.html#method.compute_hir_hash +[`compute_hir_hash`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_ast_lowering/fn.compute_hir_hash.html ### Stable Crate Id @@ -122,9 +121,9 @@ much more. By default, all Rust symbols are mangled and incorporate the stable crate id. This allows multiple versions of the same crate to be included together. Cargo -automatically generates `-C metadata` hashes based on a variety of factors, -like the package version, source, and the target kind (a lib and test can have -the same crate name, so they need to be disambiguated). +automatically generates `-C metadata` hashes based on a variety of factors, like +the package version, source, and target kind (a lib and test can have the same +crate name, so they need to be disambiguated). [`StableCrateId`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_span/def_id/struct.StableCrateId.html [`StableCrateId::new`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_span/def_id/struct.StableCrateId.html#method.new @@ -154,7 +153,7 @@ will also look at the [sysroot] to find dependencies. As crates are loaded, they are kept in the [`CStore`] with the crate metadata wrapped in the [`CrateMetadata`] struct. After resolution and expansion, the -`CStore` will make its way into the [`GlobalCtxt`] for the rest of +`CStore` will make its way into the [`GlobalCtxt`] for the rest of the compilation. [name resolution]: ../name-resolution.md diff --git a/src/backend/monomorph.md b/src/backend/monomorph.md index 7ebb4d2b1..e9d98597e 100644 --- a/src/backend/monomorph.md +++ b/src/backend/monomorph.md @@ -1,7 +1,5 @@ # Monomorphization - - As you probably know, Rust has a very expressive type system that has extensive support for generic types. But of course, assembly is not generic, so we need to figure out the concrete types of all the generics before the code can diff --git a/src/backend/updating-llvm.md b/src/backend/updating-llvm.md index 92d4ce32f..ebef15d40 100644 --- a/src/backend/updating-llvm.md +++ b/src/backend/updating-llvm.md @@ -1,7 +1,5 @@ # Updating LLVM - - Rust supports building against multiple LLVM versions: @@ -116,14 +114,14 @@ so let's go through each in detail. at the time of the branch, and the remaining part is the current date. -2. Apply Rust-specific patches to the llvm-project repository. +1. Apply Rust-specific patches to the llvm-project repository. All features and bugfixes are upstream, but there's often some weird build-related patches that don't make sense to upstream. These patches are typically the latest patches in the rust-lang/llvm-project branch that rustc is currently using. -3. Build the new LLVM in the `rust` repository. +1. Build the new LLVM in the `rust` repository. To do this, you'll want to update the `src/llvm-project` repository to your branch, and the revision you've created. @@ -144,14 +142,14 @@ so let's go through each in detail. Note that `profile = "compiler"` and other defaults set by `./x setup` download LLVM from CI instead of building it from source. You should disable this temporarily to make sure your changes are being used. - This is done by having the following setting in `config.toml`: + This is done by having the following setting in `bootstrap.toml`: ```toml [llvm] download-ci-llvm = false ``` -4. Test for regressions across other platforms. LLVM often has at least one bug +1. Test for regressions across other platforms. LLVM often has at least one bug for non-tier-1 architectures, so it's good to do some more testing before sending this to bors! If you're low on resources you can send the PR as-is now to bors, though, and it'll get tested anyway. @@ -170,22 +168,17 @@ so let's go through each in detail. * `./src/ci/docker/run.sh dist-various-2` * `./src/ci/docker/run.sh armhf-gnu` -5. Prepare a PR to `rust-lang/rust`. Work with maintainers of +1. Prepare a PR to `rust-lang/rust`. Work with maintainers of `rust-lang/llvm-project` to get your commit in a branch of that repository, and then you can send a PR to `rust-lang/rust`. You'll change at least `src/llvm-project` and will likely also change [`llvm-wrapper`] as well. - + > For prior art, here are some previous LLVM updates: - > - [LLVM 11](https://github.com/rust-lang/rust/pull/73526) - > - [LLVM 12](https://github.com/rust-lang/rust/pull/81451) - > - [LLVM 13](https://github.com/rust-lang/rust/pull/87570) - > - [LLVM 14](https://github.com/rust-lang/rust/pull/93577) - > - [LLVM 15](https://github.com/rust-lang/rust/pull/99464) - > - [LLVM 16](https://github.com/rust-lang/rust/pull/109474) > - [LLVM 17](https://github.com/rust-lang/rust/pull/115959) > - [LLVM 18](https://github.com/rust-lang/rust/pull/120055) > - [LLVM 19](https://github.com/rust-lang/rust/pull/127513) + > - [LLVM 20](https://github.com/rust-lang/rust/pull/135763) Note that sometimes it's easiest to land [`llvm-wrapper`] compatibility as a PR before actually updating `src/llvm-project`. @@ -194,7 +187,7 @@ so let's go through each in detail. others interested in trying out the new LLVM can benefit from work you've done to update the C++ bindings. -3. Over the next few months, +1. Over the next few months, LLVM will continually push commits to its `release/a.b` branch. We will often want to have those bug fixes as well. The merge process for that is to use `git merge` itself to merge LLVM's @@ -202,9 +195,9 @@ so let's go through each in detail. This is typically done multiple times when necessary while LLVM's release branch is baking. -4. LLVM then announces the release of version `a.b`. +1. LLVM then announces the release of version `a.b`. -5. After LLVM's official release, +1. After LLVM's official release, we follow the process of creating a new branch on the rust-lang/llvm-project repository again, this time with a new date. diff --git a/src/borrow_check/moves_and_initialization/move_paths.md b/src/borrow_check/moves_and_initialization/move_paths.md index ad9c75d62..95518fbc0 100644 --- a/src/borrow_check/moves_and_initialization/move_paths.md +++ b/src/borrow_check/moves_and_initialization/move_paths.md @@ -1,7 +1,5 @@ # Move paths - - In reality, it's not enough to track initialization at the granularity of local variables. Rust also allows us to do moves and initialization at the field granularity: diff --git a/src/borrow_check/region_inference.md b/src/borrow_check/region_inference.md index 85e71b4fa..0d55ab955 100644 --- a/src/borrow_check/region_inference.md +++ b/src/borrow_check/region_inference.md @@ -1,7 +1,5 @@ # Region inference (NLL) - - The MIR-based region checking code is located in [the `rustc_mir::borrow_check` module][nll]. diff --git a/src/borrow_check/region_inference/closure_constraints.md b/src/borrow_check/region_inference/closure_constraints.md index be279b773..b95c9f723 100644 --- a/src/borrow_check/region_inference/closure_constraints.md +++ b/src/borrow_check/region_inference/closure_constraints.md @@ -31,9 +31,9 @@ It starts by calling `fn try_promote_type_test_subject`. This function takes the We then promote the `lower_bound` into the context of the caller. If the lower bound is equal to a placeholder, we replace it with `'static` -We then look at all universal regions `uv` which are required to outlive `lower_bound`, i.e. for which borrow checking adding region constraints. For each of these we then emit a `ClosureOutlivesRequirement` for non-local universal regions which are known to outlive `uv`. +We then look at all universal regions `uv` which are required to be outlived by `lower_bound`, i.e. for which borrow checking added region constraints. For each of these we then emit a `ClosureOutlivesRequirement` for all non-local universal regions which are known to outlive `uv`. -As we've already built the region graph of the closure at this point and emitted errors if that one is inconsistent, we are also able to assume that the outlive constraints `uv: lower_bound` hold. +As we've already built the region graph of the closure at this point and separately check that it is consistent, we are also able to assume the outlive constraints `uv: lower_bound` here. So if we have a type-outlives bounds we can't prove, e.g. `T: 'local_infer`, we use the region graph to go to universal variables `'a` with `'a: local_infer`. In case `'a` are local, we then use the assumed outlived constraints to go to non-local ones. diff --git a/src/borrow_check/region_inference/constraint_propagation.md b/src/borrow_check/region_inference/constraint_propagation.md index 4c30d25e0..c3f8c03cb 100644 --- a/src/borrow_check/region_inference/constraint_propagation.md +++ b/src/borrow_check/region_inference/constraint_propagation.md @@ -1,7 +1,5 @@ # Constraint propagation - - The main work of the region inference is **constraint propagation**, which is done in the [`propagate_constraints`] function. There are three sorts of constraints that are used in NLL, and we'll explain how diff --git a/src/borrow_check/region_inference/lifetime_parameters.md b/src/borrow_check/region_inference/lifetime_parameters.md index fadfac404..2d337dbc0 100644 --- a/src/borrow_check/region_inference/lifetime_parameters.md +++ b/src/borrow_check/region_inference/lifetime_parameters.md @@ -1,7 +1,5 @@ # Universal regions - - "Universal regions" is the name that the code uses to refer to "named lifetimes" -- e.g., lifetime parameters and `'static`. The name derives from the fact that such lifetimes are "universally quantified" diff --git a/src/borrow_check/region_inference/member_constraints.md b/src/borrow_check/region_inference/member_constraints.md index fd7c87ffc..2804c9772 100644 --- a/src/borrow_check/region_inference/member_constraints.md +++ b/src/borrow_check/region_inference/member_constraints.md @@ -1,7 +1,5 @@ # Member constraints - - A member constraint `'m member of ['c_1..'c_N]` expresses that the region `'m` must be *equal* to some **choice regions** `'c_i` (for some `i`). These constraints cannot be expressed by users, but they diff --git a/src/borrow_check/region_inference/placeholders_and_universes.md b/src/borrow_check/region_inference/placeholders_and_universes.md index 91c8c4526..11fd2a5fc 100644 --- a/src/borrow_check/region_inference/placeholders_and_universes.md +++ b/src/borrow_check/region_inference/placeholders_and_universes.md @@ -1,7 +1,5 @@ # Placeholders and universes - - From time to time we have to reason about regions that we can't concretely know. For example, consider this program: diff --git a/src/borrow_check/two_phase_borrows.md b/src/borrow_check/two_phase_borrows.md index bcd487821..b77ae0946 100644 --- a/src/borrow_check/two_phase_borrows.md +++ b/src/borrow_check/two_phase_borrows.md @@ -76,7 +76,7 @@ borrow. [`AutoBorrow`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/adjustment/enum.AutoBorrow.html [converted]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_mir_build/thir/cx/expr/trait.ToBorrowKind.html#method.to_borrow_kind [`BorrowKind`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/mir/enum.BorrowKind.html -[`GatherBorrows`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/mir/visit/trait.Visitor.html#method.visit_local +[`GatherBorrows`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_borrowck/borrow_set/struct.GatherBorrows.html [`BorrowData`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_borrowck/borrow_set/struct.BorrowData.html ## Checking two-phase borrows diff --git a/src/bug-fix-procedure.md b/src/bug-fix-procedure.md index 4857cf5e0..6b13c9702 100644 --- a/src/bug-fix-procedure.md +++ b/src/bug-fix-procedure.md @@ -1,6 +1,4 @@ -# Procedures for Breaking Changes - - +# Procedures for breaking changes This page defines the best practices procedure for making bug fixes or soundness corrections in the compiler that can cause existing code to stop compiling. This @@ -80,41 +78,11 @@ approachable and practical; it may make sense to direct users to an RFC or some other issue for the full details. The issue also serves as a place where users can comment with questions or other concerns. -A template for these breaking-change tracking issues can be found below. An -example of how such an issue should look can be [found +A template for these breaking-change tracking issues can be found +[here][template]. An example of how such an issue should look can be [found here][breaking-change-issue]. -The issue should be tagged with (at least) `B-unstable` and `T-compiler`. - -### Tracking issue template - -This is a template to use for tracking issues: - -``` -This is the **summary issue** for the `YOUR_LINT_NAME_HERE` -future-compatibility warning and other related errors. The goal of -this page is describe why this change was made and how you can fix -code that is affected by it. It also provides a place to ask questions -or register a complaint if you feel the change should not be made. For -more information on the policy around future-compatibility warnings, -see our [breaking change policy guidelines][guidelines]. - -[guidelines]: LINK_TO_THIS_RFC - -#### What is the warning for? - -*Describe the conditions that trigger the warning and how they can be -fixed. Also explain why the change was made.** - -#### When will this warning become a hard error? - -At the beginning of each 6-week release cycle, the Rust compiler team -will review the set of outstanding future compatibility warnings and -nominate some of them for **Final Comment Period**. Toward the end of -the cycle, we will review any comments and make a final determination -whether to convert the warning into a hard error or remove it -entirely. -``` +[template]: https://github.com/rust-lang/rust/issues/new?template=tracking_issue_future.md ### Issuing future compatibility warnings @@ -227,7 +195,7 @@ that we use for unstable features: Ideally, breaking changes should have landed on the **stable branch** of the compiler before they are finalized. - + ### Removing a lint diff --git a/src/building/bootstrapping/bootstrap-in-dependencies.md b/src/building/bootstrapping/bootstrap-in-dependencies.md new file mode 100644 index 000000000..68c5c2386 --- /dev/null +++ b/src/building/bootstrapping/bootstrap-in-dependencies.md @@ -0,0 +1,53 @@ +# `cfg(bootstrap)` in compiler dependencies + +The rust compiler uses some external crates that can run into cyclic dependencies with the compiler itself: the compiler needs an updated crate to build, but the crate needs an updated compiler. This page describes how `#[cfg(bootstrap)]` can be used to break this cycle. + +## Enabling `#[cfg(bootstrap)]` + +Usually the use of `#[cfg(bootstrap)]` in an external crate causes a warning: + +``` +warning: unexpected `cfg` condition name: `bootstrap` + --> src/main.rs:1:7 + | +1 | #[cfg(bootstrap)] + | ^^^^^^^^^ + | + = help: expected names are: `docsrs`, `feature`, and `test` and 31 more + = help: consider using a Cargo feature instead + = help: or consider adding in `Cargo.toml` the `check-cfg` lint config for the lint: + [lints.rust] + unexpected_cfgs = { level = "warn", check-cfg = ['cfg(bootstrap)'] } + = help: or consider adding `println!("cargo::rustc-check-cfg=cfg(bootstrap)");` to the top of the `build.rs` + = note: see for more information about checking conditional configuration + = note: `#[warn(unexpected_cfgs)]` on by default +``` + +This warning can be silenced by adding these lines to the project's `Cargo.toml`: + +```toml +[lints.rust] +unexpected_cfgs = { level = "warn", check-cfg = ['cfg(bootstrap)'] } +``` + +Now `#[cfg(bootstrap)]` can be used in the crate just like it can be in the compiler: when the bootstrap compiler is used, code annotated with `#[cfg(bootstrap)]` is compiled, otherwise code annotated with `#[cfg(not(bootstrap))]` is compiled. + +## The update dance + +As a concrete example we'll use a change where the `#[naked]` attribute was made into an unsafe attribute, which caused a cyclic dependency with the `compiler-builtins` crate. + +### Step 1: accept the new behavior in the compiler ([#139797](https://github.com/rust-lang/rust/pull/139797)) + +In this example it is possible to accept both the old and new behavior at the same time by disabling an error. + +### Step 2: update the crate ([#821](https://github.com/rust-lang/compiler-builtins/pull/821)) + +Now in the crate, use `#[cfg(bootstrap)]` to use the old behavior, or `#[cfg(not(bootstrap))]` to use the new behavior. + +### Step 3: update the crate version used by the compiler ([#139934](https://github.com/rust-lang/rust/pull/139934)) + +For `compiler-builtins` this meant a version bump, in other cases it may be a git submodule update. + +### Step 4: remove the old behavior from the compiler ([#139753](https://github.com/rust-lang/rust/pull/139753)) + +The updated crate can now be used. In this example that meant that the old behavior could be removed. diff --git a/src/building/bootstrapping/debugging-bootstrap.md b/src/building/bootstrapping/debugging-bootstrap.md new file mode 100644 index 000000000..9c5ebbd36 --- /dev/null +++ b/src/building/bootstrapping/debugging-bootstrap.md @@ -0,0 +1,192 @@ +# Debugging bootstrap + +There are two main ways to debug bootstrap itself. The first is through println logging, and the second is through the `tracing` feature. + +> FIXME: this section should be expanded + +## `println` logging + +Bootstrap has extensive unstructured logging. Most of it is gated behind the `--verbose` flag (pass `-vv` for even more detail). + +If you want to know which `Step` ran a command, you could invoke bootstrap like so: + +``` +$ ./x dist rustc --dry-run -vv +learning about cargo +running: RUSTC_BOOTSTRAP="1" "/home/jyn/src/rust2/build/x86_64-unknown-linux-gnu/stage0/bin/cargo" "metadata" "--format-version" "1" "--no-deps" "--manifest-path" "/home/jyn/src/rust2/Cargo.toml" (failure_mode=Exit) (created at src/bootstrap/src/core/metadata.rs:81:25, executed at src/bootstrap/src/core/metadata.rs:92:50) +running: RUSTC_BOOTSTRAP="1" "/home/jyn/src/rust2/build/x86_64-unknown-linux-gnu/stage0/bin/cargo" "metadata" "--format-version" "1" "--no-deps" "--manifest-path" "/home/jyn/src/rust2/library/Cargo.toml" (failure_mode=Exit) (created at src/bootstrap/src/core/metadata.rs:81:25, executed at src/bootstrap/src/core/metadata.rs:92:50) +> Assemble { target_compiler: Compiler { stage: 1, host: x86_64-unknown-linux-gnu } } + > Libdir { compiler: Compiler { stage: 1, host: x86_64-unknown-linux-gnu }, target: x86_64-unknown-linux-gnu } + > Sysroot { compiler: Compiler { stage: 1, host: x86_64-unknown-linux-gnu }, force_recompile: false } +Removing sysroot /home/jyn/src/rust2/build/tmp-dry-run/x86_64-unknown-linux-gnu/stage1 to avoid caching bugs + < Sysroot { compiler: Compiler { stage: 1, host: x86_64-unknown-linux-gnu }, force_recompile: false } + < Libdir { compiler: Compiler { stage: 1, host: x86_64-unknown-linux-gnu }, target: x86_64-unknown-linux-gnu } +... +``` + +This will go through all the recursive dependency calculations, where `Step`s internally call `builder.ensure()`, without actually running cargo or the compiler. + +In some cases, even this may not be enough logging (if so, please add more!). In that case, you can omit `--dry-run`, which will show the normal output inline with the debug logging: + +``` + c Sysroot { compiler: Compiler { stage: 0, host: x86_64-unknown-linux-gnu }, force_recompile: false } +using sysroot /home/jyn/src/rust2/build/x86_64-unknown-linux-gnu/stage0-sysroot +Building stage0 library artifacts (x86_64-unknown-linux-gnu) +running: cd "/home/jyn/src/rust2" && env ... RUSTC_VERBOSE="2" RUSTC_WRAPPER="/home/jyn/src/rust2/build/bootstrap/debug/rustc" "/home/jyn/src/rust2/build/x86_64-unknown-linux-gnu/stage0/bin/cargo" "build" "--target" "x86_64-unknown-linux-gnu" "-Zbinary-dep-depinfo" "-Zroot-dir=/home/jyn/src/rust2" "-v" "-v" "--manifest-path" "/home/jyn/src/rust2/library/sysroot/Cargo.toml" "--message-format" "json-render-diagnostics" + 0.293440230s INFO prepare_target{force=false package_id=sysroot v0.0.0 (/home/jyn/src/rust2/library/sysroot) target="sysroot"}: cargo::core::compiler::fingerprint: fingerprint error for sysroot v0.0.0 (/home/jyn/src/rust2/library/sysroot)/Build/TargetInner { name_inferred: true, ..: lib_target("sysroot", ["lib"], "/home/jyn/src/rust2/library/sysroot/src/lib.rs", Edition2021) } +... +``` + +In most cases this should not be necessary. + +TODO: we should convert all this to structured logging so it's easier to control precisely. + +## `tracing` in bootstrap + +Bootstrap has conditional [`tracing`][tracing] setup to provide structured logging. + +[tracing]: https://docs.rs/tracing/0.1.41/tracing/index.html + +### Enabling `tracing` output + +Bootstrap will conditionally build `tracing` support and enable `tracing` output if the `BOOTSTRAP_TRACING` env var is set. + +#### Basic usage + +Example basic usage[^just-trace]: + +[^just-trace]: It is not recommended to use *just* `BOOTSTRAP_TRACING=TRACE` because that will dump *everything* at `TRACE` level, including logs intentionally gated behind custom targets as they are too verbose even for `TRACE` level by default. + +```bash +$ BOOTSTRAP_TRACING=bootstrap=TRACE ./x build library --stage 1 +``` + +Example output[^unstable]: + +``` +$ BOOTSTRAP_TRACING=bootstrap=TRACE ./x check src/bootstrap/ +Building bootstrap + Compiling bootstrap v0.0.0 (/home/joe/repos/rust/src/bootstrap) + Finished `dev` profile [unoptimized] target(s) in 2.74s + DEBUG bootstrap parsing flags + bootstrap::core::config::flags::Flags::parse args=["check", "src/bootstrap/"] + DEBUG bootstrap parsing config based on flags + DEBUG bootstrap creating new build based on config + bootstrap::Build::build + TRACE bootstrap setting up job management + TRACE bootstrap downloading rustfmt early + bootstrap::handling hardcoded subcommands (Format, Suggest, Perf) + DEBUG bootstrap not a hardcoded subcommand; returning to normal handling, cmd=Check { all_targets: false } + DEBUG bootstrap handling subcommand normally + bootstrap::executing real run + bootstrap::(1) executing dry-run sanity-check + bootstrap::(2) executing actual run +Checking stage0 library artifacts (x86_64-unknown-linux-gnu) + Finished `release` profile [optimized + debuginfo] target(s) in 0.04s +Checking stage0 compiler artifacts {rustc-main, rustc_abi, rustc_arena, rustc_ast, rustc_ast_ir, rustc_ast_lowering, rustc_ast_passes, rustc_ast_pretty, rustc_attr_data_structures, rustc_attr_parsing, rustc_baked_icu_data, rustc_borrowck, rustc_builtin_macros, rustc_codegen_llvm, rustc_codegen_ssa, rustc_const_eval, rustc_data_structures, rustc_driver, rustc_driver_impl, rustc_error_codes, rustc_error_messages, rustc_errors, rustc_expand, rustc_feature, rustc_fluent_macro, rustc_fs_util, rustc_graphviz, rustc_hir, rustc_hir_analysis, rustc_hir_pretty, rustc_hir_typeck, rustc_incremental, rustc_index, rustc_index_macros, rustc_infer, rustc_interface, rustc_lexer, rustc_lint, rustc_lint_defs, rustc_llvm, rustc_log, rustc_macros, rustc_metadata, rustc_middle, rustc_mir_build, rustc_mir_dataflow, rustc_mir_transform, rustc_monomorphize, rustc_next_trait_solver, rustc_parse, rustc_parse_format, rustc_passes, rustc_pattern_analysis, rustc_privacy, rustc_query_impl, rustc_query_system, rustc_resolve, rustc_sanitizers, rustc_serialize, rustc_session, rustc_smir, rustc_span, rustc_symbol_mangling, rustc_target, rustc_trait_selection, rustc_traits, rustc_transmute, rustc_ty_utils, rustc_type_ir, rustc_type_ir_macros, stable_mir} (x86_64-unknown-linux-gnu) + Finished `release` profile [optimized + debuginfo] target(s) in 0.23s +Checking stage0 bootstrap artifacts (x86_64-unknown-linux-gnu) + Checking bootstrap v0.0.0 (/home/joe/repos/rust/src/bootstrap) + Finished `release` profile [optimized + debuginfo] target(s) in 0.64s + DEBUG bootstrap checking for postponed test failures from `test --no-fail-fast` +Build completed successfully in 0:00:08 +``` + +#### Controlling tracing output + +The env var `BOOTSTRAP_TRACING` accepts a [`tracing` env-filter][tracing-env-filter]. + +There are two orthogonal ways to control which kind of tracing logs you want: + +1. You can specify the log **level**, e.g. `DEBUG` or `TRACE`. +2. You can also control the log **target**, e.g. `bootstrap` or `bootstrap::core::config` vs custom targets like `CONFIG_HANDLING`. + - Custom targets are used to limit what is output when `BOOTSTRAP_TRACING=bootstrap=TRACE` is used, as they can be too verbose even for `TRACE` level by default. Currently used custom targets: + - `CONFIG_HANDLING` + +The `TRACE` filter will enable *all* `trace` level or less verbose level tracing output. + +You can of course combine them (custom target logs are typically gated behind `TRACE` log level additionally): + +```bash +$ BOOTSTRAP_TRACING=CONFIG_HANDLING=TRACE ./x build library --stage 1 +``` + +[^unstable]: This output is always subject to further changes. + +[tracing-env-filter]: https://docs.rs/tracing-subscriber/0.3.19/tracing_subscriber/filter/struct.EnvFilter.html + +##### FIXME(#96176): specific tracing for `compiler()` vs `compiler_for()` + +The additional targets `COMPILER` and `COMPILER_FOR` are used to help trace what +`builder.compiler()` and `builder.compiler_for()` does. They should be removed +if [#96176][cleanup-compiler-for] is resolved. + +[cleanup-compiler-for]: https://github.com/rust-lang/rust/issues/96176 + +### Rendering step graph + +When you run bootstrap with the `BOOTSTRAP_TRACING` environment variable configured, bootstrap will automatically output a DOT file that shows all executed steps and their dependencies. The files will have a prefix `bootstrap-steps`. You can use e.g. `xdot` to visualize the file or e.g. `dot -Tsvg` to convert the DOT file to a SVG file. + +A separate DOT file will be outputted for dry-run and non-dry-run execution. + +### Using `tracing` in bootstrap + +Both `tracing::*` macros and the `tracing::instrument` proc-macro attribute need to be gated behind `tracing` feature. Examples: + +```rs +#[cfg(feature = "tracing")] +use tracing::instrument; + +struct Foo; + +impl Step for Foo { + type Output = (); + + #[cfg_attr(feature = "tracing", instrument(level = "trace", name = "Foo::should_run", skip_all))] + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + trace!(?run, "entered Foo::should_run"); + + todo!() + } + + #[cfg_attr( + feature = "tracing", + instrument( + level = "trace", + name = "Foo::run", + skip_all, + fields(compiler = ?builder.compiler), + ), + )] + fn run(self, builder: &Builder<'_>) -> Self::Output { + trace!(?run, "entered Foo::run"); + + todo!() + } +} +``` + +For `#[instrument]`, it's recommended to: + +- Gate it behind `trace` level for fine-granularity, possibly `debug` level for core functions. +- Explicitly pick an instrumentation name via `name = ".."` to distinguish between e.g. `run` of different steps. +- Take care to not cause diverging behavior via tracing, e.g. building extra things only when tracing infra is enabled. + +### Profiling bootstrap + +You can set the `BOOTSTRAP_PROFILE=1` environment variable to enable command execution profiling during bootstrap. This generates: + +* A Chrome trace file (for visualization in `chrome://tracing` or [Perfetto](https://ui.perfetto.dev)) if tracing is enabled via `BOOTSTRAP_TRACING=COMMAND=trace` +* A plain-text summary file, `bootstrap-profile-{pid}.txt`, listing all commands sorted by execution time (slowest first), along with cache hits and working directories + +Note: the `.txt` report is always generated when `BOOTSTRAP_PROFILE=1` is set — tracing is not required. + +Example usage: + +```bash +$ BOOTSTRAP_PROFILE=1 BOOTSTRAP_TRACING=COMMAND=trace ./x build library +``` + +### rust-analyzer integration? + +Unfortunately, because bootstrap is a `rust-analyzer.linkedProjects`, you can't ask r-a to check/build bootstrap itself with `tracing` feature enabled to get relevant completions, due to lack of support as described in . diff --git a/src/building/bootstrapping/intro.md b/src/building/bootstrapping/intro.md index f829884fb..7f5309782 100644 --- a/src/building/bootstrapping/intro.md +++ b/src/building/bootstrapping/intro.md @@ -6,8 +6,8 @@ of the same compiler. This raises a chicken-and-egg paradox: where did the first compiler come from? It must have been written in a different language. In Rust's case it was -[written in OCaml][ocaml-compiler]. However it was abandoned long ago and the -only way to build a modern version of rustc is a slightly less modern +[written in OCaml][ocaml-compiler]. However, it was abandoned long ago, and the +only way to build a modern version of rustc is with a slightly less modern version. This is exactly how `x.py` works: it downloads the current beta release of @@ -17,5 +17,8 @@ In this section, we give a high-level overview of [what Bootstrap does](./what-bootstrapping-does.md), followed by a high-level introduction to [how Bootstrap does it](./how-bootstrap-does-it.md). +Additionally, see [debugging bootstrap](./debugging-bootstrap.md) to learn +about debugging methods. + [boot]: https://en.wikipedia.org/wiki/Bootstrapping_(compilers) [ocaml-compiler]: https://github.com/rust-lang/rust/tree/ef75860a0a72f79f97216f8aaa5b388d98da6480/src/boot diff --git a/src/building/bootstrapping/what-bootstrapping-does.md b/src/building/bootstrapping/what-bootstrapping-does.md index 1dd5402f4..da425d8d3 100644 --- a/src/building/bootstrapping/what-bootstrapping-does.md +++ b/src/building/bootstrapping/what-bootstrapping-does.md @@ -1,15 +1,13 @@ # What Bootstrapping does - - [*Bootstrapping*][boot] is the process of using a compiler to compile itself. More accurately, it means using an older compiler to compile a newer version of the same compiler. This raises a chicken-and-egg paradox: where did the first compiler come from? It must have been written in a different language. In Rust's case it was -[written in OCaml][ocaml-compiler]. However it was abandoned long ago and the -only way to build a modern version of `rustc` is a slightly less modern version. +[written in OCaml][ocaml-compiler]. However, it was abandoned long ago, and the +only way to build a modern version of `rustc` is with a slightly less modern version. This is exactly how [`./x.py`] works: it downloads the current beta release of `rustc`, then uses it to compile the new compiler. @@ -45,13 +43,13 @@ compiler. ```mermaid graph TD - s0c["stage0 compiler (1.63)"]:::downloaded -->|A| s0l("stage0 std (1.64)"):::with-s0c; + s0c["stage0 compiler (1.86.0-beta.1)"]:::downloaded -->|A| s0l("stage0 std (1.86.0-beta.1)"):::downloaded; s0c & s0l --- stepb[ ]:::empty; - stepb -->|B| s0ca["stage0 compiler artifacts (1.64)"]:::with-s0c; - s0ca -->|copy| s1c["stage1 compiler (1.64)"]:::with-s0c; - s1c -->|C| s1l("stage1 std (1.64)"):::with-s1c; + stepb -->|B| s0ca["stage0 compiler artifacts (1.87.0-dev)"]:::with-s0c; + s0ca -->|copy| s1c["stage1 compiler (1.87.0-dev)"]:::with-s0c; + s1c -->|C| s1l("stage1 std (1.87.0-dev)"):::with-s1c; s1c & s1l --- stepd[ ]:::empty; - stepd -->|D| s1ca["stage1 compiler artifacts (1.64)"]:::with-s1c; + stepd -->|D| s1ca["stage1 compiler artifacts (1.87.0-dev)"]:::with-s1c; s1ca -->|copy| s2c["stage2 compiler"]:::with-s1c; classDef empty width:0px,height:0px; @@ -62,19 +60,21 @@ graph TD ### Stage 0: the pre-compiled compiler -The stage0 compiler is usually the current _beta_ `rustc` compiler and its +The stage0 compiler is by default the very recent _beta_ `rustc` compiler and its associated dynamic libraries, which `./x.py` will download for you. (You can -also configure `./x.py` to use something else.) +also configure `./x.py` to change stage0 to something else.) + +The precompiled stage0 compiler is then used only to compile [`src/bootstrap`] and [`compiler/rustc`] +with precompiled stage0 std. + +Note that to build the stage1 compiler we use the precompiled stage0 compiler and std. +Therefore, to use a compiler with a std that is freshly built from the tree, you need to +build the stage2 compiler. -The stage0 compiler is then used only to compile [`src/bootstrap`], -[`library/std`], and [`compiler/rustc`]. When assembling the libraries and -binaries that will become the stage1 `rustc` compiler, the freshly compiled -`std` and `rustc` are used. There are two concepts at play here: a compiler -(with its set of dependencies) and its 'target' or 'object' libraries (`std` and -`rustc`). Both are staged, but in a staggered manner. +There are two concepts at play here: a compiler (with its set of dependencies) and its +'target' or 'object' libraries (`std` and `rustc`). Both are staged, but in a staggered manner. [`compiler/rustc`]: https://github.com/rust-lang/rust/tree/master/compiler/rustc -[`library/std`]: https://github.com/rust-lang/rust/tree/master/library/std [`src/bootstrap`]: https://github.com/rust-lang/rust/tree/master/src/bootstrap ### Stage 1: from current code, by an earlier compiler @@ -84,16 +84,14 @@ The rustc source code is then compiled with the `stage0` compiler to produce the ### Stage 2: the truly current compiler -We then rebuild our `stage1` compiler with itself to produce the `stage2` +We then rebuild the compiler using `stage1` compiler with in-tree std to produce the `stage2` compiler. -In theory, the `stage1` compiler is functionally identical to the `stage2` -compiler, but in practice there are subtle differences. In particular, the -`stage1` compiler itself was built by `stage0` and hence not by the source in -your working directory. This means that the ABI generated by the `stage0` -compiler may not match the ABI that would have been made by the `stage1` -compiler, which can cause problems for dynamic libraries, tests, and tools using -`rustc_private`. +The `stage1` compiler itself was built by precompiled `stage0` compiler and std +and hence not by the source in your working directory. This means that the ABI +generated by the `stage0` compiler may not match the ABI that would have been made +by the `stage1` compiler, which can cause problems for dynamic libraries, tests +and tools using `rustc_private`. Note that the `proc_macro` crate avoids this issue with a `C` FFI layer called `proc_macro::bridge`, allowing it to be used with `stage1`. @@ -101,9 +99,10 @@ Note that the `proc_macro` crate avoids this issue with a `C` FFI layer called The `stage2` compiler is the one distributed with `rustup` and all other install methods. However, it takes a very long time to build because one must first build the new compiler with an older compiler and then use that to build the new -compiler with itself. For development, you usually only want the `stage1` -compiler, which you can build with `./x build library`. See [Building the -compiler](../how-to-build-and-run.html#building-the-compiler). +compiler with itself. + +For development, you usually only want to use `--stage 1` flag to build things. +See [Building the compiler](../how-to-build-and-run.html#building-the-compiler). ### Stage 3: the same-result test @@ -114,10 +113,11 @@ something has broken. ### Building the stages The script [`./x`] tries to be helpful and pick the stage you most likely meant -for each subcommand. These defaults are as follows: +for each subcommand. Here are some `x` commands with their default stages: -- `check`: `--stage 0` -- `doc`: `--stage 0` +- `check`: `--stage 1` +- `clippy`: `--stage 1` +- `doc`: `--stage 1` - `build`: `--stage 1` - `test`: `--stage 1` - `dist`: `--stage 2` @@ -191,8 +191,8 @@ include, but are not limited to: without building `rustc` from source ('build with `stage0`, then test the artifacts'). If you're working on the standard library, this is normally the test command you want. -- `./x build --stage 0` means to build with the beta `rustc`. -- `./x doc --stage 0` means to document using the beta `rustdoc`. +- `./x build --stage 0` means to build with the stage0 `rustc`. +- `./x doc --stage 0` means to document using the stage0 `rustdoc`. #### Examples of what *not* to do @@ -208,9 +208,6 @@ include, but are not limited to: ### Building vs. running -Note that `build --stage N compiler/rustc` **does not** build the stage N -compiler: instead it builds the stage N+1 compiler _using_ the stage N compiler. - In short, _stage 0 uses the `stage0` compiler to create `stage0` artifacts which will later be uplifted to be the stage1 compiler_. @@ -268,23 +265,6 @@ However, when cross-compiling, `stage1` `std` will only run on the host. So the (See in the table how `stage2` only builds non-host `std` targets). -### Why does only libstd use `cfg(bootstrap)`? - -For docs on `cfg(bootstrap)` itself, see [Complications of -Bootstrapping](#complications-of-bootstrapping). - -The `rustc` generated by the `stage0` compiler is linked to the freshly-built -`std`, which means that for the most part only `std` needs to be `cfg`-gated, so -that `rustc` can use features added to `std` immediately after their addition, -without need for them to get into the downloaded `beta` compiler. - -Note this is different from any other Rust program: `stage1` `rustc` is built by -the _beta_ compiler, but using the _master_ version of `libstd`! - -The only time `rustc` uses `cfg(bootstrap)` is when it adds internal lints that -use diagnostic items, or when it uses unstable library features that were -recently changed. - ### What is a 'sysroot'? When you build a project with `cargo`, the build artifacts for dependencies are @@ -394,8 +374,8 @@ will be rare to want to use it. Finally, `MAGIC_EXTRA_RUSTFLAGS` bypasses the this is `compiletest`. For unit tests and doc tests this is the `libtest` runner. -Most test runner accept `--help`, which you can use to find out the options -accepted by the runner. +Most test runners accept `--help`, +which you can use to find out the options accepted by the runner. ## Environment Variables @@ -459,7 +439,6 @@ compiler itself uses to run. These aren't actually used by artifacts the new compiler generates. This step also copies the `rustc` and `rustdoc` binaries we generated into `build/$HOST/stage/bin`. -The `stage1/bin/rustc` is a fully functional compiler, but it doesn't yet have -any libraries to link built binaries or libraries to. The next 3 steps will -provide those libraries for it; they are mostly equivalent to constructing the -`stage1/bin` compiler so we don't go through them individually here. +The `stage1/bin/rustc` is a fully functional compiler built with stage0 (precompiled) compiler and std. +To use a compiler built entirely from source with the in-tree compiler and std, you need to build the +stage2 compiler, which is compiled using the stage1 (in-tree) compiler and std. diff --git a/src/building/bootstrapping/writing-tools-in-bootstrap.md b/src/building/bootstrapping/writing-tools-in-bootstrap.md new file mode 100644 index 000000000..41d0cf8d9 --- /dev/null +++ b/src/building/bootstrapping/writing-tools-in-bootstrap.md @@ -0,0 +1,23 @@ +# Writing tools in Bootstrap + +There are three types of tools you can write in bootstrap: + +- **`Mode::ToolBootstrap`** + Use this for tools that don’t need anything from the in-tree compiler and can run with the stage0 `rustc`. + The output is placed in the "bootstrap-tools" directory. This mode is for general-purpose tools built + entirely with the stage0 compiler, including target libraries and only works for stage 0. + +- **`Mode::ToolStd`** + Use this for tools that rely on the locally built std. The output goes into the "stageN-tools" directory. + This mode is rarely used, mainly for `compiletest` which requires `libtest`. + +- **`Mode::ToolRustc`** + Use this for tools that depend on both the locally built `rustc` and the target `std`. This is more complex than + the other modes because the tool must be built with the same compiler used for `rustc` and placed in the "stageN-tools" + directory. When you choose `Mode::ToolRustc`, `ToolBuild` implementation takes care of this automatically. + If you need to use the builder’s compiler for something specific, you can get it from `ToolBuildResult`, which is + returned by the tool's [`Step`]. + +Regardless of the tool type you must return `ToolBuildResult` from the tool’s [`Step`] implementation and use `ToolBuild` inside it. + +[`Step`]: https://doc.rust-lang.org/nightly/nightly-rustc/bootstrap/core/builder/trait.Step.html diff --git a/src/building/compiler-documenting.md b/src/building/compiler-documenting.md index 948571ce8..b031462ea 100644 --- a/src/building/compiler-documenting.md +++ b/src/building/compiler-documenting.md @@ -36,7 +36,7 @@ like the standard library (std) or the compiler (rustc). - Document internal rustc items Compiler documentation is not built by default. - To create it by default with `x doc`, modify `config.toml`: + To create it by default with `x doc`, modify `bootstrap.toml`: ```toml [build] diff --git a/src/building/how-to-build-and-run.md b/src/building/how-to-build-and-run.md index f9a089c23..b07d3533f 100644 --- a/src/building/how-to-build-and-run.md +++ b/src/building/how-to-build-and-run.md @@ -1,6 +1,22 @@ # How to build and run the compiler - +
+ +For `profile = "library"` users, or users who use `download-rustc = true | "if-unchanged"`, please be advised that +the `./x test library/std` flow where `download-rustc` is active (i.e. no compiler changes) is currently broken. +This is tracked in . Only the `./x test` flow is affected in this +case, `./x {check,build} library/std` should still work. + +In the short-term, you may need to disable `download-rustc` for `./x test library/std`. This can be done either by: + +1. `./x test library/std --set rust.download-rustc=false` +2. Or set `rust.download-rustc=false` in `bootstrap.toml`. + +Unfortunately that will require building the stage 1 compiler. The bootstrap team is working on this, but +implementing a maintainable fix is taking some time. + +
+ The compiler is built using a tool called `x.py`. You will need to have Python installed to run it. @@ -63,7 +79,7 @@ cd rust > **NOTE**: A shallow clone limits which `git` commands can be run. > If you intend to work on and contribute to the compiler, it is > generally recommended to fully clone the repository [as shown above](#get-the-source-code), -> or to perform a [partial clone](#shallow-clone-the-repository) instead. +> or to perform a [partial clone](#partial-clone-the-repository) instead. > > For example, `git bisect` and `git blame` require access to the commit history, > so they don't work if the repository was cloned with `--depth 1`. @@ -159,15 +175,15 @@ similar to the one declared in section [What is `x.py`](#what-is-xpy), but it works as an independent process to execute the `x.py` rather than calling the shell to run the platform related scripts. -## Create a `config.toml` +## Create a `bootstrap.toml` To start, run `./x setup` and select the `compiler` defaults. This will do some initialization -and create a `config.toml` for you with reasonable defaults. If you use a different default (which +and create a `bootstrap.toml` for you with reasonable defaults. If you use a different default (which you'll likely want to do if you want to contribute to an area of rust other than the compiler, such as rustdoc), make sure to read information about that default (located in `src/bootstrap/defaults`) as the build process may be different for other defaults. -Alternatively, you can write `config.toml` by hand. See `config.example.toml` for all the available +Alternatively, you can write `bootstrap.toml` by hand. See `bootstrap.example.toml` for all the available settings and explanations of them. See `src/bootstrap/defaults` for common settings to change. If you have already built `rustc` and you change settings related to LLVM, then you may have to @@ -206,7 +222,7 @@ See the chapters on Note that building will require a relatively large amount of storage space. You may want to have upwards of 10 or 15 gigabytes available to build the compiler. -Once you've created a `config.toml`, you are now ready to run +Once you've created a `bootstrap.toml`, you are now ready to run `x`. There are a lot of options here, but let's start with what is probably the best "go to" command for building a local compiler: @@ -217,7 +233,6 @@ probably the best "go to" command for building a local compiler: This may *look* like it only builds the standard library, but that is not the case. What this command does is the following: -- Build `std` using the stage0 compiler - Build `rustc` using the stage0 compiler - This produces the stage1 compiler - Build `std` using the stage1 compiler @@ -241,8 +256,7 @@ build. The **full** `rustc` build (what you get with `./x build --stage 2 compiler/rustc`) has quite a few more steps: - Build `rustc` with the stage1 compiler. - - The resulting compiler here is called the "stage2" compiler. -- Build `std` with stage2 compiler. + - The resulting compiler here is called the "stage2" compiler, which uses stage1 std from the previous command. - Build `librustdoc` and a bunch of other things with the stage2 compiler. You almost never need to do this. @@ -250,14 +264,14 @@ You almost never need to do this. ### Build specific components If you are working on the standard library, you probably don't need to build -the compiler unless you are planning to use a recently added nightly feature. -Instead, you can just build using the bootstrap compiler. +every other default component. Instead, you can build a specific component by +providing its name, like this: ```bash -./x build --stage 0 library +./x build --stage 1 library ``` -If you choose the `library` profile when running `x setup`, you can omit `--stage 0` (it's the +If you choose the `library` profile when running `x setup`, you can omit `--stage 1` (it's the default). ## Creating a rustup toolchain @@ -271,7 +285,6 @@ you will likely need to build at some point; for example, if you want to run the entire test suite). ```bash -rustup toolchain link stage0 build/host/stage0-sysroot # beta compiler + stage0 std rustup toolchain link stage1 build/host/stage1 rustup toolchain link stage2 build/host/stage2 ``` @@ -326,7 +339,7 @@ involve proc macros or build scripts, you must be sure to explicitly build targe host platform (in this case, `x86_64-unknown-linux-gnu`). If you want to always build for other targets without needing to pass flags to `x build`, -you can configure this in the `[build]` section of your `config.toml` like so: +you can configure this in the `[build]` section of your `bootstrap.toml` like so: ```toml [build] @@ -336,8 +349,8 @@ target = ["x86_64-unknown-linux-gnu", "wasm32-wasip1"] Note that building for some targets requires having external dependencies installed (e.g. building musl targets requires a local copy of musl). Any target-specific configuration (e.g. the path to a local copy of musl) -will need to be provided by your `config.toml`. -Please see `config.example.toml` for information on target-specific configuration keys. +will need to be provided by your `bootstrap.toml`. +Please see `bootstrap.example.toml` for information on target-specific configuration keys. For examples of the complete configuration necessary to build a target, please visit [the rustc book](https://doc.rust-lang.org/rustc/platform-support.html), diff --git a/src/building/new-target.md b/src/building/new-target.md index 1d9fa1b52..436aec8ee 100644 --- a/src/building/new-target.md +++ b/src/building/new-target.md @@ -4,7 +4,9 @@ These are a set of steps to add support for a new target. There are numerous end states and paths to get there, so not all sections may be relevant to your desired goal. - +See also the associated documentation in the [target tier policy]. + +[target tier policy]: https://doc.rust-lang.org/rustc/target-tier-policy.html#adding-a-new-target ## Specifying a new LLVM @@ -33,7 +35,7 @@ able to configure Rust to treat your build as the system LLVM to avoid redundant builds. You can tell Rust to use a pre-built version of LLVM using the `target` section -of `config.toml`: +of `bootstrap.toml`: ```toml [target.x86_64-unknown-linux-gnu] @@ -51,8 +53,8 @@ for codegen tests. This tool is normally built with LLVM, but if you use your own preinstalled LLVM, you will need to provide `FileCheck` in some other way. On Debian-based systems, you can install the `llvm-N-tools` package (where `N` is the LLVM version number, e.g. `llvm-8-tools`). Alternately, you can specify -the path to `FileCheck` with the `llvm-filecheck` config item in `config.toml` -or you can disable codegen test with the `codegen-tests` item in `config.toml`. +the path to `FileCheck` with the `llvm-filecheck` config item in `bootstrap.toml` +or you can disable codegen test with the `codegen-tests` item in `bootstrap.toml`. ## Creating a target specification @@ -81,7 +83,7 @@ Look for existing targets to use as examples. After adding your target to the `rustc_target` crate you may want to add `core`, `std`, ... with support for your new target. In that case you will probably need access to some `target_*` cfg. Unfortunately when building with -stage0 (the beta compiler), you'll get an error that the target cfg is +stage0 (a precompiled compiler), you'll get an error that the target cfg is unexpected because stage0 doesn't know about the new target specification and we pass `--check-cfg` in order to tell it to check. @@ -137,14 +139,14 @@ After this, run `cargo update -p libc` to update the lockfiles. Beware that if you patch to a local `path` dependency, this will enable warnings for that dependency. Some dependencies are not warning-free, and due -to the `deny-warnings` setting in `config.toml`, the build may suddenly start +to the `deny-warnings` setting in `bootstrap.toml`, the build may suddenly start to fail. To work around warnings, you may want to: - Modify the dependency to remove the warnings -- Or for local development purposes, suppress the warnings by setting deny-warnings = false in config.toml. +- Or for local development purposes, suppress the warnings by setting deny-warnings = false in bootstrap.toml. ```toml -# config.toml +# bootstrap.toml [rust] deny-warnings = false ``` @@ -170,8 +172,8 @@ compiler, you can use it instead of the JSON file for both arguments. ## Promoting a target from tier 2 (target) to tier 2 (host) There are two levels of tier 2 targets: - a) Targets that are only cross-compiled (`rustup target add`) - b) Targets that [have a native toolchain][tier2-native] (`rustup toolchain install`) +- Targets that are only cross-compiled (`rustup target add`) +- Targets that [have a native toolchain][tier2-native] (`rustup toolchain install`) [tier2-native]: https://doc.rust-lang.org/nightly/rustc/target-tier-policy.html#tier-2-with-host-tools diff --git a/src/building/optimized-build.md b/src/building/optimized-build.md index 3080dc6bf..863ed9749 100644 --- a/src/building/optimized-build.md +++ b/src/building/optimized-build.md @@ -1,7 +1,5 @@ # Optimized build of the compiler - - There are multiple additional build configuration options and techniques that can be used to compile a build of `rustc` that is as optimized as possible (for example when building `rustc` for a Linux distribution). The status of these configuration options for various Rust targets is tracked [here]. @@ -13,7 +11,7 @@ This page describes how you can use these approaches when building `rustc` yours Link-time optimization is a powerful compiler technique that can increase program performance. To enable (Thin-)LTO when building `rustc`, set the `rust.lto` config option to `"thin"` -in `config.toml`: +in `bootstrap.toml`: ```toml [rust] @@ -34,7 +32,7 @@ Enabling LTO on Linux has [produced] speed-ups by up to 10%. Using a different memory allocator for `rustc` can provide significant performance benefits. If you want to enable the `jemalloc` allocator, you can set the `rust.jemalloc` option to `true` -in `config.toml`: +in `bootstrap.toml`: ```toml [rust] @@ -46,7 +44,7 @@ jemalloc = true ## Codegen units Reducing the amount of codegen units per `rustc` crate can produce a faster build of the compiler. -You can modify the number of codegen units for `rustc` and `libstd` in `config.toml` with the +You can modify the number of codegen units for `rustc` and `libstd` in `bootstrap.toml` with the following options: ```toml @@ -67,7 +65,7 @@ RUSTFLAGS="-C target_cpu=x86-64-v3" ./x build ... ``` If you also want to compile LLVM for a specific instruction set, you can set `llvm` flags -in `config.toml`: +in `bootstrap.toml`: ```toml [llvm] @@ -109,11 +107,16 @@ like Python or LLVM. Here is an example of how can `opt-dist` be used locally (outside of CI): -1. Build the tool with the following command: +1. Enable metrics in your `bootstrap.toml` file, because `opt-dist` expects it to be enabled: + ```toml + [build] + metrics = true + ``` +2. Build the tool with the following command: ```bash ./x build tools/opt-dist ``` -2. Run the tool with the `local` mode and provide necessary parameters: +3. Run the tool with the `local` mode and provide necessary parameters: ```bash ./build/host/stage0-tools-bin/opt-dist local \ --target-triple \ # select target, e.g. "x86_64-unknown-linux-gnu" @@ -126,4 +129,4 @@ Here is an example of how can `opt-dist` be used locally (outside of CI): [`Environment`]: https://github.com/rust-lang/rust/blob/ee451f8faccf3050c76cdcd82543c917b40c7962/src/tools/opt-dist/src/environment.rs#L5 > Note: if you want to run the actual CI pipeline, instead of running `opt-dist` locally, -> you can execute `DEPLOY=1 src/ci/docker/run.sh dist-x86_64-linux`. +> you can execute `cargo run --manifest-path src/ci/citool/Cargo.toml run-local dist-x86_64-linux`. diff --git a/src/building/prerequisites.md b/src/building/prerequisites.md index f49f6bb05..6761cabac 100644 --- a/src/building/prerequisites.md +++ b/src/building/prerequisites.md @@ -38,4 +38,4 @@ incremental compilation ([see here][config]). This will make compilation take longer (especially after a rebase), but will save a ton of space from the incremental caches. -[config]: ./how-to-build-and-run.md#create-a-configtoml +[config]: ./how-to-build-and-run.md#create-a-bootstraptoml diff --git a/src/building/quickstart.md b/src/building/quickstart.md index 9a8ab353e..97314d803 100644 --- a/src/building/quickstart.md +++ b/src/building/quickstart.md @@ -61,9 +61,6 @@ and check the output. Use `--bless` if you've made a change and want to update the `.stderr` files with the new output. -> `./x suggest` can also be helpful for suggesting which tests to run after a -> change. - Congrats, you are now ready to make a change to the compiler! If you have more questions, [the full chapter](./how-to-build-and-run.md) might contain the answers, and if it doesn't, feel free to ask for help on diff --git a/src/building/suggested.md b/src/building/suggested.md index bf5ffbc00..35c7e935b 100644 --- a/src/building/suggested.md +++ b/src/building/suggested.md @@ -1,10 +1,8 @@ -# Suggested Workflows +# Suggested workflows The full bootstrapping process takes quite a while. Here are some suggestions to make your life easier. - - ## Installing a pre-push hook CI will automatically fail your build if it doesn't pass `tidy`, our internal @@ -20,8 +18,53 @@ your `.git/hooks` folder as `pre-push` (without the `.sh` extension!). You can also install the hook as a step of running `./x setup`! +## Config extensions + +When working on different tasks, you might need to switch between different bootstrap configurations. +Sometimes you may want to keep an old configuration for future use. But saving raw config values in +random files and manually copying and pasting them can quickly become messy, especially if you have a +long history of different configurations. + +To simplify managing multiple configurations, you can create config extensions. + +For example, you can create a simple config file named `cross.toml`: + +```toml +[build] +build = "x86_64-unknown-linux-gnu" +host = ["i686-unknown-linux-gnu"] +target = ["i686-unknown-linux-gnu"] + + +[llvm] +download-ci-llvm = false + +[target.x86_64-unknown-linux-gnu] +llvm-config = "/path/to/llvm-19/bin/llvm-config" +``` + +Then, include this in your `bootstrap.toml`: + +```toml +include = ["cross.toml"] +``` + +You can also include extensions within extensions recursively. + +**Note:** In the `include` field, the overriding logic follows a right-to-left order. For example, +in `include = ["a.toml", "b.toml"]`, extension `b.toml` overrides `a.toml`. Also, parent extensions +always overrides the inner ones. + ## Configuring `rust-analyzer` for `rustc` +### Checking the "library" tree + +Checking the "library" tree requires a stage1 compiler, which can be a heavy process on some computers. +For this reason, bootstrap has a flag called `--skip-std-check-if-no-download-rustc` that skips checking the +"library" tree if `rust.download-rustc` isn't available. If you want to avoid putting a heavy load on your computer +with `rust-analyzer`, you can add the `--skip-std-check-if-no-download-rustc` flag to your `./x check` command in +the `rust-analyzer` configuration. + ### Project-local rust-analyzer setup `rust-analyzer` can help you check and format your code whenever you save a @@ -54,7 +97,7 @@ for two reasons: additional rebuilds in some cases. To avoid these problems: -- Add `--build-dir=build-rust-analyzer` to all of the custom `x` commands in +- Add `--build-dir=build/rust-analyzer` to all of the custom `x` commands in your editor's rust-analyzer configuration. (Feel free to choose a different directory name if desired.) - Modify the `rust-analyzer.rustfmt.overrideCommand` setting so that it points @@ -63,10 +106,7 @@ To avoid these problems: copy of `rust-analyzer-proc-macro-srv` in that other build directory. Using separate build directories for command-line builds and rust-analyzer -requires extra disk space, and also means that running `./x clean` on the -command-line will not clean out the separate build directory. To clean the -separate build directory, run `./x clean --build-dir=build-rust-analyzer` -instead. +requires extra disk space. ### Visual Studio Code @@ -100,7 +140,7 @@ Task] instead: ### Neovim -For Neovim users there are several options for configuring for rustc. The +For Neovim users, there are a few options. The easiest way is by using [neoconf.nvim](https://github.com/folke/neoconf.nvim/), which allows for project-local configuration files with the native LSP. The steps for how to use it are below. Note that they require rust-analyzer to @@ -120,10 +160,59 @@ create a `.vim/coc-settings.json`. The settings can be edited with [`src/etc/rust_analyzer_settings.json`]. Another way is without a plugin, and creating your own logic in your -configuration. To do this you must translate the JSON to Lua yourself. The -translation is 1:1 and fairly straight-forward. It must be put in the -`["rust-analyzer"]` key of the setup table, which is [shown -here](https://github.com/neovim/nvim-lspconfig/blob/master/doc/server_configurations.md#rust_analyzer). +configuration. The following code will work for any checkout of rust-lang/rust (newer than February 2025): + +```lua +local function expand_config_variables(option) + local var_placeholders = { + ['${workspaceFolder}'] = function(_) + return vim.lsp.buf.list_workspace_folders()[1] + end, + } + + if type(option) == "table" then + local mt = getmetatable(option) + local result = {} + for k, v in pairs(option) do + result[expand_config_variables(k)] = expand_config_variables(v) + end + return setmetatable(result, mt) + end + if type(option) ~= "string" then + return option + end + local ret = option + for key, fn in pairs(var_placeholders) do + ret = ret:gsub(key, fn) + end + return ret +end +lspconfig.rust_analyzer.setup { + root_dir = function() + local default = lspconfig.rust_analyzer.config_def.default_config.root_dir() + -- the default root detection uses the cargo workspace root. + -- but for rust-lang/rust, the standard library is in its own workspace. + -- use the git root instead. + local compiler_config = vim.fs.joinpath(default, "../src/bootstrap/defaults/config.compiler.toml") + if vim.fs.basename(default) == "library" and vim.uv.fs_stat(compiler_config) then + return vim.fs.dirname(default) + end + return default + end, + on_init = function(client) + local path = client.workspace_folders[1].name + local config = vim.fs.joinpath(path, "src/etc/rust_analyzer_zed.json") + if vim.uv.fs_stat(config) then + -- load rust-lang/rust settings + local file = io.open(config) + local json = vim.json.decode(file:read("*a")) + client.config.settings["rust-analyzer"] = expand_config_variables(json.lsp["rust-analyzer"].initialization_options) + client.notify("workspace/didChangeConfiguration", { settings = client.config.settings }) + end + return true + end +} +``` If you would like to use the build task that is described above, you may either make your own command in your config, or you can install a plugin such as @@ -135,24 +224,34 @@ and follow the same instructions as above. ### Emacs Emacs provides support for rust-analyzer with project-local configuration -through [Eglot](https://www.gnu.org/software/emacs/manual/html_node/eglot/). +through [Eglot](https://www.gnu.org/software/emacs/manual/html_node/eglot/). Steps for setting up Eglot with rust-analyzer can be [found -here](https://rust-analyzer.github.io/manual.html#eglot). +here](https://rust-analyzer.github.io/manual.html#eglot). Having set up Emacs & Eglot for Rust development in general, you can run `./x setup editor` and select `emacs`, which will prompt you to create `.dir-locals.el` with the recommended configuration for Eglot. -The recommended settings live at [`src/etc/rust_analyzer_eglot.el`]. +The recommended settings live at [`src/etc/rust_analyzer_eglot.el`]. For more information on project-specific Eglot configuration, consult [the manual](https://www.gnu.org/software/emacs/manual/html_node/eglot/Project_002dspecific-configuration.html). ### Helix -Helix comes with built-in LSP and rust-analyzer support. +Helix comes with built-in LSP and rust-analyzer support. It can be configured through `languages.toml`, as described -[here](https://docs.helix-editor.com/languages.html). +[here](https://docs.helix-editor.com/languages.html). You can run `./x setup editor` and select `helix`, which will prompt you to create `languages.toml` with the recommended configuration for Helix. The -recommended settings live at [`src/etc/rust_analyzer_helix.toml`]. +recommended settings live at [`src/etc/rust_analyzer_helix.toml`]. + +### Zed + +Zed comes with built-in LSP and rust-analyzer support. +It can be configured through `.zed/settings.json`, as described +[here](https://zed.dev/docs/configuring-languages). Selecting `zed` +in `./x setup editor` will prompt you to create a `.zed/settings.json` +file which will configure Zed with the recommended configuration. The +recommended `rust-analyzer` settings live +at [`src/etc/rust_analyzer_zed.json`]. ## Check, check, and check again @@ -169,23 +268,6 @@ run the tests at some later time. You can then use `git bisect` to track down is that you are left with a fairly fine-grained set of commits at the end, all of which build and pass tests. This often helps reviewing. -## `x suggest` - -The `x suggest` subcommand suggests (and runs) a subset of the extensive -`rust-lang/rust` tests based on files you have changed. This is especially -useful for new contributors who have not mastered the arcane `x` flags yet and -more experienced contributors as a shorthand for reducing mental effort. In all -cases it is useful not to run the full tests (which can take on the order of -tens of minutes) and just run a subset which are relevant to your changes. For -example, running `tidy` and `linkchecker` is useful when editing Markdown files, -whereas UI tests are much less likely to be helpful. While `x suggest` is a -useful tool, it does not guarantee perfect coverage (just as PR CI isn't a -substitute for bors). See the [dedicated chapter](../tests/suggest-tests.md) for -more information and contribution instructions. - -Please note that `x suggest` is in a beta state currently and the tests that it -will suggest are limited. - ## Configuring `rustup` to use nightly Some parts of the bootstrap process uses pinned, nightly versions of tools like @@ -214,51 +296,15 @@ lets you use `cargo fmt`. [the section on vscode]: suggested.md#configuring-rust-analyzer-for-rustc [the section on rustup]: how-to-build-and-run.md?highlight=rustup#creating-a-rustup-toolchain -## Faster builds with `--keep-stage`. - -Sometimes just checking whether the compiler builds is not enough. A common -example is that you need to add a `debug!` statement to inspect the value of -some state or better understand the problem. In that case, you don't really need -a full build. By bypassing bootstrap's cache invalidation, you can often get -these builds to complete very fast (e.g., around 30 seconds). The only catch is -this requires a bit of fudging and may produce compilers that don't work (but -that is easily detected and fixed). - -The sequence of commands you want is as follows: - -- Initial build: `./x build library` - - As [documented previously], this will build a functional stage1 compiler as - part of running all stage0 commands (which include building a `std` - compatible with the stage1 compiler) as well as the first few steps of the - "stage 1 actions" up to "stage1 (sysroot stage1) builds std". -- Subsequent builds: `./x build library --keep-stage 1` - - Note that we added the `--keep-stage 1` flag here - -[documented previously]: ./how-to-build-and-run.md#building-the-compiler - -As mentioned, the effect of `--keep-stage 1` is that we just _assume_ that the -old standard library can be re-used. If you are editing the compiler, this is -almost always true: you haven't changed the standard library, after all. But -sometimes, it's not true: for example, if you are editing the "metadata" part of -the compiler, which controls how the compiler encodes types and other states -into the `rlib` files, or if you are editing things that wind up in the metadata -(such as the definition of the MIR). - -**The TL;DR is that you might get weird behavior from a compile when using -`--keep-stage 1`** -- for example, strange [ICEs](../appendix/glossary.html#ice) -or other panics. In that case, you should simply remove the `--keep-stage 1` -from the command and rebuild. That ought to fix the problem. - -You can also use `--keep-stage 1` when running tests. Something like this: +## Faster Builds with CI-rustc -- Initial test run: `./x test tests/ui` -- Subsequent test run: `./x test tests/ui --keep-stage 1` - -### Iterating the standard library with `--keep-stage` - -If you are making changes to the standard library, you can use `./x build ---keep-stage 0 library` to iteratively rebuild the standard library without -rebuilding the compiler. +If you are not working on the compiler, you often don't need to build the compiler tree. +For example, you can skip building the compiler and only build the `library` tree or the +tools under `src/tools`. To achieve that, you have to enable this by setting the `download-rustc` +option in your configuration. This tells bootstrap to use the latest nightly compiler for `stage > 0` +steps, meaning it will have two precompiled compilers: stage0 compiler and `download-rustc` compiler +for `stage > 0` steps. This way, it will never need to build the in-tree compiler. As a result, your +build time will be significantly reduced by not building the in-tree compiler. ## Using incremental compilation @@ -270,7 +316,7 @@ subsequent rebuilds: ``` If you don't want to include the flag with every command, you can enable it in -the `config.toml`: +the `bootstrap.toml`: ```toml [rust] @@ -332,44 +378,37 @@ git worktree add -b my-feature ../rust2 master You can then use that rust2 folder as a separate workspace for modifying and building `rustc`! -## Using nix-shell +## Working with nix -If you're using nix, you can use the following nix-shell to work on Rust: +Several nix configurations are defined in `src/tools/nix-dev-shell`. -```nix -{ pkgs ? import {} }: -pkgs.mkShell { - name = "rustc"; - nativeBuildInputs = with pkgs; [ - binutils cmake ninja pkg-config python3 git curl cacert patchelf nix - ]; - buildInputs = with pkgs; [ - openssl glibc.out glibc.static - ]; - # Avoid creating text files for ICEs. - RUSTC_ICE = "0"; - # Provide `libstdc++.so.6` for the self-contained lld. - LD_LIBRARY_PATH = "${with pkgs; lib.makeLibraryPath [ - stdenv.cc.cc.lib - ]}"; -} +If you're using direnv, you can create a symbol link to `src/tools/nix-dev-shell/envrc-flake` or `src/tools/nix-dev-shell/envrc-shell` + +```bash +ln -s ./src/tools/nix-dev-shell/envrc-flake ./.envrc # Use flake ``` +or +```bash +ln -s ./src/tools/nix-dev-shell/envrc-shell ./.envrc # Use nix-shell +``` + +### Note Note that when using nix on a not-NixOS distribution, it may be necessary to set -**`patch-binaries-for-nix = true` in `config.toml`**. Bootstrap tries to detect +**`patch-binaries-for-nix = true` in `bootstrap.toml`**. Bootstrap tries to detect whether it's running in nix and enable patching automatically, but this detection can have false negatives. -You can also use your nix shell to manage `config.toml`: +You can also use your nix shell to manage `bootstrap.toml`: ```nix let config = pkgs.writeText "rustc-config" '' - # Your config.toml content goes here + # Your bootstrap.toml content goes here '' pkgs.mkShell { /* ... */ - # This environment variable tells bootstrap where our config.toml is. + # This environment variable tells bootstrap where our bootstrap.toml is. RUST_BOOTSTRAP_CONFIG = config; } ``` @@ -388,4 +427,5 @@ load this completion. [`src/etc/rust_analyzer_settings.json`]: https://github.com/rust-lang/rust/blob/master/src/etc/rust_analyzer_settings.json [`src/etc/rust_analyzer_eglot.el`]: https://github.com/rust-lang/rust/blob/master/src/etc/rust_analyzer_eglot.el [`src/etc/rust_analyzer_helix.toml`]: https://github.com/rust-lang/rust/blob/master/src/etc/rust_analyzer_helix.toml +[`src/etc/rust_analyzer_zed.json`]: https://github.com/rust-lang/rust/blob/master/src/etc/rust_analyzer_zed.json [`src/etc/pre-push.sh`]: https://github.com/rust-lang/rust/blob/master/src/etc/pre-push.sh diff --git a/src/cli.md b/src/cli.md index 408ae2070..4c77007ea 100644 --- a/src/cli.md +++ b/src/cli.md @@ -28,6 +28,6 @@ adding a new command-line argument. unstable-options` flag. [cli-docs]: https://doc.rust-lang.org/rustc/command-line-arguments.html -[forge guide for new options]: https://forge.rust-lang.org/compiler/new_option.html +[forge guide for new options]: https://forge.rust-lang.org/compiler/proposals-and-stabilization.html#compiler-flags [unstable book]: https://doc.rust-lang.org/nightly/unstable-book/ [`parse_bool`]: https://github.com/rust-lang/rust/blob/e5335592e78354e33d798d20c04bcd677c1df62d/src/librustc_session/options.rs#L307-L313 diff --git a/src/closure.md b/src/closure.md index 718a0e5d7..427919cd5 100644 --- a/src/closure.md +++ b/src/closure.md @@ -157,7 +157,7 @@ The other option is to step through the code using lldb or gdb. 1. `rust-lldb build/host/stage1/bin/rustc test.rs` 2. In lldb: - 1. `b upvar.rs:134` // Setting the breakpoint on a certain line in the upvar.rs file` + 1. `b upvar.rs:134` // Setting the breakpoint on a certain line in the upvar.rs file 2. `r` // Run the program until it hits the breakpoint Let's start with [`upvar.rs`][upvar]. This file has something called diff --git a/src/coherence.md b/src/coherence.md index b3af101fb..73f9213bf 100644 --- a/src/coherence.md +++ b/src/coherence.md @@ -1,4 +1,3 @@ - # Coherence > NOTE: this is based on [notes by @lcnr](https://github.com/rust-lang/rust/pull/121848) diff --git a/src/compiler-debugging.md b/src/compiler-debugging.md index a2a6c8085..edd2aa6c5 100644 --- a/src/compiler-debugging.md +++ b/src/compiler-debugging.md @@ -1,7 +1,5 @@ # Debugging the compiler - - This chapter contains a few tips to debug the compiler. These tips aim to be useful no matter what you are working on. Some of the other chapters have advice about specific parts of the compiler (e.g. the [Queries Debugging and @@ -11,13 +9,13 @@ chapter](./backend/debugging.md)). ## Configuring the compiler By default, rustc is built without most debug information. To enable debug info, -set `debug = true` in your config.toml. +set `debug = true` in your bootstrap.toml. Setting `debug = true` turns on many different debug options (e.g., `debug-assertions`, `debug-logging`, etc.) which can be individually tweaked if you want to, but many people simply set `debug = true`. -If you want to use GDB to debug rustc, please set `config.toml` with options: +If you want to use GDB to debug rustc, please set `bootstrap.toml` with options: ```toml [rust] @@ -35,14 +33,14 @@ debuginfo-level = 2 The default configuration will enable `symbol-mangling-version` v0. This requires at least GDB v10.2, -otherwise you need to disable new symbol-mangling-version in `config.toml`. +otherwise you need to disable new symbol-mangling-version in `bootstrap.toml`. ```toml [rust] new-symbol-mangling = false ``` -> See the comments in `config.example.toml` for more info. +> See the comments in `bootstrap.example.toml` for more info. You will need to rebuild the compiler after changing any configuration option. @@ -275,7 +273,7 @@ Here are some notable ones: | `rustc_dump_def_parents` | Dumps the chain of `DefId` parents of certain definitions. | | `rustc_dump_item_bounds` | Dumps the [`item_bounds`] of an item. | | `rustc_dump_predicates` | Dumps the [`predicates_of`] an item. | -| `rustc_dump_vtable` | | +| `rustc_dump_vtable` | Dumps the vtable layout of an impl, or a type alias of a dyn type. | | `rustc_hidden_type_of_opaques` | Dumps the [hidden type of each opaque types][opaq] in the crate. | | `rustc_layout` | [See this section](#debugging-type-layouts). | | `rustc_object_lifetime_default` | Dumps the [object lifetime defaults] of an item. | @@ -301,7 +299,8 @@ Right below you can find elaborate explainers on a selected few. Some compiler options for debugging specific features yield graphviz graphs - e.g. the `#[rustc_mir(borrowck_graphviz_postflow="suffix.dot")]` attribute -dumps various borrow-checker dataflow graphs. +on a function dumps various borrow-checker dataflow graphs in conjunction with +`-Zdump-mir-dataflow`. These all produce `.dot` files. To view these files, install graphviz (e.g. `apt-get install graphviz`) and then run the following commands: @@ -368,12 +367,12 @@ error: layout_of(&'a u32) = Layout { error: aborting due to previous error ``` -[`Layout`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_target/abi/struct.Layout.html +[`Layout`]: https://doc.rust-lang.org/nightly/nightly-rustc/stable_mir/abi/struct.Layout.html ## Configuring CodeLLDB for debugging `rustc` -If you are using VSCode, and have edited your `config.toml` to request debugging +If you are using VSCode, and have edited your `bootstrap.toml` to request debugging level 1 or 2 for the parts of the code you're interested in, then you should be able to use the [CodeLLDB] extension in VSCode to debug it. diff --git a/src/compiler-src.md b/src/compiler-src.md index c538fc8b7..d67bacb1b 100644 --- a/src/compiler-src.md +++ b/src/compiler-src.md @@ -1,7 +1,5 @@ # High-level overview of the compiler source - - Now that we have [seen what the compiler does][orgch], let's take a look at the structure of the [`rust-lang/rust`] repository, where the rustc source code lives. @@ -62,21 +60,20 @@ huge. There is also the `rustc` crate which is the actual binary (i.e. the [`rustc_driver`] crate, which drives the various parts of compilation in other crates. -The dependency structure of these crates is complex, but roughly it is +The dependency order of these crates is complex, but roughly it is something like this: -- `rustc` (the binary) calls [`rustc_driver::main`][main]. - - [`rustc_driver`] depends on a lot of other crates, but the main one is - [`rustc_interface`]. - - [`rustc_interface`] depends on most of the other compiler crates. It - is a fairly generic interface for driving the whole compilation. - - Most of the other `rustc_*` crates depend on [`rustc_middle`], - which defines a lot of central data structures in the compiler. - - [`rustc_middle`] and most of the other crates depend on a - handful of crates representing the early parts of the - compiler (e.g. the parser), fundamental data structures (e.g. - [`Span`]), or error reporting: [`rustc_data_structures`], - [`rustc_span`], [`rustc_errors`], etc. +1. `rustc` (the binary) calls [`rustc_driver::main`][main]. +1. [`rustc_driver`] depends on a lot of other crates, but the main one is + [`rustc_interface`]. +1. [`rustc_interface`] depends on most of the other compiler crates. It is a + fairly generic interface for driving the whole compilation. +1. Most of the other `rustc_*` crates depend on [`rustc_middle`], which defines + a lot of central data structures in the compiler. +1. [`rustc_middle`] and most of the other crates depend on a handful of crates + representing the early parts of the compiler (e.g. the parser), fundamental + data structures (e.g. [`Span`]), or error reporting: + [`rustc_data_structures`], [`rustc_span`], [`rustc_errors`], etc. [`rustc_data_structures`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_data_structures/index.html [`rustc_driver`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_driver/index.html @@ -87,8 +84,12 @@ something like this: [`Span`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_span/struct.Span.html [main]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_driver/fn.main.html -You can see the exact dependencies by reading the [`Cargo.toml`] for the various -crates, just like a normal Rust crate. +You can see the exact dependencies by running `cargo tree`, +just like you would for any other Rust package: + +```console +cargo tree --package rustc_driver +``` One final thing: [`src/llvm-project`] is a submodule for our fork of LLVM. During bootstrapping, LLVM is built and the [`compiler/rustc_llvm`] crate diff --git a/src/const-eval.md b/src/const-eval.md index ee0269601..ca6a35a5e 100644 --- a/src/const-eval.md +++ b/src/const-eval.md @@ -17,7 +17,7 @@ Prominent examples are: * need to be known to check for overlapping patterns Additionally constant evaluation can be used to reduce the workload or binary -size at runtime by precomputing complex operations at compiletime and only +size at runtime by precomputing complex operations at compile time and only storing the result. All uses of constant evaluation can either be categorized as "influencing the type system" @@ -35,7 +35,7 @@ They're the wrappers of the `const_eval` query. Statics are special; all other functions do not represent statics correctly and have thus assertions preventing their use on statics. -The `const_eval_*` functions use a [`ParamEnv`](./param_env/param_env_summary.html) of environment +The `const_eval_*` functions use a [`ParamEnv`](./typing_parameter_envs.html) of environment in which the constant is evaluated (e.g. the function within which the constant is used) and a [`GlobalId`]. The `GlobalId` is made up of an `Instance` referring to a constant or static or of an `Instance` of a function and an index into the function's `Promoted` table. diff --git a/src/const-eval/interpret.md b/src/const-eval/interpret.md index 51a539de5..08382b12f 100644 --- a/src/const-eval/interpret.md +++ b/src/const-eval/interpret.md @@ -1,7 +1,5 @@ # Interpreter - - The interpreter is a virtual machine for executing MIR without compiling to machine code. It is usually invoked via `tcx.const_eval_*` functions. The interpreter is shared between the compiler (for compile-time function diff --git a/src/contributing.md b/src/contributing.md index ad1d33265..963bef3af 100644 --- a/src/contributing.md +++ b/src/contributing.md @@ -1,6 +1,4 @@ -# Contribution Procedures - - +# Contribution procedures ## Bug reports @@ -81,7 +79,7 @@ smaller user-facing changes. into a PR that ends up not getting merged!** [See this document][mcpinfo] for more info on MCPs. -[mcpinfo]: https://forge.rust-lang.org/compiler/mcp.html +[mcpinfo]: https://forge.rust-lang.org/compiler/proposals-and-stabilization.html#how-do-i-submit-an-mcp [zulip]: https://rust-lang.zulipchat.com/#narrow/stream/131828-t-compiler ### Performance @@ -150,6 +148,20 @@ when contributing to Rust under [the git section](./git.md). [t-compiler]: https://rust-lang.zulipchat.com/#narrow/stream/131828-t-compiler [triagebot]: https://github.com/rust-lang/rust/blob/master/triagebot.toml +### Keeping your branch up-to-date + +The CI in rust-lang/rust applies your patches directly against the current master, +not against the commit your branch is based on. This can lead to unexpected failures +if your branch is outdated, even when there are no explicit merge conflicts. + +Before submitting or updating a PR, make sure to update your branch +as mentioned [here](git.md#keeping-things-up-to-date) if it's significantly +behind the master branch (e.g., more than 100 commits behind). +This fetches the latest master branch and rebases your changes on top of it, +ensuring your PR is tested against the latest code. + +After rebasing, it's recommended to [run the relevant tests locally](tests/intro.md) to catch any issues before CI runs. + ### r? All pull requests are reviewed by another person. We have a bot, @@ -346,11 +358,11 @@ function in the same way as other pull requests. [`src/doc`]: https://github.com/rust-lang/rust/tree/master/src/doc [std-root]: https://github.com/rust-lang/rust/blob/master/library/std/src/lib.rs#L1 -To find documentation-related issues, sort by the [A-docs label]. +To find documentation-related issues, use the [A-docs label]. You can find documentation style guidelines in [RFC 1574]. -To build the standard library documentation, use `x doc --stage 0 library --open`. +To build the standard library documentation, use `x doc --stage 1 library --open`. To build the documentation for a book (e.g. the unstable book), use `x doc src/doc/unstable-book.` Results should appear in `build/host/doc`, as well as automatically open in your default browser. See [Building Documentation](./building/compiler-documenting.md#building-documentation) for more @@ -373,7 +385,7 @@ Just a few things to keep in mind: There is no strict limit on line lengths; let the sentence or part of the sentence flow to its proper end on the same line. - When contributing text to the guide, please contextualize the information with some time period - and/or a reason so that the reader knows how much to trust or mistrust the information. + and/or a reason so that the reader knows how much to trust the information. Aim to provide a reasonable amount of context, possibly including but not limited to: - A reason for why the data may be out of date other than "change", @@ -387,28 +399,28 @@ Just a few things to keep in mind: - jan 2021 - january 2021 - There is a CI action (in `~/.github/workflows/date-check.yml`) - that generates a monthly showing those that are over 6 months old + There is a CI action (in `.github/workflows/date-check.yml`) + that generates a monthly report showing those that are over 6 months old ([example](https://github.com/rust-lang/rustc-dev-guide/issues/2052)). For the action to pick the date, add a special annotation before specifying the date: ```md - Sep 2024 + Apr 2025 ``` Example: ```md - As of Sep 2024, the foo did the bar. + As of Apr 2025, the foo did the bar. ``` For cases where the date should not be part of the visible rendered output, use the following instead: ```md - + ``` - A link to a relevant WG, tracking issue, `rustc` rustdoc page, or similar, that may provide @@ -420,70 +432,14 @@ Just a few things to keep in mind: it might benefit from having a Table of Contents at the beginning, which you can auto-generate by including the `` marker at the top. -## Issue triage - -Sometimes, an issue will stay open, even though the bug has been fixed. -And sometimes, the original bug may go stale because something has changed in the meantime. - -It can be helpful to go through older bug reports and make sure that they are still valid. -Load up an older issue, double check that it's still true, -and leave a comment letting us know if it is or is not. -The [least recently updated sort][lru] is good for finding issues like this. - -[Thanks to `@rustbot`][rustbot], anyone can help triage issues by adding -appropriate labels to issues that haven't been triaged yet: +#### ⚠️ Note: Where to contribute `rustc-dev-guide` changes -[lru]: https://github.com/rust-lang/rust/issues?q=is%3Aissue+is%3Aopen+sort%3Aupdated-asc -[rustbot]: ./rustbot.md +For detailed information about where to contribute rustc-dev-guide changes and the benefits of doing so, see [the rustc-dev-guide working group documentation](https://forge.rust-lang.org/wg-rustc-dev-guide/index.html#where-to-contribute-rustc-dev-guide-changes). - +Please see . -| Labels | Color | Description | -|--------|-------|-------------| -| [A-] |   Yellow | The **area** of the project an issue relates to. | -| [B-] |   Magenta | Issues which are **blockers**. | -| [beta-] |   Dark Blue | Tracks changes which need to be [backported to beta][beta-backport] | -| [C-] |   Light Purple | The **category** of an issue. | -| [D-] |   Mossy Green | Issues for **diagnostics**. | -| [E-] |   Green | The **experience** level necessary to fix an issue. | -| [F-] |   Peach | Issues for **nightly features**. | -| [I-] |   Red | The **importance** of the issue. | -| [I-\*-nominated] |   Red | The issue has been nominated for discussion at the next meeting of the corresponding team. | -| [I-prioritize] |   Red | The issue has been nominated for prioritization by the team tagged with a **T**-prefixed label. | -| [L-] |   Teal | The relevant **lint**. | -| [metabug] |   Purple | Bugs that collect other bugs. | -| [O-] |   Purple Grey | The **operating system** or platform that the issue is specific to. | -| [P-] |   Orange | The issue **priority**. These labels can be assigned by anyone that understand the issue and is able to prioritize it, and remove the [I-prioritize] label. | -| [regression-] |   Pink | Tracks regressions from a stable release. | -| [relnotes] |   Light Orange | Changes that should be documented in the release notes of the next release. | -| [S-] |   Gray | Tracks the **status** of pull requests. | -| [S-tracking-] |   Steel Blue | Tracks the **status** of [tracking issues]. | -| [stable-] |   Dark Blue | Tracks changes which need to be [backported to stable][stable-backport] in anticipation of a point release. | -| [T-] |   Blue | Denotes which **team** the issue belongs to. | -| [WG-] |   Green | Denotes which **working group** the issue belongs to. | - - -[A-]: https://github.com/rust-lang/rust/labels?q=A -[B-]: https://github.com/rust-lang/rust/labels?q=B -[C-]: https://github.com/rust-lang/rust/labels?q=C -[D-]: https://github.com/rust-lang/rust/labels?q=D -[E-]: https://github.com/rust-lang/rust/labels?q=E -[F-]: https://github.com/rust-lang/rust/labels?q=F -[I-]: https://github.com/rust-lang/rust/labels?q=I -[L-]: https://github.com/rust-lang/rust/labels?q=L -[O-]: https://github.com/rust-lang/rust/labels?q=O -[P-]: https://github.com/rust-lang/rust/labels?q=P -[S-]: https://github.com/rust-lang/rust/labels?q=S -[T-]: https://github.com/rust-lang/rust/labels?q=T -[WG-]: https://github.com/rust-lang/rust/labels?q=WG [stable-]: https://github.com/rust-lang/rust/labels?q=stable [beta-]: https://github.com/rust-lang/rust/labels?q=beta [I-\*-nominated]: https://github.com/rust-lang/rust/labels?q=nominated diff --git a/src/conventions.md b/src/conventions.md index 37af8121c..4356cf246 100644 --- a/src/conventions.md +++ b/src/conventions.md @@ -1,11 +1,13 @@ -This file offers some tips on the coding conventions for rustc. This +# Coding conventions + +This file offers some tips on the coding conventions for rustc. This chapter covers [formatting](#formatting), [coding for correctness](#cc), [using crates from crates.io](#cio), and some tips on [structuring your PR for easy review](#er). -# Formatting and the tidy script +## Formatting and the tidy script rustc is moving towards the [Rust standard coding style][fmt]. @@ -20,31 +22,50 @@ Formatting is checked by the `tidy` script. It runs automatically when you do `./x test` and can be run in isolation with `./x fmt --check`. If you want to use format-on-save in your editor, the pinned version of -`rustfmt` is built under `build//stage0/bin/rustfmt`. You'll have to -pass the `--edition=2021` argument yourself when calling -`rustfmt` directly. +`rustfmt` is built under `build//stage0/bin/rustfmt`. [fmt]: https://github.com/rust-dev-tools/fmt-rfcs [`rustfmt`]:https://github.com/rust-lang/rustfmt -## Formatting C++ code +### Formatting C++ code The compiler contains some C++ code for interfacing with parts of LLVM that don't have a stable C API. When modifying that code, use this command to format it: -```sh -./x test tidy --extra-checks=cpp:fmt --bless +```console +./x test tidy --extra-checks cpp:fmt --bless ``` This uses a pinned version of `clang-format`, to avoid relying on the local environment. +### Formatting and linting Python code + +The Rust repository contains quite a lot of Python code. We try to keep +it both linted and formatted by the [ruff] tool. + +When modifying Python code, use this command to format it: + +```console +./x test tidy --extra-checks py:fmt --bless +``` + +And, the following command to run lints: + +```console +./x test tidy --extra-checks py:lint +``` + +These use a pinned version of `ruff`, to avoid relying on the local environment. + +[ruff]: https://github.com/astral-sh/ruff + -## Copyright notice +### Copyright notice In the past, files began with a copyright and license notice. Please **omit** @@ -54,41 +75,42 @@ MIT/Apache-2.0). All of the copyright notices should be gone by now, but if you come across one in the rust-lang/rust repo, feel free to open a PR to remove it. -## Line length +### Line length Lines should be at most 100 characters. It's even better if you can keep things to 80. -**Ignoring the line length limit.** Sometimes – in particular for -tests – it can be necessary to exempt yourself from this limit. In -that case, you can add a comment towards the top of the file like so: +Sometimes, and particularly for tests, it can be necessary to exempt yourself from this limit. +In that case, you can add a comment towards the top of the file like so: ```rust // ignore-tidy-linelength ``` -## Tabs vs spaces +### Tabs vs spaces -Prefer 4-space indent. +Prefer 4-space indents. -# Coding for correctness +## Coding for correctness Beyond formatting, there are a few other tips that are worth following. -## Prefer exhaustive matches +### Prefer exhaustive matches Using `_` in a match is convenient, but it means that when new variants are added to the enum, they may not get handled correctly. Ask yourself: if a new variant were added to this enum, what's the chance that it would want to use the `_` code, versus having some -other treatment? Unless the answer is "low", then prefer an -exhaustive match. (The same advice applies to `if let` and `while -let`, which are effectively tests for a single variant.) +other treatment? Unless the answer is "low", then prefer an +exhaustive match. + +The same advice applies to `if let` and `while let`, +which are effectively tests for a single variant. -## Use "TODO" comments for things you don't want to forget +### Use "TODO" comments for things you don't want to forget As a useful tool to yourself, you can insert a `// TODO` comment for something that you want to get back to before you land your PR: @@ -115,16 +137,16 @@ if foo { -# Using crates from crates.io +## Using crates from crates.io See the [crates.io dependencies][crates] section. -# How to structure your PR +## How to structure your PR How you prepare the commits in your PR can make a big difference for the -reviewer. Here are some tips. +reviewer. Here are some tips. **Isolate "pure refactorings" into their own commit.** For example, if you rename a method, then put that rename into its own commit, along @@ -151,7 +173,7 @@ require that every intermediate commit successfully builds – we only expect to be able to bisect at a PR level. However, if you *can* make individual commits build, that is always helpful. -# Naming conventions +## Naming conventions Apart from normal Rust style/naming conventions, there are also some specific to the compiler. @@ -165,4 +187,5 @@ to the compiler. crate-related, often the spelling is changed to `krate`. [tcx]: ./ty.md + [crates]: ./crates-io.md diff --git a/src/coroutine-closures.md b/src/coroutine-closures.md index 04742d03c..2617c824a 100644 --- a/src/coroutine-closures.md +++ b/src/coroutine-closures.md @@ -1,6 +1,8 @@ +# Async closures/"coroutine-closures" + Please read [RFC 3668](https://rust-lang.github.io/rfcs/3668-async-closures.html) to understand the general motivation of the feature. This is a very technical and somewhat "vertical" chapter; ideally we'd split this and sprinkle it across all the relevant chapters, but for the purposes of understanding async closures *holistically*, I've put this together all here in one chapter. -# Coroutine-closures -- a technical deep dive +## Coroutine-closures -- a technical deep dive Coroutine-closures are a generalization of async closures, being special syntax for closure expressions which return a coroutine, notably one that is allowed to capture from the closure's upvars. @@ -8,9 +10,11 @@ For now, the only usable kind of coroutine-closure is the async closure, and sup As a consequence of the code being somewhat general, this document may flip between calling them "async closures" and "coroutine-closures". The future that is returned by the async closure will generally be called the "coroutine" or the "child coroutine". -## HIR +### HIR -Async closures (and in the future, other coroutine flavors such as `gen`) are represented in HIR as a `hir::Closure` whose closure-kind is `ClosureKind::CoroutineClosure(_)`[^k1], which wraps an async block, which is also represented in HIR as a `hir::Closure`) and whose closure-kind is `ClosureKind::Closure(CoroutineKind::Desugared(_, CoroutineSource::Closure))`[^k2]. +Async closures (and in the future, other coroutine flavors such as `gen`) are represented in HIR as a `hir::Closure`. +The closure-kind of the `hir::Closure` is `ClosureKind::CoroutineClosure(_)`[^k1], which wraps an async block, which is also represented in HIR as a `hir::Closure`. +The closure-kind of the async block is `ClosureKind::Closure(CoroutineKind::Desugared(_, CoroutineSource::Closure))`[^k2]. [^k1]: @@ -24,7 +28,7 @@ Like `async fn`, when lowering an async closure's body, we need to unconditional [^l3]: -## `rustc_middle::ty` Representation +### `rustc_middle::ty` Representation For the purposes of keeping the implementation mostly future-compatible (i.e. with gen `|| {}` and `async gen || {}`), most of this section calls async closures "coroutine-closures". @@ -72,7 +76,7 @@ To most easily construct the `Coroutine` that a coroutine-closure returns, you c Most of the args to that function will be components that you can get out of the `CoroutineArgs`, except for the `goal_kind: ClosureKind` which controls which flavor of coroutine to return based off of the `ClosureKind` passed in -- i.e. it will prepare the by-ref coroutine if `ClosureKind::Fn | ClosureKind::FnMut`, and the by-move coroutine if `ClosureKind::FnOnce`. -## Trait Hierarchy +### Trait Hierarchy We introduce a parallel hierarchy of `Fn*` traits that are implemented for . The motivation for the introduction was covered in a blog post: [Async Closures](https://hackmd.io/@compiler-errors/async-closures). @@ -98,11 +102,11 @@ We mention above that "regular" callable types can implement `AsyncFn*`, but the See the "follow-up: when do..." section below for an elaborated answer. The full answer describes a pretty interesting and hopefully thorough heuristic that is used to ensure that most async closures "just work". -## Tale of two bodies... +### Tale of two bodies... When async closures are called with `AsyncFn`/`AsyncFnMut`, they return a coroutine that borrows from the closure. However, when they are called via `AsyncFnOnce`, we consume that closure, and cannot return a coroutine that borrows from data that is now dropped. -To work around around this limitation, we synthesize a separate by-move MIR body for calling `AsyncFnOnce::call_once` on a coroutine-closure that can be called by-ref. +To work around this limitation, we synthesize a separate by-move MIR body for calling `AsyncFnOnce::call_once` on a coroutine-closure that can be called by-ref. This body operates identically to the "normal" coroutine returned from calling the coroutine-closure, except for the fact that it has a different set of upvars, since we must *move* the captures from the parent coroutine-closure into the child coroutine. @@ -120,7 +124,7 @@ Since we've synthesized a new def id, this query is also responsible for feeding [^b3]: -## Closure signature inference +### Closure signature inference The closure signature inference algorithm for async closures is a bit more complicated than the inference algorithm for "traditional" closures. Like closures, we iterate through all of the clauses that may be relevant (for the expectation type passed in)[^deduce1]. @@ -173,7 +177,7 @@ s.as_bytes(); So *instead*, we use this alias (in this case, a projection: `AsyncFnKindHelper::Upvars<'env, ...>`) to delay the computation of the *tupled upvars* and give us something to put in its place, while still allowing us to return a `TyKind::Coroutine` (which is a rigid type) and we may successfully confirm the built-in traits we need (in our case, `Future`), since the `Future` implementation doesn't depend on the upvars at all. -## Upvar analysis +### Upvar analysis By and large, the upvar analysis for coroutine-closures and their child coroutines proceeds like normal upvar analysis. However, there are several interesting bits that happen to account for async closures' special natures: @@ -262,7 +266,7 @@ let c = async || { If either of these cases apply, then we should capture the borrow with the lifetime of the parent coroutine-closure's env. Luckily, if this function is not correct, then the program is not unsound, since we still borrowck and validate the choices made from this function -- the only side-effect is that the user may receive unnecessary borrowck errors. -## Instance resolution +### Instance resolution If a coroutine-closure has a closure-kind of `FnOnce`, then its `AsyncFnOnce::call_once` and `FnOnce::call_once` implementations resolve to the coroutine-closure's body[^res1], and the `Future::poll` of the coroutine that gets returned resolves to the body of the child closure. @@ -282,7 +286,7 @@ This is represented by the `ConstructCoroutineInClosureShim`[^i1]. The `receiver [^i3]: -## Borrow-checking +### Borrow-checking It turns out that borrow-checking async closures is pretty straightforward. After adding a new `DefiningTy::CoroutineClosure`[^bck1] variant, and teaching borrowck how to generate the signature of the coroutine-closure[^bck2], borrowck proceeds totally fine. diff --git a/src/crates-io.md b/src/crates-io.md index 403d61a81..677b1fc03 100644 --- a/src/crates-io.md +++ b/src/crates-io.md @@ -1,4 +1,4 @@ -# crates.io Dependencies +# crates.io dependencies The Rust compiler supports building with some dependencies from `crates.io`. Examples are `log` and `env_logger`. @@ -11,7 +11,7 @@ you should avoid adding dependencies to the compiler for several reasons: - The dependency may have transitive dependencies that have one of the above problems. - + Note that there is no official policy for vetting new dependencies to the compiler. Decisions are made on a case-by-case basis, during code review. diff --git a/src/debugging-support-in-rustc.md b/src/debugging-support-in-rustc.md index ac629934e..bd4f795ce 100644 --- a/src/debugging-support-in-rustc.md +++ b/src/debugging-support-in-rustc.md @@ -1,7 +1,5 @@ # Debugging support in the Rust compiler - - This document explains the state of debugging tools support in the Rust compiler (rustc). It gives an overview of GDB, LLDB, WinDbg/CDB, as well as infrastructure around Rust compiler to debug Rust code. diff --git a/src/diagnostics.md b/src/diagnostics.md index ccd101165..82191e0a6 100644 --- a/src/diagnostics.md +++ b/src/diagnostics.md @@ -1,6 +1,4 @@ -# Errors and Lints - - +# Errors and lints A lot of effort has been put into making `rustc` have great error messages. This chapter is about how to emit compile errors and lints from the compiler. @@ -111,7 +109,7 @@ Here are a few examples: their crate, making this a hard error would make refactoring and development very painful. - [future-incompatible lints]: - these are silencable lints. + these are silenceable lints. It was decided that making them fixed errors would cause too much breakage, so warnings are instead emitted, and will eventually be turned into fixed (hard) errors. @@ -553,7 +551,7 @@ compiler](#linting-early-in-the-compiler). [AST nodes]: the-parser.md -[AST lowering]: ast-lowering.md +[AST lowering]: ./hir/lowering.md [HIR nodes]: hir.md [MIR nodes]: mir/index.md [macro expansion]: macro-expansion.md @@ -601,8 +599,8 @@ The trait implementation allows you to check certain syntactic constructs as the linter walks the AST. You can then choose to emit lints in a very similar way to compile errors. -You also declare the metadata of a particular lint via the `declare_lint!` -macro. This includes the name, the default level, a short description, and some +You also declare the metadata of a particular lint via the [`declare_lint!`] +macro. This macro includes the name, the default level, a short description, and some more details. Note that the lint and the lint pass must be registered with the compiler. @@ -671,6 +669,8 @@ example-use-loop = denote infinite loops with `loop {"{"} ... {"}"}` .suggestion = use `loop` ``` +[`declare_lint!`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_lint_defs/macro.declare_lint.html + ### Edition-gated lints Sometimes we want to change the behavior of a lint in a new edition. To do this, @@ -770,7 +770,7 @@ store.register_renamed("single_use_lifetime", "single_use_lifetimes"); [`store.register_removed`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_lint/struct.LintStore.html#method.register_removed [`rustc_lint::register_builtins`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_lint/fn.register_builtins.html -### Lint Groups +### Lint groups Lints can be turned on in groups. These groups are declared in the [`register_builtins`][rbuiltins] function in [`rustc_lint::lib`][builtin]. The @@ -864,19 +864,17 @@ struct](https://doc.rust-lang.org/nightly/nightly-rustc/rustc_errors/json/struct (and sub-structs) for the JSON serialization. Don't confuse this with [`errors::Diag`](https://doc.rust-lang.org/nightly/nightly-rustc/rustc_errors/struct.Diag.html)! -## `#[rustc_on_unimplemented(...)]` +## `#[rustc_on_unimplemented]` -The `#[rustc_on_unimplemented]` attribute allows trait definitions to add specialized -notes to error messages when an implementation was expected but not found. -You can refer to the trait's generic arguments by name and to the resolved type using `Self`. - -For example: +This attribute allows trait definitions to modify error messages when an implementation was +expected but not found. The string literals in the attribute are format strings and can be +formatted with named parameters. See the Formatting +section below for what parameters are permitted. ```rust,ignore -#![feature(rustc_attrs)] - -#[rustc_on_unimplemented="an iterator over elements of type `{A}` \ - cannot be built from a collection of type `{Self}`"] +#[rustc_on_unimplemented(message = "an iterator over \ + elements of type `{A}` cannot be built from a \ + collection of type `{Self}`")] trait MyIterator { fn next(&mut self) -> A; } @@ -893,32 +891,26 @@ fn main() { When the user compiles this, they will see the following; ```txt -error[E0277]: the trait bound `&[{integer}]: MyIterator` is not satisfied - --> :14:5 +error[E0277]: an iterator over elements of type `char` cannot be built from a collection of type `&[{integer}]` + --> src/main.rs:13:19 | -14 | iterate_chars(&[1, 2, 3][..]); - | ^^^^^^^^^^^^^ an iterator over elements of type `char` cannot be built from a collection of type `&[{integer}]` +13 | iterate_chars(&[1, 2, 3][..]); + | ------------- ^^^^^^^^^^^^^^ the trait `MyIterator` is not implemented for `&[{integer}]` + | | + | required by a bound introduced by this call | - = help: the trait `MyIterator` is not implemented for `&[{integer}]` - = note: required by `iterate_chars` +note: required by a bound in `iterate_chars` ``` -`rustc_on_unimplemented` also supports advanced filtering for better targeting -of messages, as well as modifying specific parts of the error message. You -target the text of: - +You can modify the contents of: - the main error message (`message`) - the label (`label`) - - an extra note (`note`) + - the note(s) (`note`) For example, the following attribute ```rust,ignore -#[rustc_on_unimplemented( - message="message", - label="label", - note="note" -)] +#[rustc_on_unimplemented(message = "message", label = "label", note = "note")] trait MyIterator { fn next(&mut self) -> A; } @@ -928,48 +920,61 @@ Would generate the following output: ```text error[E0277]: message - --> :14:5 + --> :10:19 | -14 | iterate_chars(&[1, 2, 3][..]); - | ^^^^^^^^^^^^^ label +10 | iterate_chars(&[1, 2, 3][..]); + | ------------- ^^^^^^^^^^^^^^ label + | | + | required by a bound introduced by this call | - = note: note = help: the trait `MyIterator` is not implemented for `&[{integer}]` - = note: required by `iterate_chars` + = note: note +note: required by a bound in `iterate_chars` ``` +The functionality discussed so far is also available with +[`#[diagnostic::on_unimplemented]`](https://doc.rust-lang.org/nightly/reference/attributes/diagnostics.html#the-diagnosticon_unimplemented-attribute). +If you can, you should use that instead. + +### Filtering + To allow more targeted error messages, it is possible to filter the -application of these fields based on a variety of attributes when using -`on`: +application of these fields with `on`. +You can filter on the following boolean flags: - `crate_local`: whether the code causing the trait bound to not be fulfilled is part of the user's crate. This is used to avoid suggesting code changes that would require modifying a dependency. - - Any of the generic arguments that can be substituted in the text can be - referred by name as well for filtering, like `Rhs="i32"`, except for - `Self`. - - `_Self`: to filter only on a particular calculated trait resolution, like - `Self="std::iter::Iterator"`. This is needed because `Self` is a - keyword which cannot appear in attributes. - - `direct`: user-specified rather than derived obligation. - - `from_method`: usable both as boolean (whether the flag is present, like - `crate_local`) or matching against a particular method. Currently used - for `try`. - - `from_desugaring`: usable both as boolean (whether the flag is present) - or matching against a particular desugaring. The desugaring is identified - with its variant name in the `DesugaringKind` enum. - -For example, the `Iterator` trait can be annotated in the following way: + - `direct`: whether this is an user-specified rather than derived obligation. + - `from_desugaring`: whether we are in some kind of desugaring, like `?` + or a `try` block for example. This flag can also be matched on, see below. + +You can match on the following names and values, using `name = "value"`: + - `cause`: Match against one variant of the `ObligationCauseCode` + enum. Only `"MainFunctionType"` is supported. + - `from_desugaring`: Match against a particular variant of the `DesugaringKind` + enum. The desugaring is identified by its variant name, for example + `"QuestionMark"` for `?` desugaring or `"TryBlock"` for `try` blocks. + - `Self` and any generic arguments of the trait, like `Self = "alloc::string::String"` + or `Rhs="i32"`. + +The compiler can provide several values to match on, for example: + - the self_ty, pretty printed with and without type arguments resolved. + - `"{integral}"`, if self_ty is an integral of which the type is known. + - `"[]"`, `"[{ty}]"`, `"[{ty}; _]"`, `"[{ty}; $N]"` when applicable. + - references to said slices and arrays. + - `"fn"`, `"unsafe fn"` or `"#[target_feature] fn"` when self is a function. + - `"{integer}"` and `"{float}"` if the type is a number but we haven't inferred it yet. + - combinations of the above, like `"[{integral}; _]"`. + +For example, the `Iterator` trait can be filtered in the following way: ```rust,ignore #[rustc_on_unimplemented( - on( - _Self="&str", - note="call `.chars()` or `.as_bytes()` on `{Self}`" - ), - message="`{Self}` is not an iterator", - label="`{Self}` is not an iterator", - note="maybe try calling `.iter()` or a similar method" + on(Self = "&str", note = "call `.chars()` or `.as_bytes()` on `{Self}`"), + message = "`{Self}` is not an iterator", + label = "`{Self}` is not an iterator", + note = "maybe try calling `.iter()` or a similar method" )] pub trait Iterator {} ``` @@ -998,15 +1003,47 @@ error[E0277]: `&str` is not an iterator = note: required by `std::iter::IntoIterator::into_iter` ``` -If you need to filter on multiple attributes, you can use `all`, `any` or -`not` in the following way: +The `on` filter accepts `all`, `any` and `not` predicates similar to the `cfg` attribute: ```rust,ignore -#[rustc_on_unimplemented( - on( - all(_Self="&str", T="std::string::String"), - note="you can coerce a `{T}` into a `{Self}` by writing `&*variable`" - ) -)] -pub trait From: Sized { /* ... */ } +#[rustc_on_unimplemented(on( + all(Self = "&str", T = "alloc::string::String"), + note = "you can coerce a `{T}` into a `{Self}` by writing `&*variable`" +))] +pub trait From: Sized { + /* ... */ +} +``` + +### Formatting + +The string literals are format strings that accept parameters wrapped in braces +but positional and listed parameters and format specifiers are not accepted. +The following parameter names are valid: +- `Self` and all generic parameters of the trait. +- `This`: the name of the trait the attribute is on, without generics. +- `Trait`: the name of the "sugared" trait. See `TraitRefPrintSugared`. +- `ItemContext`: the kind of `hir::Node` we're in, things like `"an async block"`, + `"a function"`, `"an async function"`, etc. + +Something like: + +```rust,ignore +#![feature(rustc_attrs)] + +#[rustc_on_unimplemented(message = "Self = `{Self}`, \ + T = `{T}`, this = `{This}`, trait = `{Trait}`, \ + context = `{ItemContext}`")] +pub trait From: Sized { + fn from(x: T) -> Self; +} + +fn main() { + let x: i8 = From::from(42_i32); +} +``` + +Will format the message into +```text +"Self = `i8`, T = `i32`, this = `From`, trait = `From`, context = `a function`" ``` diff --git a/src/diagnostics/diagnostic-structs.md b/src/diagnostics/diagnostic-structs.md index e01b8f2f1..4e5c3413c 100644 --- a/src/diagnostics/diagnostic-structs.md +++ b/src/diagnostics/diagnostic-structs.md @@ -1,14 +1,18 @@ # Diagnostic and subdiagnostic structs rustc has three diagnostic traits that can be used to create diagnostics: -`Diagnostic`, `LintDiagnostic`, and `Subdiagnostic`. For simple diagnostics, -instead of using the `Diag` API to create and emit diagnostics, -derived impls can be used. They are only suitable for simple diagnostics that +`Diagnostic`, `LintDiagnostic`, and `Subdiagnostic`. + +For simple diagnostics, +derived impls can be used, e.g. `#[derive(Diagnostic)]`. They are only suitable for simple diagnostics that don't require much logic in deciding whether or not to add additional subdiagnostics. -Such diagnostic can be translated into -different languages and each has a slug that uniquely identifies the -diagnostic. +In cases where diagnostics require more complex or dynamic behavior, such as conditionally adding subdiagnostics, +customizing the rendering logic, or selecting messages at runtime, you will need to manually implement +the corresponding trait (`Diagnostic`, `LintDiagnostic`, or `Subdiagnostic`). +This approach provides greater flexibility and is recommended for diagnostics that go beyond simple, static structures. + +Diagnostic can be translated into different languages and each has a slug that uniquely identifies the diagnostic. ## `#[derive(Diagnostic)]` and `#[derive(LintDiagnostic)]` @@ -142,7 +146,7 @@ tcx.dcx().emit_err(FieldAlreadyDeclared { }); ``` -### Reference +### Reference for `#[derive(Diagnostic)]` and `#[derive(LintDiagnostic)]` `#[derive(Diagnostic)]` and `#[derive(LintDiagnostic)]` support the following attributes: @@ -330,7 +334,34 @@ function ([example][subdiag_use_1] and [example][subdiag_use_2]) on a diagnostic or by assigning it to a `#[subdiagnostic]`-annotated field of a diagnostic struct. -### Reference +### Argument sharing and isolation + +Subdiagnostics add their own arguments (i.e., certain fields in their structure) to the `Diag` structure before rendering the information. +`Diag` structure also stores the arguments from the main diagnostic, so the subdiagnostic can also use the arguments from the main diagnostic. + +However, when a subdiagnostic is added to a main diagnostic by implementing `#[derive(Subdiagnostic)]`, +the following rules, introduced in [rust-lang/rust#142724](https://github.com/rust-lang/rust/pull/142724) +apply to the handling of arguments (i.e., variables used in Fluent messages): + +**Argument isolation between sub diagnostics**: +Arguments set by a subdiagnostic are only available during the rendering of that subdiagnostic. +After the subdiagnostic is rendered, all arguments it introduced are restored from the main diagnostic. +This ensures that multiple subdiagnostics do not pollute each other's argument scope. +For example, when using a `Vec`, it iteratively adds the same argument over and over again. + +**Same argument override between sub and main diagnostics**: +If a subdiagnostic sets a argument with the same name as a arguments already in the main diagnostic, +it will report an error at runtime unless both have exactly the same value. +It has two benefits: +- preserves the flexibility that arguments in the main diagnostic are allowed to appear in the attributes of the subdiagnostic. +For example, There is an attribute `#[suggestion(code = "{new_vis}")]` in the subdiagnostic, but `new_vis` is the field in the main diagnostic struct. +- prevents accidental overwriting or deletion of arguments required by the main diagnostic or other subdiagnostics. + +These rules guarantee that arguments injected by subdiagnostics are strictly scoped to their own rendering. +The main diagnostic's arguments remain unaffected by subdiagnostic logic, even in the presence of name collisions. +Additionally, subdiagnostics can access arguments from the main diagnostic with the same name when needed. + +### Reference for `#[derive(Subdiagnostic)]` `#[derive(Subdiagnostic)]` supports the following attributes: - `#[label(slug)]`, `#[help(slug)]`, `#[warning(slug)]` or `#[note(slug)]` diff --git a/src/diagnostics/error-codes.md b/src/diagnostics/error-codes.md index 1b6b87e4c..1693432b9 100644 --- a/src/diagnostics/error-codes.md +++ b/src/diagnostics/error-codes.md @@ -20,7 +20,7 @@ explanations should help users understand why their code cannot be accepted by the compiler. Rust prides itself on helpful error messages and long-form explanations are no exception. However, before error explanations are overhauled[^new-explanations] it is a bit open as to how exactly they should be -written, as always: ask your reviewer or ask around on the Rust Discord or Zulip. +written, as always: ask your reviewer or ask around on the Rust Zulip. [^new-explanations]: See the draft RFC [here][new-explanations-rfc]. diff --git a/src/diagnostics/lintstore.md b/src/diagnostics/lintstore.md index bd2b02529..7b98bc621 100644 --- a/src/diagnostics/lintstore.md +++ b/src/diagnostics/lintstore.md @@ -54,7 +54,7 @@ Lints are registered via the [`LintStore::register_lint`] function. This should happen just once for any lint, or an ICE will occur. Once the registration is complete, we "freeze" the lint store by placing it in -an `Lrc`. +an `Arc`. Lint passes are registered separately into one of the categories (pre-expansion, early, late, late module). Passes are registered as a closure diff --git a/src/early_late_parameters.md b/src/early_late_parameters.md index 6d1365529..c472bdc2c 100644 --- a/src/early_late_parameters.md +++ b/src/early_late_parameters.md @@ -1,8 +1,6 @@ # Early vs Late bound parameters - - > **NOTE**: This chapter largely talks about early/late bound as being solely relevant when discussing function item types/function definitions. This is potentially not completely true, async blocks and closures should likely be discussed somewhat in this chapter. ## What does it mean to be "early" bound or "late" bound @@ -126,9 +124,9 @@ In this example we call `foo`'s function item type twice, each time with a borro If the lifetime parameter on `foo` was late bound this would be able to compile as each caller could provide a different lifetime argument for its borrow. See the following example which demonstrates this using the `bar` function defined above: ```rust -#fn foo<'a: 'a>(b: &'a String) -> &'a String { b } -#fn bar<'a>(b: &'a String) -> &'a String { b } - +# fn foo<'a: 'a>(b: &'a String) -> &'a String { b } +# fn bar<'a>(b: &'a String) -> &'a String { b } +# // Early bound parameters are instantiated here, however as `'a` is // late bound it is not provided here. let b = bar; @@ -174,7 +172,8 @@ As mentioned previously, the distinction between early and late bound parameters - When naming a function (early) - When calling a function (late) -There currently is no syntax for explicitly specifying generic arguments for late bound parameters as part of the call step, only specifying generic arguments when naming a function. The syntax `foo::<'static>();`, despite being part of a function call, behaves as `(foo::<'static>)();` and instantiates the early bound generic parameters on the function item type. +There is currently no syntax for explicitly specifying generic arguments for late bound parameters during the call step; generic arguments can only be specified for early bound parameters when naming a function. +The syntax `foo::<'static>();`, despite being part of a function call, behaves as `(foo::<'static>)();` and instantiates the early bound generic parameters on the function item type. See the following example: ```rust @@ -220,24 +219,24 @@ Then, for the first case, we can call each function with a single lifetime argum ```rust #![deny(late_bound_lifetime_arguments)] -#fn free_function<'a: 'a, 'b>(_: &'a (), _: &'b ()) {} +# fn free_function<'a: 'a, 'b>(_: &'a (), _: &'b ()) {} # -#struct Foo; +# struct Foo; # -#trait Trait: Sized { -# fn trait_method<'a: 'a, 'b>(self, _: &'a (), _: &'b ()); -# fn trait_function<'a: 'a, 'b>(_: &'a (), _: &'b ()); -#} +# trait Trait: Sized { +# fn trait_method<'a: 'a, 'b>(self, _: &'a (), _: &'b ()); +# fn trait_function<'a: 'a, 'b>(_: &'a (), _: &'b ()); +# } # -#impl Trait for Foo { -# fn trait_method<'a: 'a, 'b>(self, _: &'a (), _: &'b ()) {} -# fn trait_function<'a: 'a, 'b>(_: &'a (), _: &'b ()) {} -#} +# impl Trait for Foo { +# fn trait_method<'a: 'a, 'b>(self, _: &'a (), _: &'b ()) {} +# fn trait_function<'a: 'a, 'b>(_: &'a (), _: &'b ()) {} +# } # -#impl Foo { -# fn inherent_method<'a: 'a, 'b>(self, _: &'a (), _: &'b ()) {} -# fn inherent_function<'a: 'a, 'b>(_: &'a (), _: &'b ()) {} -#} +# impl Foo { +# fn inherent_method<'a: 'a, 'b>(self, _: &'a (), _: &'b ()) {} +# fn inherent_function<'a: 'a, 'b>(_: &'a (), _: &'b ()) {} +# } # // Specifying as many arguments as there are early // bound parameters is always a future compat warning @@ -251,24 +250,24 @@ free_function::<'static>(&(), &()); For the second case we call each function with more lifetime arguments than there are lifetime parameters (be it early or late bound) and note that method calls result in a FCW as opposed to the free/associated functions which result in a hard error: ```rust -#fn free_function<'a: 'a, 'b>(_: &'a (), _: &'b ()) {} +# fn free_function<'a: 'a, 'b>(_: &'a (), _: &'b ()) {} # -#struct Foo; +# struct Foo; # -#trait Trait: Sized { -# fn trait_method<'a: 'a, 'b>(self, _: &'a (), _: &'b ()); -# fn trait_function<'a: 'a, 'b>(_: &'a (), _: &'b ()); -#} +# trait Trait: Sized { +# fn trait_method<'a: 'a, 'b>(self, _: &'a (), _: &'b ()); +# fn trait_function<'a: 'a, 'b>(_: &'a (), _: &'b ()); +# } # -#impl Trait for Foo { -# fn trait_method<'a: 'a, 'b>(self, _: &'a (), _: &'b ()) {} -# fn trait_function<'a: 'a, 'b>(_: &'a (), _: &'b ()) {} -#} +# impl Trait for Foo { +# fn trait_method<'a: 'a, 'b>(self, _: &'a (), _: &'b ()) {} +# fn trait_function<'a: 'a, 'b>(_: &'a (), _: &'b ()) {} +# } # -#impl Foo { -# fn inherent_method<'a: 'a, 'b>(self, _: &'a (), _: &'b ()) {} -# fn inherent_function<'a: 'a, 'b>(_: &'a (), _: &'b ()) {} -#} +# impl Foo { +# fn inherent_method<'a: 'a, 'b>(self, _: &'a (), _: &'b ()) {} +# fn inherent_function<'a: 'a, 'b>(_: &'a (), _: &'b ()) {} +# } # // Specifying more arguments than there are early // bound parameters is a future compat warning when @@ -421,4 +420,4 @@ impl<'a> Fn<()> for FooFnItem<'a> { type Output = &'a String; /* fn call(...) -> ... { ... } */ } -``` \ No newline at end of file +``` diff --git a/src/effects.md b/src/effects.md index 1fda7bcbb..87b0103a7 100644 --- a/src/effects.md +++ b/src/effects.md @@ -1,66 +1,156 @@ -# Effects and effect checking - -Note: all of this describes the implementation of the unstable `effects` and -`const_trait_impl` features. None of this implementation is usable or visible from -stable Rust. - -The implementation of const traits and `~const` bounds is a limited effect system. -It is used to allow trait bounds on `const fn` to be used within the `const fn` for -method calls. Within the function, in order to know whether a method on a trait -bound is `const`, we need to know whether there is a `~const` bound for the trait. -In order to know whether we can instantiate a `~const` bound on a `const fn`, we -need to know whether there is a `const_trait` impl for the type and trait being -used (or whether the `const fn` is used at runtime, then any type implementing the -trait is ok, just like with other bounds). - -We perform these checks via a const generic boolean that gets attached to all -`const fn` and `const trait`. The following sections will explain the desugarings -and the way we perform the checks at call sites. - -The const generic boolean is inverted to the meaning of `const`. In the compiler -it is called `host`, because it enables "host APIs" like `static` items, network -access, disk access, random numbers and everything else that isn't available in -`const` contexts. So `false` means "const", `true` means "not const" and if it's -a generic parameter, it means "maybe const" (meaning we're in a const fn or const -trait). - -## `const fn` - -All `const fn` have a `#[rustc_host] const host: bool` generic parameter that is -hidden from users. Any `~const Trait` bounds in the generics list or `where` bounds -of a `const fn` get converted to `Trait + Trait` bounds. The `Trait` -exists so that associated types of the generic param can be used from projections -like `::Assoc`, because there are no `` projections for now. - -## `#[const_trait] trait`s - -The `#[const_trait]` attribute gives the marked trait a `#[rustc_host] const host: bool` -generic parameter. All functions of the trait "inherit" this generic parameter, just like -they have all the regular generic parameters of the trait. Any `~const Trait` super-trait -bounds get desugared to `Trait + Trait` in order to allow using associated -types and consts of the super traits in the trait declaration. This is necessary, because -`::Assoc` is always `>::Assoc` as there is -no `` syntax. - -## `typeck` performing method and function call checks. - -When generic parameters are instantiated for any items, the `host` generic parameter -is always instantiated as an inference variable. This is a special kind of inference var -that is not part of the type or const inference variables, similar to how we have -special inference variables for type variables that we know to be an integer, but not -yet which one. These separate inference variables fall back to `true` at -the end of typeck (in `fallback_effects`) to ensure that `let _ = some_fn_item_name;` -will keep compiling. - -All actually used (in function calls, casts, or anywhere else) function items, will -have the `enforce_context_effects` method invoked. -It trivially returns if the function being called has no `host` generic parameter. - -In order to error if a non-const function is called in a const context, we have not -yet disabled the const-check logic that happens on MIR, because -`enforce_context_effects` does not yet perform this check. - -The function call's `host` parameter is then equated to the context's `host` value, -which almost always trivially succeeds, as it was an inference var. If the inference -var has already been bound (since the function item is invoked twice), the second -invocation checks it against the first. +# Effects and const condition checking + +## The `HostEffect` predicate + +[`HostEffectPredicate`]s are a kind of predicate from `~const Tr` or `const Tr` +bounds. It has a trait reference, and a `constness` which could be `Maybe` or +`Const` depending on the bound. Because `~const Tr`, or rather `Maybe` bounds +apply differently based on whichever contexts they are in, they have different +behavior than normal bounds. Where normal trait bounds on a function such as +`T: Tr` are collected within the [`predicates_of`] query to be proven when a +function is called and to be assumed within the function, bounds such as +`T: ~const Tr` will behave as a normal trait bound and add `T: Tr` to the result +from `predicates_of`, but also adds a `HostEffectPredicate` to the +[`const_conditions`] query. + +On the other hand, `T: const Tr` bounds do not change meaning across contexts, +therefore they will result in `HostEffect(T: Tr, const)` being added to +`predicates_of`, and not `const_conditions`. + +[`HostEffectPredicate`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_type_ir/predicate/struct.HostEffectPredicate.html +[`predicates_of`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/struct.TyCtxt.html#method.predicates_of +[`const_conditions`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/struct.TyCtxt.html#method.const_conditions + +## The `const_conditions` query + +`predicates_of` represents a set of predicates that need to be proven to use an +item. For example, to use `foo` in the example below: + +```rust +fn foo() where T: Default {} +``` + +We must be able to prove that `T` implements `Default`. In a similar vein, +`const_conditions` represents a set of predicates that need to be proven to use +an item *in const contexts*. If we adjust the example above to use `const` trait +bounds: + +```rust +const fn foo() where T: ~const Default {} +``` + +Then `foo` would get a `HostEffect(T: Default, maybe)` in the `const_conditions` +query, suggesting that in order to call `foo` from const contexts, one must +prove that `T` has a const implementation of `Default`. + +## Enforcement of `const_conditions` + +`const_conditions` are currently checked in various places. + +Every call in HIR from a const context (which includes `const fn` and `const` +items) will check that `const_conditions` of the function we are calling hold. +This is done in [`FnCtxt::enforce_context_effects`]. Note that we don't check +if the function is only referred to but not called, as the following code needs +to compile: + +```rust +const fn hi() -> T { + T::default() +} +const X: fn() -> u32 = hi::; +``` + +For a trait `impl` to be well-formed, we must be able to prove the +`const_conditions` of the trait from the `impl`'s environment. This is checked +in [`wfcheck::check_impl`]. + +Here's an example: + +```rust +const trait Bar {} +const trait Foo: ~const Bar {} +// `const_conditions` contains `HostEffect(Self: Bar, maybe)` + +impl const Bar for () {} +impl const Foo for () {} +// ^ here we check `const_conditions` for the impl to be well-formed +``` + +Methods of trait impls must not have stricter bounds than the method of the +trait that they are implementing. To check that the methods are compatible, a +hybrid environment is constructed with the predicates of the `impl` plus the +predicates of the trait method, and we attempt to prove the predicates of the +impl method. We do the same for `const_conditions`: + +```rust +const trait Foo { + fn hi(); +} + +impl Foo for Vec { + fn hi(); + // ^ we can't prove `T: ~const PartialEq` given `T: ~const Clone` and + // `T: ~const Default`, therefore we know that the method on the impl + // is stricter than the method on the trait. +} +``` + +These checks are done in [`compare_method_predicate_entailment`]. A similar +function that does the same check for associated types is called +[`compare_type_predicate_entailment`]. Both of these need to consider +`const_conditions` when in const contexts. + +In MIR, as part of const checking, `const_conditions` of items that are called +are revalidated again in [`Checker::revalidate_conditional_constness`]. + +[`compare_method_predicate_entailment`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir_analysis/check/compare_impl_item/fn.compare_method_predicate_entailment.html +[`compare_type_predicate_entailment`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir_analysis/check/compare_impl_item/fn.compare_type_predicate_entailment.html +[`FnCtxt::enforce_context_effects`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir_typeck/fn_ctxt/struct.FnCtxt.html#method.enforce_context_effects +[`wfcheck::check_impl`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir_analysis/check/wfcheck/fn.check_impl.html +[`Checker::revalidate_conditional_constness`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_const_eval/check_consts/check/struct.Checker.html#method.revalidate_conditional_constness + +## `explicit_implied_const_bounds` on associated types and traits + +Bounds on associated types, opaque types, and supertraits such as +```rust +trait Foo: ~const PartialEq { + type X: ~const PartialEq; +} + +fn foo() -> impl ~const PartialEq { + // ^ unimplemented syntax +} +``` + +Have their bounds represented differently. Unlike `const_conditions` which need +to be proved for callers, and can be assumed inside the definition (e.g. trait +bounds on functions), these bounds need to be proved at definition (at the impl, +or when returning the opaque) but can be assumed for callers. The non-const +equivalent of these bounds are called [`explicit_item_bounds`]. + +These bounds are checked in [`compare_impl_item::check_type_bounds`] for HIR +typeck, [`evaluate_host_effect_from_item_bounds`] in the old solver and +[`consider_additional_alias_assumptions`] in the new solver. + +[`explicit_item_bounds`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/struct.TyCtxt.html#method.explicit_item_bounds +[`compare_impl_item::check_type_bounds`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir_analysis/check/compare_impl_item/fn.check_type_bounds.html +[`evaluate_host_effect_from_item_bounds`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_trait_selection/traits/effects/fn.evaluate_host_effect_from_item_bounds.html +[`consider_additional_alias_assumptions`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_next_trait_solver/solve/assembly/trait.GoalKind.html#tymethod.consider_additional_alias_assumptions + +## Proving `HostEffectPredicate`s + +`HostEffectPredicate`s are implemented both in the [old solver] and the [new +trait solver]. In general, we can prove a `HostEffect` predicate when either of +these conditions are met: + +* The predicate can be assumed from caller bounds; +* The type has a `const` `impl` for the trait, *and* that const conditions on +the impl holds, *and* that the `explicit_implied_const_bounds` on the trait +holds; or +* The type has a built-in implementation for the trait in const contexts. For +example, `Fn` may be implemented by function items if their const conditions +are satisfied, or `Destruct` is implemented in const contexts if the type can +be dropped at compile time. + +[old solver]: https://doc.rust-lang.org/nightly/nightly-rustc/src/rustc_trait_selection/traits/effects.rs.html +[new trait solver]: https://doc.rust-lang.org/nightly/nightly-rustc/src/rustc_next_trait_solver/solve/effect_goals.rs.html diff --git a/src/external-repos.md b/src/external-repos.md index a7ab3d773..5fb7eeee8 100644 --- a/src/external-repos.md +++ b/src/external-repos.md @@ -9,7 +9,7 @@ There are three main ways we use dependencies: As a general rule: - Use crates.io for libraries that could be useful for others in the ecosystem - Use subtrees for tools that depend on compiler internals and need to be updated if there are breaking -changes + changes - Use submodules for tools that are independent of the compiler ## External Dependencies (subtrees) @@ -18,9 +18,13 @@ The following external projects are managed using some form of a `subtree`: * [clippy](https://github.com/rust-lang/rust-clippy) * [miri](https://github.com/rust-lang/miri) +* [portable-simd](https://github.com/rust-lang/portable-simd) * [rustfmt](https://github.com/rust-lang/rustfmt) * [rust-analyzer](https://github.com/rust-lang/rust-analyzer) * [rustc_codegen_cranelift](https://github.com/rust-lang/rustc_codegen_cranelift) +* [rustc-dev-guide](https://github.com/rust-lang/rustc-dev-guide) +* [compiler-builtins](https://github.com/rust-lang/compiler-builtins) +* [stdarch](https://github.com/rust-lang/stdarch) In contrast to `submodule` dependencies (see below for those), the `subtree` dependencies are just regular files and directories which can @@ -32,18 +36,61 @@ implement a new tool feature or test, that should happen in one collective rustc `subtree` dependencies are currently managed by two distinct approaches: * Using `git subtree` - * `clippy` ([sync guide](https://doc.rust-lang.org/nightly/clippy/development/infrastructure/sync.html#performing-the-sync-from-rust-langrust-to-clippy)) - * `rustfmt` - * `rustc_codegen_cranelift` ([sync script](https://github.com/rust-lang/rustc_codegen_cranelift/blob/113af154d459e41b3dc2c5d7d878e3d3a8f33c69/scripts/rustup.sh#L7)) -* Using the [josh] tool - * `miri` ([sync guide](https://github.com/rust-lang/miri/blob/master/CONTRIBUTING.md#advanced-topic-syncing-with-the-rustc-repo)) - * `rust-analyzer` ([sync script](https://github.com/rust-lang/rust-analyzer/blob/2e13684be123eca7181aa48e043e185d8044a84a/xtask/src/release.rs#L147)) + * `clippy` ([sync guide](https://doc.rust-lang.org/nightly/clippy/development/infrastructure/sync.html#performing-the-sync-from-rust-langrust-to-clippy)) + * `portable-simd` ([sync script](https://github.com/rust-lang/portable-simd/blob/master/subtree-sync.sh)) + * `rustfmt` + * `rustc_codegen_cranelift` ([sync script](https://github.com/rust-lang/rustc_codegen_cranelift/blob/113af154d459e41b3dc2c5d7d878e3d3a8f33c69/scripts/rustup.sh#L7)) +* Using the [josh](#synchronizing-a-josh-subtree) tool + * `miri` + * `rust-analyzer` + * `rustc-dev-guide` + * `compiler-builtins` + * `stdarch` -The [josh] tool is an alternative to git subtrees, which manages git history in a different way and scales better to larger repositories. Specific tooling is required to work with josh, you can check out the `miri` or `rust-analyzer` scripts linked above for inspiration. If you want to migrate a subtree from `git subtree` to josh, you can check out [this guide](https://hackmd.io/7pOuxnkdQDaL1Y1FQr65xg). +### Josh subtrees -Below you can find a guide on how to perform push and pull synchronization with the main rustc repo using `git subtree`, although these instructions might differ repo from repo. +The [josh] tool is an alternative to git subtrees, which manages git history in a different way and scales better to larger repositories. Specific tooling is required to work with josh. We provide a helper [`rustc-josh-sync`][josh-sync] tool to help with the synchronization, described [below](#synchronizing-a-josh-subtree). -### Synchronizing a subtree +### Synchronizing a Josh subtree + +We use a dedicated tool called [`rustc-josh-sync`][josh-sync] for performing Josh subtree updates. +The commands below can be used for all our Josh subtrees, although note that `miri` +requires you to perform some [additional steps](https://github.com/rust-lang/miri/blob/master/CONTRIBUTING.md#advanced-topic-syncing-with-the-rustc-repo) during pulls. + +You can install the tool using the following command: +``` +cargo install --locked --git https://github.com/rust-lang/josh-sync +``` + +Both pulls (synchronize changes from rust-lang/rust into the subtree) and pushes (synchronize +changes from the subtree to rust-lang/rust) are performed from the subtree repository (so first +switch to its repository checkout directory in your terminal). + +#### Performing pull +1) Checkout a new branch that will be used to create a PR into the subtree +2) Run the pull command + ``` + rustc-josh-sync pull + ``` +3) Push the branch to your fork and create a PR into the subtree repository + - If you have `gh` CLI installed, `rustc-josh-sync` can create the PR for you. + +#### Performing push + +> NOTE: +> Before you proceed, look at some guidance related to Git [on josh-sync README]. + +1) Run the push command to create a branch named `` in a `rustc` fork under the `` account + ``` + rustc-josh-sync push + ``` +2) Create a PR from `` into `rust-lang/rust` + +### Creating a new Josh subtree dependency + +If you want to migrate a repository dependency from `git subtree` or `git submodule` to josh, you can check out [this guide](https://hackmd.io/7pOuxnkdQDaL1Y1FQr65xg). + +### Synchronizing a git subtree Periodically the changes made to subtree based dependencies need to be synchronized between this repository and the upstream tool repositories. @@ -125,3 +172,5 @@ the week leading up to the beta cut. [toolstate website]: https://rust-lang-nursery.github.io/rust-toolstate/ [Toolstate chapter]: https://forge.rust-lang.org/infra/toolstate.html [josh]: https://josh-project.github.io/josh/intro.html +[josh-sync]: https://github.com/rust-lang/josh-sync +[on josh-sync README]: https://github.com/rust-lang/josh-sync#git-peculiarities diff --git a/src/feature-gates.md b/src/feature-gates.md index 24ce9bb71..9806f73c4 100644 --- a/src/feature-gates.md +++ b/src/feature-gates.md @@ -1,4 +1,4 @@ -# Feature Gates +# Feature gates This chapter is intended to provide basic help for adding, removing, and modifying feature gates. diff --git a/src/fuzzing.md b/src/fuzzing.md index 869fc2f71..300053786 100644 --- a/src/fuzzing.md +++ b/src/fuzzing.md @@ -73,21 +73,32 @@ To build a corpus, you may want to use: - The rustc/rust-analyzer/clippy test suites (or even source code) --- though avoid tests that are already known to cause failures, which often begin with comments - like `// failure-status: 101` or `// known-bug: #NNN`. -- The already-fixed ICEs in [Glacier][glacier] --- though avoid the unfixed - ones in `ices/`! + like `//@ failure-status: 101` or `//@ known-bug: #NNN`. +- The already-fixed ICEs in the archived [Glacier][glacier] repository --- though + avoid the unfixed ones in `ices/`! + +[glacier]: https://github.com/rust-lang/glacier ## Extra credit Here are a few things you can do to help the Rust project after filing an ICE. -- [Bisect][bisect] the bug to figure out when it was introduced +- [Bisect][bisect] the bug to figure out when it was introduced. + If you find the regressing PR / commit, you can mark the issue with the label + `S-has-bisection`. If not, consider applying `E-needs-bisection` instead. - Fix "distractions": problems with the test case that don't contribute to triggering the ICE, such as syntax errors or borrow-checking errors -- Minimize the test case (see below) -- Add the minimal test case to [Glacier][glacier] +- Minimize the test case (see below). If successful, you can label the + issue with `S-has-mcve`. Otherwise, you can apply `E-needs-mcve`. +- Add the minimal test case to the rust-lang/rust repo as a [crashes test]. + While you're at it, consider including other "untracked" crashes in your PR. + Please don't forget to mark your issue with `S-bug-has-test` afterwards. + +See also [applying and removing labels][labeling]. [bisect]: https://rust-lang.github.io/cargo-bisect-rustc/ +[crashes test]: tests/compiletest.html#crashes-tests +[labeling]: https://forge.rust-lang.org/release/issue-triaging.html#applying-and-removing-labels ## Minimization @@ -123,7 +134,7 @@ what actually results in superior throughput. You may want to build rustc from source with debug assertions to find additional bugs, though this is a trade-off: it can slow down fuzzing by requiring extra work for every execution. To enable debug assertions, add this -to `config.toml` when compiling rustc: +to `bootstrap.toml` when compiling rustc: ```toml [rust] @@ -143,7 +154,6 @@ ICEs that require debug assertions to reproduce should be tagged - [tree-splicer][tree-splicer] generates new source files by combining existing ones while maintaining correct syntax -[glacier]: https://github.com/rust-lang/glacier [fuzz-rustc]: https://github.com/dwrensha/fuzz-rustc [icemaker]: https://github.com/matthiaskrgr/icemaker/ [tree-splicer]: https://github.com/langston-barrett/tree-splicer/ diff --git a/src/getting-started.md b/src/getting-started.md index 03d2811e8..87e26d379 100644 --- a/src/getting-started.md +++ b/src/getting-started.md @@ -3,8 +3,6 @@ Thank you for your interest in contributing to Rust! There are many ways to contribute, and we appreciate all of them. - - If this is your first time contributing, the [walkthrough] chapter can give you a good example of how a typical contribution would go. @@ -13,7 +11,6 @@ quick guide for the most useful things. For more information, [see this chapter on how to build and run the compiler](./building/how-to-build-and-run.md). [internals]: https://internals.rust-lang.org -[rust-discord]: http://discord.gg/rust-lang [rust-zulip]: https://rust-lang.zulipchat.com [coc]: https://www.rust-lang.org/policies/code-of-conduct [walkthrough]: ./walkthrough.md @@ -22,8 +19,7 @@ chapter on how to build and run the compiler](./building/how-to-build-and-run.md ## Asking Questions If you have questions, please make a post on the [Rust Zulip server][rust-zulip] or -[internals.rust-lang.org][internals]. If you are contributing to Rustup, be aware they are not on -Zulip - you can ask questions in `#wg-rustup` [on Discord][rust-discord]. +[internals.rust-lang.org][internals]. See the [list of teams and working groups][governance] and [the Community page][community] on the official website for more resources. @@ -32,19 +28,23 @@ official website for more resources. As a reminder, all contributors are expected to follow our [Code of Conduct][coc]. -The compiler team (or `t-compiler`) usually hangs out in Zulip [in this -"stream"][z]; it will be easiest to get questions answered there. +The compiler team (or `t-compiler`) usually hangs out in Zulip in +[the #t-compiler channel][z-t-compiler]; +questions about how the compiler works can go in [#t-compiler/help][z-help]. -[z]: https://rust-lang.zulipchat.com/#narrow/stream/131828-t-compiler +[z-t-compiler]: https://rust-lang.zulipchat.com/#narrow/channel/131828-t-compiler +[z-help]: https://rust-lang.zulipchat.com/#narrow/channel/182449-t-compiler.2Fhelp **Please ask questions!** A lot of people report feeling that they are "wasting -expert time", but nobody on `t-compiler` feels this way. Contributors are +expert's time", but nobody on `t-compiler` feels this way. Contributors are important to us. Also, if you feel comfortable, prefer public topics, as this means others can see the questions and answers, and perhaps even integrate them back into this guide :) +**Tip**: If you're not a native English speaker and feel unsure about writing, try using a translator to help. But avoid using LLM tools that generate long, complex words. In daily teamwork, **simple and clear words** are best for easy understanding. Even small typos or grammar mistakes can make you seem more human, and people connect better with humans. + ### Experts Not all `t-compiler` members are experts on all parts of `rustc`; it's a @@ -89,7 +89,7 @@ filtering the search to areas you're interested in. For example: Not all important or beginner work has issue labels. See below for how to find work that isn't labelled. -[help-wanted-search]: https://github.com/issues?q=is%3Aopen+is%3Aissue+org%3Arust-lang+no%3Aassignee+label%3AE-easy%2C%22good+first+issue%22%2Cgood-first-issue%2CE-medium%2CEasy%2CE-help-wanted%2CE-mentor+-label%3AS-blocked+ +[help-wanted-search]: https://github.com/issues?q=is%3Aopen+is%3Aissue+org%3Arust-lang+no%3Aassignee+label%3AE-easy%2C%22good+first+issue%22%2Cgood-first-issue%2CE-medium%2CEasy%2CE-help-wanted%2CE-mentor+-label%3AS-blocked+-linked%3Apr+ [Triage]: ./contributing.md#issue-triage ### Recurring work @@ -98,10 +98,7 @@ Some work is too large to be done by a single person. In this case, it's common issues" to co-ordinate the work between contributors. Here are some example tracking issues where it's easy to pick up work without a large time commitment: -- [Rustdoc Askama Migration](https://github.com/rust-lang/rust/issues/108868) -- [Diagnostic Translation](https://github.com/rust-lang/rust/issues/100717) - [Move UI tests to subdirectories](https://github.com/rust-lang/rust/issues/73494) -- [Port run-make tests from Make to Rust](https://github.com/rust-lang/rust/issues/121876) If you find more recurring work, please feel free to add it here! @@ -137,6 +134,10 @@ pull request, continuing the work on the feature. [abandoned-prs]: https://github.com/rust-lang/rust/pulls?q=is%3Apr+label%3AS-inactive+is%3Aclosed +### Writing tests + +Issues that have been resolved but do not have a regression test are marked with the `E-needs-test` label. Writing unit tests is a low-risk, lower-priority task that offers new contributors a great opportunity to familiarize themselves with the testing infrastructure and contribution workflow. + ### Contributing to std (standard library) See [std-dev-guide](https://std-dev-guide.rust-lang.org/). @@ -157,28 +158,21 @@ feel comfortable jumping straight into the large `rust-lang/rust` codebase. The following tasks are doable without much background knowledge but are incredibly helpful: -- [Cleanup crew][iceb]: find minimal reproductions of ICEs, bisect - regressions, etc. This is a way of helping that saves a ton of time for - others to fix an error later. - [Writing documentation][wd]: if you are feeling a bit more intrepid, you could try to read a part of the code and write doc comments for it. This will help you to learn some part of the compiler while also producing a useful artifact! - [Triaging issues][triage]: categorizing, replicating, and minimizing issues is very helpful to the Rust maintainers. - [Working groups][wg]: there are a bunch of working groups on a wide variety of rust-related things. -- Answer questions in the _Get Help!_ channels on the [Rust Discord - server][rust-discord], on [users.rust-lang.org][users], or on - [StackOverflow][so]. +- Answer questions on [users.rust-lang.org][users], or on [Stack Overflow][so]. - Participate in the [RFC process](https://github.com/rust-lang/rfcs). - Find a [requested community library][community-library], build it, and publish it to [Crates.io](http://crates.io). Easier said than done, but very, very valuable! -[rust-discord]: https://discord.gg/rust-lang [users]: https://users.rust-lang.org/ [so]: http://stackoverflow.com/questions/tagged/rust [community-library]: https://github.com/rust-lang/rfcs/labels/A-community-library -[iceb]: ./notification-groups/cleanup-crew.md [wd]: ./contributing.md#writing-documentation [wg]: https://rust-lang.github.io/compiler-team/working-groups/ [triage]: ./contributing.md#issue-triage diff --git a/src/git.md b/src/git.md index 177495b53..8f0511a45 100644 --- a/src/git.md +++ b/src/git.md @@ -1,7 +1,5 @@ # Using Git - - The Rust project uses [Git] to manage its source code. In order to contribute, you'll need some familiarity with its features so that your changes can be incorporated into the compiler. @@ -38,13 +36,13 @@ If you've cloned your fork, then you will be able to reference it with `origin` in your local repo. It may be helpful to also set up a remote for the official rust-lang/rust repo via -```sh +```console git remote add upstream https://github.com/rust-lang/rust.git ``` if you're using HTTPS, or -```sh +```console git remote add upstream git@github.com:rust-lang/rust.git ``` @@ -112,7 +110,7 @@ See [Rebasing](#rebasing) for more about rebasing. This is not a problem from git's perspective. If you run `git remote -v`, it will say something like this: -``` +```console $ git remote -v origin git@github.com:jyn514/rust.git (fetch) origin git@github.com:jyn514/rust.git (push) @@ -142,7 +140,8 @@ The most common cause is that you rebased after a change and ran `git add .` wit `x` to update the submodules. Alternatively, you might have run `cargo fmt` instead of `x fmt` and modified files in a submodule, then committed the changes. -To fix it, do the following things: +To fix it, do the following things (if you changed a submodule other than cargo, +replace `src/tools/cargo` with the path to that submodule): 1. See which commit has the accidental changes: `git log --stat -n1 src/tools/cargo` 2. Revert the changes to that commit: `git checkout ~ src/tools/cargo`. Type `~` @@ -158,11 +157,11 @@ To fix it, do the following things: ### I see "error: cannot rebase" when I try to rebase These are two common errors to see when rebasing: -``` +```console error: cannot rebase: Your index contains uncommitted changes. error: Please commit or stash them. ``` -``` +```console error: cannot rebase: You have unstaged changes. error: Please commit or stash them. ``` @@ -174,7 +173,7 @@ commit your changes, or make a temporary commit called a "stash" to have them st when you finish rebasing. You may want to configure git to make this "stash" automatically, which will prevent the "cannot rebase" error in nearly all cases: -``` +```console git config --global rebase.autostash true ``` @@ -205,7 +204,7 @@ git reset --hard master `git push` will not work properly and say something like this: -``` +```console ! [rejected] issue-xxxxx -> issue-xxxxx (non-fast-forward) error: failed to push some refs to 'https://github.com/username/rust.git' hint: Updates were rejected because the tip of your current branch is behind @@ -226,7 +225,7 @@ didn't write, it likely means you're trying to rebase over the wrong branch. For have a `rust-lang/rust` remote `upstream`, but ran `git rebase origin/master` instead of `git rebase upstream/master`. The fix is to abort the rebase and use the correct branch instead: -``` +```console git rebase --abort git rebase -i upstream/master ``` @@ -243,7 +242,7 @@ When updating your local repository with `git pull`, you may notice that sometim Git says you have modified some files that you have never edited. For example, running `git status` gives you something like (note the `new commits` mention): -``` +```console On branch master Your branch is up to date with 'origin/master'. @@ -256,9 +255,12 @@ Changes not staged for commit: no changes added to commit (use "git add" and/or "git commit -a") ``` -These changes are not changes to files: they are changes to submodules (more on this -[later](#git-submodules)). To get rid of those, run `./x --help`, which will automatically update -the submodules. +These changes are not changes to files: they are changes to submodules (more on this [later](#git-submodules)). +To get rid of those: + +```console +git submodule update +``` Some submodules are not actually needed; for example, `src/llvm-project` doesn't need to be checked out if you're using `download-ci-llvm`. To avoid having to keep fetching its history, you can use @@ -278,12 +280,12 @@ merged. To do that, you need to rebase your work on top of rust-lang/rust. To rebase your feature branch on top of the newest version of the master branch of rust-lang/rust, checkout your branch, and then run this command: -``` +```console git pull --rebase https://github.com/rust-lang/rust.git master ``` > If you are met with the following error: -> ``` +> ```console > error: cannot pull with rebase: Your index contains uncommitted changes. > error: please commit or stash them. > ``` @@ -300,13 +302,13 @@ reapply the changes fails because your changes conflicted with other changes that have been made. You can tell that this happened because you'll see lines in the output that look like -``` +```console CONFLICT (content): Merge conflict in file.rs ``` When you open these files, you'll see sections of the form -``` +```console <<<<<<< HEAD Original code ======= @@ -336,17 +338,17 @@ your fork with `git push --force-with-lease`. ### Keeping things up to date -The above section on [Rebasing](#rebasing) is a specific +The [above section](#rebasing) is a specific guide on rebasing work and dealing with merge conflicts. Here is some general advice about how to keep your local repo up-to-date with upstream changes: Using `git pull upstream master` while on your local master branch regularly -will keep it up-to-date. You will also want to rebase your feature branches +will keep it up-to-date. You will also want to keep your feature branches up-to-date as well. After pulling, you can checkout the feature branches and rebase them: -``` +```console git checkout master git pull upstream master --ff-only # to make certain there are no merge commits git rebase master feature_branch @@ -384,7 +386,7 @@ change the order in which they are applied, or "squash" them into each other. Alternatively, you can sacrifice the commit history like this: -``` +```console # squash all the changes into one commit so you only have to worry about conflicts once git rebase -i --keep-base master # and squash all changes along the way git rebase master @@ -422,7 +424,7 @@ it shows you the differences between your old diff and your new diff. Here's an example of `git range-diff` output (taken from [Git's docs][range-diff-example-docs]): -``` +```console -: ------- > 1: 0ddba11 Prepare for the inevitable! 1: c0debee = 2: cab005e Add a helpful message at the start 2: f00dbal ! 3: decafe1 Describe a bug @@ -499,7 +501,7 @@ Git and Github's default diff view for large moves *within* a file is quite poor line as deleted and each line as added, forcing you to compare each line yourself. Git has an option to show moved lines in a different color: -``` +```console git log -p --color-moved=dimmed-zebra --color-moved-ws=allow-indentation-change ``` @@ -515,7 +517,7 @@ that was force-pushed to make sure there are no unexpected changes. Many large files in the repo are autogenerated. To view a diff that ignores changes to those files, you can use the following syntax (e.g. Cargo.lock): -``` +```console git log -p ':!Cargo.lock' ``` @@ -545,7 +547,7 @@ The contents of submodules are ignored by Git: submodules are in some sense isol from the rest of the repository. However, if you try to `cd src/llvm-project` and then run `git status`: -``` +```console HEAD detached at 9567f08afc943 nothing to commit, working tree clean ``` @@ -576,7 +578,7 @@ that Git can nicely and fairly conveniently handle for us. Sometimes you might run into (when you run `git status`) -``` +```console Changes not staged for commit: (use "git add ..." to update what will be committed) (use "git restore ..." to discard changes in working directory) @@ -586,7 +588,7 @@ Changes not staged for commit: and when you try to run `git submodule update` it breaks horribly with errors like -``` +```console error: RPC failed; curl 92 HTTP/2 stream 7 was not closed cleanly: CANCEL (err 8) error: 2782 bytes of body are still expected fetch-pack: unexpected disconnect while reading sideband packet @@ -597,8 +599,8 @@ fatal: Fetched in submodule path 'src/llvm-project', but it did not contain 5a51 If you see `(new commits, modified content)` you can run -```bash -$ git submodule foreach git reset --hard +```console +git submodule foreach git reset --hard ``` and then try `git submodule update` again. @@ -607,7 +609,7 @@ and then try `git submodule update` again. If that doesn't work, you can try to deinit all git submodules... -``` +```console git submodule deinit -f --all ``` @@ -618,7 +620,7 @@ completely messed up for some reason. Sometimes, for some forsaken reason, you might run into -```text +```console fatal: not a git repository: src/gcc/../../.git/modules/src/gcc ``` diff --git a/src/guides/editions.md b/src/guides/editions.md index 336e391df..b65fbb13c 100644 --- a/src/guides/editions.md +++ b/src/guides/editions.md @@ -1,7 +1,5 @@ # Editions - - This chapter gives an overview of how Edition support works in rustc. This assumes that you are familiar with what Editions are (see the [Edition Guide]). @@ -91,7 +89,7 @@ stability. ## Edition parsing For the most part, the lexer is edition-agnostic. -Within [`StringReader`], tokens can be modified based on edition-specific behavior. +Within [`Lexer`], tokens can be modified based on edition-specific behavior. For example, C-String literals like `c"foo"` are split into multiple tokens in editions before 2021. This is also where things like reserved prefixes are handled for the 2021 edition. @@ -114,7 +112,7 @@ For example, the deprecated `start...end` pattern syntax emits the [`ellipsis_inclusive_range_patterns`] lint on editions before 2021, and in 2021 is an hard error via the `emit_err` method. -[`StringReader`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_parse/lexer/struct.StringReader.html +[`Lexer`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_parse/lexer/struct.Lexer.html [`ParseSess::edition`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_session/parse/struct.ParseSess.html#structfield.edition [`ellipsis_inclusive_range_patterns`]: https://doc.rust-lang.org/nightly/rustc/lints/listing/warn-by-default.html#ellipsis-inclusive-range-patterns @@ -193,6 +191,23 @@ When a user runs `cargo fix --edition`, cargo will pass the `--force-warn rust-2 flag to force all of these lints to appear during the edition migration. Cargo also passes `--cap-lints=allow` so that no other lints interfere with the edition migration. +Make sure that the example code sets the correct edition. The example should illustrate the previous edition, and show what the migration warning would look like. For example, this lint for a 2024 migration shows an example in 2021: + +```rust,ignore +declare_lint! { + /// The `keyword_idents_2024` lint detects ... + /// + /// ### Example + /// + /// ```rust,edition2021 + /// #![warn(keyword_idents_2024)] + /// fn gen() {} + /// ``` + /// + /// {{produces}} +} +``` + Migration lints can be either `Allow` or `Warn` by default. If it is `Allow`, users usually won't see this warning unless they are doing an edition migration manually or there is a problem during the migration. @@ -334,3 +349,40 @@ In general it is recommended to avoid these special cases except for very high v [into-iter]: https://doc.rust-lang.org/nightly/edition-guide/rust-2021/IntoIterator-for-arrays.html [panic-macro]: https://doc.rust-lang.org/nightly/edition-guide/rust-2021/panic-macro-consistency.html [`non_fmt_panics`]: https://doc.rust-lang.org/nightly/rustc/lints/listing/warn-by-default.html#non-fmt-panics + +### Migrating the standard library edition + +Updating the edition of the standard library itself roughly involves the following process: + +- Wait until the newly stabilized edition has reached beta and the bootstrap compiler has been updated. +- Apply migration lints. This can be an involved process since some code is in external submodules[^std-submodules], and the standard library makes heavy use of conditional compilation. Also, running `cargo fix --edition` can be impractical on the standard library itself. One approach is to individually add `#![warn(...)]` at the top of each crate for each lint, run `./x check library`, apply the migrations, remove the `#![warn(...)]` and commit each migration separately. You'll likely need to run `./x check` with `--target` for many different targets to get full coverage (otherwise you'll likely spend days or weeks getting CI to pass)[^ed-docker]. See also the [advanced migration guide] for more tips. + - Apply migrations to [`backtrace-rs`]. [Example for 2024](https://github.com/rust-lang/backtrace-rs/pull/700). Note that this doesn't update the edition of the crate itself because that is published independently on crates.io, and that would otherwise restrict the minimum Rust version. Consider adding some `#![deny()]` attributes to avoid regressions until its edition gets updated. + - Apply migrations to [`stdarch`], and update its edition, and formatting. [Example for 2024](https://github.com/rust-lang/stdarch/pull/1710). + - Post PRs to update the backtrace and stdarch submodules, and wait for those to land. + - Apply migration lints to the standard library crates, and update their edition. I recommend working one crate at a time starting with `core`. [Example for 2024](https://github.com/rust-lang/rust/pull/138162). + +[^std-submodules]: This will hopefully change in the future to pull these submodules into `rust-lang/rust`. +[^ed-docker]: You'll also likely need to do a lot of testing for different targets, and this is where [docker testing](../tests/docker.md) comes in handy. + +[advanced migration guide]: https://doc.rust-lang.org/nightly/edition-guide/editions/advanced-migrations.html +[`backtrace-rs`]: https://github.com/rust-lang/backtrace-rs/ +[`stdarch`]: https://github.com/rust-lang/stdarch/ + +## Stabilizing an edition + +After the edition team has given the go-ahead, the process for stabilizing an edition is roughly: + +- Update [`LATEST_STABLE_EDITION`]. +- Update [`Edition::is_stable`]. +- Hunt and find any document that refers to edition by number, and update it: + - [`--edition` flag](https://github.com/rust-lang/rust/blob/master/src/doc/rustc/src/command-line-arguments.md#--edition-specify-the-edition-to-use) + - [Rustdoc attributes](https://github.com/rust-lang/rust/blob/master/src/doc/rustdoc/src/write-documentation/documentation-tests.md#attributes) +- Clean up any tests that use the `//@ edition` header to remove the `-Zunstable-options` flag to ensure they are indeed stable. Note: Ideally this should be automated, see [#133582]. +- Bless any tests that change. +- Update `lint-docs` to default to the new edition. + +See [example for 2024](https://github.com/rust-lang/rust/pull/133349). + +[`LATEST_STABLE_EDITION`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_span/edition/constant.LATEST_STABLE_EDITION.html +[`Edition::is_stable`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_span/edition/enum.Edition.html#method.is_stable +[#133582]: https://github.com/rust-lang/rust/issues/133582 diff --git a/src/hir.md b/src/hir.md index 51893d537..38ba33112 100644 --- a/src/hir.md +++ b/src/hir.md @@ -1,11 +1,9 @@ # The HIR - - The HIR – "High-Level Intermediate Representation" – is the primary IR used in most of rustc. It is a compiler-friendly representation of the abstract syntax tree (AST) that is generated after parsing, macro expansion, and name -resolution (see [Lowering](./ast-lowering.html) for how the HIR is created). +resolution (see [Lowering](./hir/lowering.md) for how the HIR is created). Many parts of HIR resemble Rust surface syntax quite closely, with the exception that some of Rust's expression forms have been desugared away. For example, `for` loops are converted into a `loop` and do not appear in @@ -100,7 +98,7 @@ The HIR uses a bunch of different identifiers that coexist and serve different p a wrapper around a [`HirId`]. For more info about HIR bodies, please refer to the [HIR chapter][hir-bodies]. -These identifiers can be converted into one another through the [HIR map][map]. +These identifiers can be converted into one another through the `TyCtxt`. [`DefId`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir/def_id/struct.DefId.html [`LocalDefId`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir/def_id/struct.LocalDefId.html @@ -110,47 +108,42 @@ These identifiers can be converted into one another through the [HIR map][map]. [`CrateNum`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir/def_id/struct.CrateNum.html [`DefIndex`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir/def_id/struct.DefIndex.html [`Body`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir/hir/struct.Body.html -[hir-map]: ./hir.md#the-hir-map [hir-bodies]: ./hir.md#hir-bodies -[map]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/hir/map/struct.Map.html -## The HIR Map +## HIR Operations Most of the time when you are working with the HIR, you will do so via -the **HIR Map**, accessible in the tcx via [`tcx.hir()`] (and defined in -the [`hir::map`] module). The [HIR map] contains a [number of methods] to -convert between IDs of various kinds and to lookup data associated -with a HIR node. +`TyCtxt`. It contains a number of methods, defined in the `hir::map` module and +mostly prefixed with `hir_`, to convert between IDs of various kinds and to +lookup data associated with a HIR node. -[`tcx.hir()`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/struct.TyCtxt.html#method.hir -[`hir::map`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/hir/map/index.html -[HIR map]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/hir/map/struct.Map.html -[number of methods]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/hir/map/struct.Map.html#methods +[`TyCtxt`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/struct.TyCtxt.html For example, if you have a [`LocalDefId`], and you would like to convert it -to a [`HirId`], you can use [`tcx.hir().local_def_id_to_hir_id(def_id)`][local_def_id_to_hir_id]. +to a [`HirId`], you can use [`tcx.local_def_id_to_hir_id(def_id)`][local_def_id_to_hir_id]. You need a `LocalDefId`, rather than a `DefId`, since only local items have HIR nodes. -[local_def_id_to_hir_id]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/hir/map/struct.Map.html#method.local_def_id_to_hir_id +[local_def_id_to_hir_id]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/struct.TyCtxt.html#method.local_def_id_to_hir_id -Similarly, you can use [`tcx.hir().find(n)`][find] to lookup the node for a +Similarly, you can use [`tcx.hir_node(n)`][hir_node] to lookup the node for a [`HirId`]. This returns a `Option>`, where [`Node`] is an enum defined in the map. By matching on this, you can find out what sort of node the `HirId` referred to and also get a pointer to the data itself. Often, you know what sort of node `n` is – e.g. if you know that `n` must be some HIR expression, you can do -[`tcx.hir().expect_expr(n)`][expect_expr], which will extract and return the +[`tcx.hir_expect_expr(n)`][expect_expr], which will extract and return the [`&hir::Expr`][Expr], panicking if `n` is not in fact an expression. -[find]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/hir/map/struct.Map.html#method.find +[hir_node]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/struct.TyCtxt.html#method.hir_node [`Node`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir/hir/enum.Node.html -[expect_expr]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/hir/map/struct.Map.html#method.expect_expr +[expect_expr]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/struct.TyCtxt.html#method.expect_expr [Expr]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir/hir/struct.Expr.html -Finally, you can use the HIR map to find the parents of nodes, via -calls like [`tcx.hir().get_parent(n)`][get_parent]. +Finally, you can find the parents of nodes, via +calls like [`tcx.parent_hir_node(n)`][parent_hir_node]. + +[parent_hir_node]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/struct.TyCtxt.html#method.parent_hir_node -[get_parent]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/hir/map/struct.Map.html#method.get_parent ## HIR Bodies @@ -158,10 +151,10 @@ A [`rustc_hir::Body`] represents some kind of executable code, such as the body of a function/closure or the definition of a constant. Bodies are associated with an **owner**, which is typically some kind of item (e.g. an `fn()` or `const`), but could also be a closure expression -(e.g. `|x, y| x + y`). You can use the HIR map to find the body -associated with a given def-id ([`maybe_body_owned_by`]) or to find -the owner of a body ([`body_owner_def_id`]). +(e.g. `|x, y| x + y`). You can use the `TyCtxt` to find the body +associated with a given def-id ([`hir_maybe_body_owned_by`]) or to find +the owner of a body ([`hir_body_owner_def_id`]). [`rustc_hir::Body`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir/hir/struct.Body.html -[`maybe_body_owned_by`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/hir/map/struct.Map.html#method.maybe_body_owned_by -[`body_owner_def_id`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/hir/map/struct.Map.html#method.body_owner_def_id +[`hir_maybe_body_owned_by`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/struct.TyCtxt.html#method.hir_maybe_body_owned_by +[`hir_body_owner_def_id`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/struct.TyCtxt.html#method.hir_body_owner_def_id diff --git a/src/hir/ambig-unambig-ty-and-consts.md b/src/hir/ambig-unambig-ty-and-consts.md new file mode 100644 index 000000000..d4f504ad2 --- /dev/null +++ b/src/hir/ambig-unambig-ty-and-consts.md @@ -0,0 +1,63 @@ +# Ambig/Unambig Types and Consts + +Types and Consts args in the HIR can be in two kinds of positions ambiguous (ambig) or unambiguous (unambig). Ambig positions are where +it would be valid to parse either a type or a const, unambig positions are where only one kind would be valid to +parse. + +```rust +fn func(arg: T) { + // ^ Unambig type position + let a: _ = arg; + // ^ Unambig type position + + func::(arg); + // ^ ^ + // ^^^^ Ambig position + + let _: [u8; 10]; + // ^^ ^^ Unambig const position + // ^^ Unambig type position +} + +``` + +Most types/consts in ambig positions are able to be disambiguated as either a type or const during parsing. Single segment paths are always represented as types in the AST but may get resolved to a const parameter during name resolution, then lowered to a const argument during ast-lowering. The only generic arguments which remain ambiguous after lowering are inferred generic arguments (`_`) in path segments. For example, in `Foo<_>` it is not clear whether the `_` argument is an inferred type argument, or an inferred const argument. + +In unambig positions, inferred arguments are represented with [`hir::TyKind::Infer`][ty_infer] or [`hir::ConstArgKind::Infer`][const_infer] depending on whether it is a type or const position respectively. +In ambig positions, inferred arguments are represented with `hir::GenericArg::Infer`. + +A naive implementation of this would result in there being potentially 5 places where you might think an inferred type/const could be found in the HIR from looking at the structure of the HIR: +1. In unambig type position as a `hir::TyKind::Infer` +2. In unambig const arg position as a `hir::ConstArgKind::Infer` +3. In an ambig position as a [`GenericArg::Type(TyKind::Infer)`][generic_arg_ty] +4. In an ambig position as a [`GenericArg::Const(ConstArgKind::Infer)`][generic_arg_const] +5. In an ambig position as a [`GenericArg::Infer`][generic_arg_infer] + +Note that places 3 and 4 would never actually be possible to encounter as we always lower to `GenericArg::Infer` in generic arg position. + +This has a few failure modes: +- People may write visitors which check for `GenericArg::Infer` but forget to check for `hir::TyKind/ConstArgKind::Infer`, only handling infers in ambig positions by accident. +- People may write visitors which check for `hir::TyKind/ConstArgKind::Infer` but forget to check for `GenericArg::Infer`, only handling infers in unambig positions by accident. +- People may write visitors which check for `GenericArg::Type/Const(TyKind/ConstArgKind::Infer)` and `GenericArg::Infer`, not realising that we never represent inferred types/consts in ambig positions as a `GenericArg::Type/Const`. +- People may write visitors which check for *only* `TyKind::Infer` and not `ConstArgKind::Infer` forgetting that there are also inferred const arguments (and vice versa). + +To make writing HIR visitors less error prone when caring about inferred types/consts we have a relatively complex system: + +1. We have different types in the compiler for when a type or const is in an unambig or ambig position, `hir::Ty` and `hir::Ty<()>`. [`AmbigArg`][ambig_arg] is an uninhabited type which we use in the `Infer` variant of `TyKind` and `ConstArgKind` to selectively "disable" it if we are in an ambig position. + +2. The [`visit_ty`][visit_ty] and [`visit_const_arg`][visit_const_arg] methods on HIR visitors only accept the ambig position versions of types/consts. Unambig types/consts are implicitly converted to ambig types/consts during the visiting process, with the `Infer` variant handled by a dedicated [`visit_infer`][visit_infer] method. + +This has a number of benefits: +- It's clear that `GenericArg::Type/Const` cannot represent inferred type/const arguments +- Implementors of `visit_ty` and `visit_const_arg` will never encounter inferred types/consts making it impossible to write a visitor that seems to work right but handles edge cases wrong +- The `visit_infer` method handles *all* cases of inferred type/consts in the HIR making it easy for visitors to handle inferred type/consts in one dedicated place and not forget cases + +[ty_infer]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir/hir/enum.TyKind.html#variant.Infer +[const_infer]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir/hir/enum.ConstArgKind.html#variant.Infer +[generic_arg_ty]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir/hir/enum.GenericArg.html#variant.Type +[generic_arg_const]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir/hir/enum.GenericArg.html#variant.Const +[generic_arg_infer]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir/hir/enum.GenericArg.html#variant.Infer +[ambig_arg]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir/hir/enum.AmbigArg.html +[visit_ty]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir/intravisit/trait.Visitor.html#method.visit_ty +[visit_const_arg]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir/intravisit/trait.Visitor.html#method.visit_const_arg +[visit_infer]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir/intravisit/trait.Visitor.html#method.visit_infer diff --git a/src/hir-debugging.md b/src/hir/debugging.md similarity index 100% rename from src/hir-debugging.md rename to src/hir/debugging.md diff --git a/src/ast-lowering.md b/src/hir/lowering.md similarity index 92% rename from src/ast-lowering.md rename to src/hir/lowering.md index 033fd4b76..c0057a69c 100644 --- a/src/ast-lowering.md +++ b/src/hir/lowering.md @@ -1,16 +1,14 @@ # AST lowering -The AST lowering step converts AST to [HIR](hir.html). +The AST lowering step converts AST to [HIR](../hir.md). This means many structures are removed if they are irrelevant for type analysis or similar syntax agnostic analyses. Examples of such structures include but are not limited to * Parenthesis * Removed without replacement, the tree structure makes order explicit -* `for` loops and `while (let)` loops - * Converted to `loop` + `match` and some `let` bindings -* `if let` - * Converted to `match` +* `for` loops + * Converted to `match` + `loop` + `match` * Universal `impl Trait` * Converted to generic arguments (but with some flags, to know that the user didn't write them) diff --git a/src/implementing_new_features.md b/src/implementing_new_features.md index d87afeaed..00bce8599 100644 --- a/src/implementing_new_features.md +++ b/src/implementing_new_features.md @@ -1,142 +1,90 @@ # Implementing new language features - +When you want to implement a new significant feature in the compiler, you need to go through this process to make sure everything goes smoothly. -When you want to implement a new significant feature in the compiler, -you need to go through this process to make sure everything goes -smoothly. +**NOTE: This section is for *language* features, not *library* features, which use [a different process].** -**NOTE: this section is for *language* features, not *library* features, -which use [a different process].** +See also [the Rust Language Design Team's procedures][lang-propose] for proposing changes to the language. [a different process]: ./stability.md +[lang-propose]: https://lang-team.rust-lang.org/how_to/propose.html ## The @rfcbot FCP process -When the change is small and uncontroversial, then it can be done -with just writing a PR and getting an r+ from someone who knows that -part of the code. However, if the change is potentially controversial, -it would be a bad idea to push it without consensus from the rest -of the team (both in the "distributed system" sense to make sure -you don't break anything you don't know about, and in the social -sense to avoid PR fights). - -If such a change seems to be too small to require a full formal RFC process -(e.g., a small standard library addition, a big refactoring of the code, a -"technically-breaking" change, or a "big bugfix" that basically amounts to a -small feature) but is still too controversial or big to get by with a single r+, -you can propose a final comment period (FCP). Or, if you're not on the relevant -team (and thus don't have @rfcbot permissions), ask someone who is to start one; -unless they have a concern themselves, they should. - -Again, the FCP process is only needed if you need consensus – if you -don't think anyone would have a problem with your change, it's OK to -get by with only an r+. For example, it is OK to add or modify -unstable command-line flags or attributes without an FCP for -compiler development or standard library use, as long as you don't -expect them to be in wide use in the nightly ecosystem. -Some teams have lighter weight processes that they use in scenarios -like this; for example, the compiler team recommends -filing a Major Change Proposal ([MCP][mcp]) as a lightweight way to -garner support and feedback without requiring full consensus. - -[mcp]: https://forge.rust-lang.org/compiler/mcp.html#public-facing-changes-require-rfcbot-fcp - -You don't need to have the implementation fully ready for r+ to propose an FCP, -but it is generally a good idea to have at least a proof -of concept so that people can see what you are talking about. - -When an FCP is proposed, it requires all members of the team to sign off the -FCP. After they all do so, there's a 10-day-long "final comment period" (hence -the name) where everybody can comment, and if no concerns are raised, the -PR/issue gets FCP approval. +When the change is small, uncontroversial, non-breaking, and does not affect the stable language in any user-observable ways or add any new unstable features, then it can be done with just writing a PR and getting an r+ from someone who knows that part of the code. However, if not, more must be done. Even for compiler-internal work, it would be a bad idea to push a controversial change without consensus from the rest of the team (both in the "distributed system" sense to make sure you don't break anything you don't know about, and in the social sense to avoid PR fights). + +For changes that need the consensus of a team, we us the process of proposing a final comment period (FCP). If you're not on the relevant team (and thus don't have @rfcbot permissions), ask someone who is to start one; unless they have a concern themselves, they should. + +The FCP process is only needed if you need consensus – if no processes require consensus for your change and you don't think anyone would have a problem with it, it's OK to rely on only an r+. For example, it is OK to add or modify unstable command-line flags or attributes in the reserved compiler-internal `rustc_` namespace without an FCP for compiler development or standard library use, as long as you don't expect them to be in wide use in the nightly ecosystem. Some teams have lighter weight processes that they use in scenarios like this; for example, the compiler team recommends filing a Major Change Proposal ([MCP][mcp]) as a lightweight way to garner support and feedback without requiring full consensus. + +[mcp]: https://forge.rust-lang.org/compiler/proposals-and-stabilization.html#how-do-i-submit-an-mcp + +You don't need to have the implementation fully ready for r+ to propose an FCP, but it is generally a good idea to have at least a proof of concept so that people can see what you are talking about. + +When an FCP is proposed, it requires all members of the team to sign off on the FCP. After they all do so, there's a 10-day-long "final comment period" (hence the name) where everybody can comment, and if no concerns are raised, the PR/issue gets FCP approval. ## The logistics of writing features -There are a few "logistic" hoops you might need to go through in -order to implement a feature in a working way. +There are a few "logistical" hoops you might need to go through in order to implement a feature in a working way. ### Warning Cycles -In some cases, a feature or bugfix might break some existing programs -in some edge cases. In that case, you might want to do a crater run -to assess the impact and possibly add a future-compatibility lint, -similar to those used for -[edition-gated lints](diagnostics.md#edition-gated-lints). +In some cases, a feature or bugfix might break some existing programs in some edge cases. In that case, you'll want to do a crater run to assess the impact and possibly add a future-compatibility lint, similar to those used for [edition-gated lints](diagnostics.md#edition-gated-lints). ### Stability -We [value the stability of Rust]. Code that works and runs on stable -should (mostly) not break. Because of that, we don't want to release -a feature to the world with only team consensus and code review - -we want to gain real-world experience on using that feature on nightly, -and we might want to change the feature based on that experience. - -To allow for that, we must make sure users don't accidentally depend -on that new feature - otherwise, especially if experimentation takes -time or is delayed and the feature takes the trains to stable, -it would end up de facto stable and we'll not be able to make changes -in it without breaking people's code. - -The way we do that is that we make sure all new features are feature -gated - they can't be used without enabling a feature gate -(`#[feature(foo)]`), which can't be done in a stable/beta compiler. -See the [stability in code] section for the technical details. - -Eventually, after we gain enough experience using the feature, -make the necessary changes, and are satisfied, we expose it to -the world using the stabilization process described [here]. -Until then, the feature is not set in stone: every part of the -feature can be changed, or the feature might be completely -rewritten or removed. Features are not supposed to gain tenure -by being unstable and unchanged for a year. +We [value the stability of Rust]. Code that works and runs on stable should (mostly) not break. Because of that, we don't want to release a feature to the world with only team consensus and code review - we want to gain real-world experience on using that feature on nightly, and we might want to change the feature based on that experience. + +To allow for that, we must make sure users don't accidentally depend on that new feature - otherwise, especially if experimentation takes time or is delayed and the feature takes the trains to stable, it would end up de facto stable and we'll not be able to make changes in it without breaking people's code. + +The way we do that is that we make sure all new features are feature gated - they can't be used without enabling a feature gate (`#[feature(foo)]`), which can't be done in a stable/beta compiler. See the [stability in code] section for the technical details. + +Eventually, after we gain enough experience using the feature, make the necessary changes, and are satisfied, we expose it to the world using the stabilization process described [here]. Until then, the feature is not set in stone: every part of the feature can be changed, or the feature might be completely rewritten or removed. Features do not gain tenure by being unstable and unchanged for long periods of time. ### Tracking Issues -To keep track of the status of an unstable feature, the -experience we get while using it on nightly, and of the -concerns that block its stabilization, every feature-gate -needs a tracking issue. General discussions about the feature should be done on the tracking issue. +To keep track of the status of an unstable feature, the experience we get while using it on +nightly, and of the concerns that block its stabilization, every feature-gate needs a tracking +issue. When creating issues and PRs related to the feature, reference this tracking issue, and when there are updates about the feature's progress, post those to the tracking issue. -For features that have an RFC, you should use the RFC's -tracking issue for the feature. +For features that are part of an accept RFC or approved lang experiment, use the tracking issue for that. -For other features, you'll have to make a tracking issue -for that feature. The issue title should be "Tracking issue -for YOUR FEATURE". Use the ["Tracking Issue" issue template][template]. +For other features, create a tracking issue for that feature. The issue title should be "Tracking issue for YOUR FEATURE". Use the ["Tracking Issue" issue template][template]. [template]: https://github.com/rust-lang/rust/issues/new?template=tracking_issue.md +### Lang experiments + +To land in the compiler, features that have user-visible effects on the language (even unstable ones) must either be part of an accepted RFC or an approved [lang experiment]. + +To propose a new lang experiment, open an issue in `rust-lang/rust` that describes the motivation and the intended solution. If it's accepted, this issue will become the tracking issue for the experiment, so use the tracking issue [template] while also including these other details. Nominate the issue for the lang team and CC `@rust-lang/lang` and `@rust-lang/lang-advisors`. When the experiment is approved, the tracking issue will be marked as `B-experimental`. + +Feature flags related to a lang experiment must be marked as `incomplete` until an RFC is accepted for the feature. + +[lang experiment]: https://lang-team.rust-lang.org/how_to/experiment.html + ## Stability in code -The below steps needs to be followed in order to implement -a new unstable feature: +The below steps needs to be followed in order to implement a new unstable feature: -1. Open a [tracking issue] - - if you have an RFC, you can use the tracking issue for the RFC. +1. Open or identify the [tracking issue]. For features that are part of an accept RFC or approved lang experiment, use the tracking issue for that. - The tracking issue should be labeled with at least `C-tracking-issue`. - For a language feature, a label `F-feature_name` should be added as well. + Label the tracking issue with `C-tracking-issue` and the relevant `F-feature_name` label (adding that label if needed). -1. Pick a name for the feature gate (for RFCs, use the name - in the RFC). +1. Pick a name for the feature gate (for RFCs, use the name in the RFC). 1. Add the feature name to `rustc_span/src/symbol.rs` in the `Symbols {...}` block. Note that this block must be in alphabetical order. -1. Add a feature gate declaration to `rustc_feature/src/unstable.rs` in the unstable - `declare_features` block. +1. Add a feature gate declaration to `rustc_feature/src/unstable.rs` in the unstable `declare_features` block. ```rust ignore /// description of feature (unstable, $feature_name, "CURRENT_RUSTC_VERSION", Some($tracking_issue_number)) ``` - If you haven't yet - opened a tracking issue (e.g. because you want initial feedback on whether the feature is likely - to be accepted), you can temporarily use `None` - but make sure to update it before the PR is - merged! + If you haven't yet opened a tracking issue (e.g. because you want initial feedback on whether the feature is likely to be accepted), you can temporarily use `None` - but make sure to update it before the PR is merged! For example: @@ -145,55 +93,36 @@ a new unstable feature: (unstable, non_ascii_idents, "CURRENT_RUSTC_VERSION", Some(55467), None), ``` - Features can be marked as incomplete, and trigger the warn-by-default [`incomplete_features` - lint] - by setting their type to `incomplete`: + Features can be marked as incomplete, and trigger the warn-by-default [`incomplete_features` lint] by setting their type to `incomplete`: [`incomplete_features` lint]: https://doc.rust-lang.org/rustc/lints/listing/warn-by-default.html#incomplete-features ```rust ignore - /// Allows unsized rvalues at arguments and parameters. - (incomplete, unsized_locals, "CURRENT_RUSTC_VERSION", Some(48055), None), + /// Allows deref patterns. + (incomplete, deref_patterns, "CURRENT_RUSTC_VERSION", Some(87121), None), ``` - To avoid [semantic merge conflicts], please use `CURRENT_RUSTC_VERSION` instead of `1.70` or - another explicit version number. + Feature flags related to a lang experiment must be marked as `incomplete` until an RFC is accepted for the feature. + + To avoid [semantic merge conflicts], use `CURRENT_RUSTC_VERSION` instead of `1.70` or another explicit version number. [semantic merge conflicts]: https://bors.tech/essay/2017/02/02/pitch/ -1. Prevent usage of the new feature unless the feature gate is set. - You can check it in most places in the compiler using the - expression `tcx.features().$feature_name` (or - `sess.features_untracked().$feature_name` if the - tcx is unavailable) +1. Prevent usage of the new feature unless the feature gate is set. You can check it in most places in the compiler using the expression `tcx.features().$feature_name()`. + + If the feature gate is not set, you should either maintain the pre-feature behavior or raise an error, depending on what makes sense. Errors should generally use [`rustc_session::parse::feature_err`]. For an example of adding an error, see [#81015]. - If the feature gate is not set, you should either maintain - the pre-feature behavior or raise an error, depending on - what makes sense. Errors should generally use [`rustc_session::parse::feature_err`]. - For an example of adding an error, see [#81015]. + For features introducing new syntax, pre-expansion gating should be used instead. During parsing, when the new syntax is parsed, the symbol must be inserted to the current crate's [`GatedSpans`] via `self.sess.gated_span.gate(sym::my_feature, span)`. - For features introducing new syntax, pre-expansion gating should be used instead. - During parsing, when the new syntax is parsed, the symbol must be inserted to the - current crate's [`GatedSpans`] via `self.sess.gated_span.gate(sym::my_feature, span)`. - - After being inserted to the gated spans, the span must be checked in the - [`rustc_ast_passes::feature_gate::check_crate`] function, which actually denies - features. Exactly how it is gated depends on the exact type of feature, but most - likely will use the `gate_all!()` macro. + After being inserted to the gated spans, the span must be checked in the [`rustc_ast_passes::feature_gate::check_crate`] function, which actually denies features. Exactly how it is gated depends on the exact type of feature, but most likely will use the `gate_all!()` macro. -1. Add a test to ensure the feature cannot be used without - a feature gate, by creating `tests/ui/feature-gates/feature-gate-$feature_name.rs`. - You can generate the corresponding `.stderr` file by running `./x test -tests/ui/feature-gates/ --bless`. +1. Add a test to ensure the feature cannot be used without a feature gate, by creating `tests/ui/feature-gates/feature-gate-$feature_name.rs`. You can generate the corresponding `.stderr` file by running `./x test tests/ui/feature-gates/ --bless`. -1. Add a section to the unstable book, in - `src/doc/unstable-book/src/language-features/$feature_name.md`. +1. Add a section to the unstable book, in `src/doc/unstable-book/src/language-features/$feature_name.md`. -1. Write a lot of tests for the new feature, preferably in `tests/ui/$feature_name/`. - PRs without tests will not be accepted! +1. Write a lot of tests for the new feature, preferably in `tests/ui/$feature_name/`. PRs without tests will not be accepted! -1. Get your PR reviewed and land it. You have now successfully - implemented a feature in Rust! +1. Get your PR reviewed and land it. You have now successfully implemented a feature in Rust! [`GatedSpans`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_session/parse/struct.GatedSpans.html [#81015]: https://github.com/rust-lang/rust/pull/81015 @@ -204,3 +133,42 @@ tests/ui/feature-gates/ --bless`. [here]: ./stabilization_guide.md [tracking issue]: #tracking-issues [add-feature-gate]: ./feature-gates.md#adding-a-feature-gate + +## Call for testing + +Once the implementation is complete, the feature will be available to nightly users but not yet part of stable Rust. This is a good time to write a blog post on [the main Rust blog][rust-blog] and issue a "call for testing". + +Some earlier such blog posts include: + +1. [The push for GATs stabilization](https://blog.rust-lang.org/2021/08/03/GATs-stabilization-push/) +2. [Changes to `impl Trait` in Rust 2024](https://blog.rust-lang.org/2024/09/05/impl-trait-capture-rules.html) +3. [Async Closures MVP: Call for Testing!](https://blog.rust-lang.org/inside-rust/2024/08/09/async-closures-call-for-testing/) + +Alternatively, [*This Week in Rust*][twir] has a [section][twir-cft] for this. One example of this having been used is: + +- [Call for testing on boolean literals as cfg predicates](https://github.com/rust-lang/rust/issues/131204#issuecomment-2569314526) + +Which option to choose might depend on how significant the language change is, though note that the [*This Week in Rust*][twir] section might be less visible than a dedicated post on the main Rust blog. + +## Polishing + +Giving users a polished experience means more than just implementing the feature in rustc. We need to think about all of the tools and resources that we ship. This work includes: + +- Documenting the language feature in the [Rust Reference][reference]. +- Extending [`rustfmt`] to format any new syntax (if applicable). +- Extending [`rust-analyzer`] (if applicable). The extent of this work can depend on the nature of the language feature, as some features don't need to be blocked on *full* support. + - When a language feature degrades the user experience simply by existing before support is implemented in [`rust-analyzer`], that may lead the lang team to raise a blocking concern. + - Examples of such might include new syntax that [`rust-analyzer`] can't parse or type inference changes it doesn't understand when those lead to bogus diagnostics. + +## Stabilization + +The final step in the feature lifecycle is [stabilization][stab], which is when the feature becomes available to all Rust users. At this point, backward incompatible changes are generally no longer permitted (see the lang team's [defined semver policies](https://rust-lang.github.io/rfcs/1122-language-semver.html) for details). To learn more about stabilization, see the [stabilization guide][stab]. + + +[stab]: ./stabilization_guide.md +[rust-blog]: https://github.com/rust-lang/blog.rust-lang.org/ +[twir]: https://github.com/rust-lang/this-week-in-rust +[twir-cft]: https://this-week-in-rust.org/blog/2025/01/22/this-week-in-rust-583/#calls-for-testing +[`rustfmt`]: https://github.com/rust-lang/rustfmt +[`rust-analyzer`]: https://github.com/rust-lang/rust-analyzer +[reference]: https://github.com/rust-lang/reference diff --git a/src/incrcomp-debugging.md b/src/incrcomp-debugging.md index 7045d3fa3..a548215cf 100644 --- a/src/incrcomp-debugging.md +++ b/src/incrcomp-debugging.md @@ -1,4 +1,4 @@ -# Debugging and Testing Dependencies +# Debugging and testing dependencies ## Testing the dependency graph diff --git a/src/llvm-coverage-instrumentation.md b/src/llvm-coverage-instrumentation.md index 3078ae094..288b90f33 100644 --- a/src/llvm-coverage-instrumentation.md +++ b/src/llvm-coverage-instrumentation.md @@ -1,6 +1,4 @@ -# LLVM Source-Based Code Coverage - - +# LLVM source-based code coverage `rustc` supports detailed source-based code and test coverage analysis with a command line option (`-C instrument-coverage`) that instruments Rust @@ -34,7 +32,7 @@ Detailed instructions and examples are documented in the [coverage map]: https://llvm.org/docs/CoverageMappingFormat.html [rustc-book-instrument-coverage]: https://doc.rust-lang.org/nightly/rustc/instrument-coverage.html -## Recommended `config.toml` settings +## Recommended `bootstrap.toml` settings When working on the coverage instrumentation code, it is usually necessary to **enable the profiler runtime** by setting `profiler = true` in `[build]`. @@ -83,7 +81,7 @@ statically links coverage-instrumented binaries with LLVM runtime code In the `rustc` source tree, `library/profiler_builtins` bundles the LLVM `compiler-rt` code into a Rust library crate. Note that when building `rustc`, -`profiler_builtins` is only included when `build.profiler = true` is set in `config.toml`. +`profiler_builtins` is only included when `build.profiler = true` is set in `bootstrap.toml`. When compiling with `-C instrument-coverage`, [`CrateLoader::postprocess()`][crate-loader-postprocess] dynamically loads @@ -115,9 +113,9 @@ human-readable coverage report. > Tests in `coverage-run` mode have an implicit `//@ needs-profiler-runtime` > directive, so they will be skipped if the profiler runtime has not been -> [enabled in `config.toml`](#recommended-configtoml-settings). +> [enabled in `bootstrap.toml`](#recommended-configtoml-settings). -Finally, the [`tests/codegen/instrument-coverage/testprog.rs`] test compiles a simple Rust program +Finally, the [`tests/codegen-llvm/instrument-coverage/testprog.rs`] test compiles a simple Rust program with `-C instrument-coverage` and compares the compiled program's LLVM IR to expected LLVM IR instructions and structured data for a coverage-enabled program, including various checks for Coverage Map-related metadata and the LLVM @@ -136,4 +134,4 @@ and `mir-opt` tests can be refreshed by running: [`tests/coverage`]: https://github.com/rust-lang/rust/tree/master/tests/coverage [`src/tools/coverage-dump`]: https://github.com/rust-lang/rust/tree/master/src/tools/coverage-dump [`tests/coverage-run-rustdoc`]: https://github.com/rust-lang/rust/tree/master/tests/coverage-run-rustdoc -[`tests/codegen/instrument-coverage/testprog.rs`]: https://github.com/rust-lang/rust/blob/master/tests/mir-opt/coverage/instrument_coverage.rs +[`tests/codegen-llvm/instrument-coverage/testprog.rs`]: https://github.com/rust-lang/rust/blob/master/tests/mir-opt/coverage/instrument_coverage.rs diff --git a/src/macro-expansion.md b/src/macro-expansion.md index ebab56ad2..54d6d2b4e 100644 --- a/src/macro-expansion.md +++ b/src/macro-expansion.md @@ -1,10 +1,5 @@ # Macro expansion - - -> N.B. [`rustc_ast`], [`rustc_expand`], and [`rustc_builtin_macros`] are all -> undergoing refactoring, so some of the links in this chapter may be broken. - Rust has a very powerful macro system. In the previous chapter, we saw how the parser sets aside macros to be expanded (using temporary [placeholders]). This chapter is about the process of expanding those macros iteratively until @@ -12,9 +7,6 @@ we have a complete [*Abstract Syntax Tree* (AST)][ast] for our crate with no unexpanded macros (or a compile error). [ast]: ./ast-validation.md -[`rustc_ast`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_ast/index.html -[`rustc_expand`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_expand/index.html -[`rustc_builtin_macros`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_builtin_macros/index.html [placeholders]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_expand/placeholders/index.html First, we discuss the algorithm that expands and integrates macro output into diff --git a/src/memory.md b/src/memory.md index 1e030ff45..f766a5189 100644 --- a/src/memory.md +++ b/src/memory.md @@ -1,4 +1,4 @@ -# Memory Management in Rustc +# Memory management in rustc Generally rustc tries to be pretty careful how it manages memory. The compiler allocates _a lot_ of data structures throughout compilation, @@ -63,7 +63,7 @@ represented as a slice `&'tcx [tcx.types.i32, tcx.types.u32]`). [`mk_args`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/context/struct.TyCtxt.html#method.mk_args [adtdefid]: ./ty_module/generic_arguments.md#adtdef-and-defid [`Predicate`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/struct.Predicate.html -[`TraitRef`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/struct.TraitRef.html +[`TraitRef`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/type.TraitRef.html [`ty::TyKind`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/sty/type.TyKind.html [traits]: ./traits/resolution.md diff --git a/src/mir/construction.md b/src/mir/construction.md index f2559a22b..8360d9ff1 100644 --- a/src/mir/construction.md +++ b/src/mir/construction.md @@ -1,7 +1,5 @@ # MIR construction - - The lowering of [HIR] to [MIR] occurs for the following (probably incomplete) list of items: diff --git a/src/mir/dataflow.md b/src/mir/dataflow.md index f31da5ca2..970e61196 100644 --- a/src/mir/dataflow.md +++ b/src/mir/dataflow.md @@ -1,7 +1,5 @@ # Dataflow Analysis - - If you work on the MIR, you will frequently come across various flavors of [dataflow analysis][wiki]. `rustc` uses dataflow to find uninitialized variables, determine what variables are live across a generator `yield` @@ -148,8 +146,7 @@ whereas this code uses [`ResultsCursor`]: ```rust,ignore let mut results = MyAnalysis::new() - .into_engine(tcx, body, def_id) - .iterate_to_fixpoint() + .iterate_to_fixpoint(tcx, body, None); .into_results_cursor(body); // Inspect the fixpoint state immediately before each `Drop` terminator. diff --git a/src/mir/drop-elaboration.md b/src/mir/drop-elaboration.md index 3b321fd44..4da612c83 100644 --- a/src/mir/drop-elaboration.md +++ b/src/mir/drop-elaboration.md @@ -1,7 +1,5 @@ # Drop elaboration - - ## Dynamic drops According to the [reference][reference-drop]: diff --git a/src/mir/index.md b/src/mir/index.md index 778c58391..8ba5f3ac8 100644 --- a/src/mir/index.md +++ b/src/mir/index.md @@ -1,7 +1,5 @@ # The MIR (Mid-level IR) - - MIR is Rust's _Mid-level Intermediate Representation_. It is constructed from [HIR](../hir.html). MIR was introduced in [RFC 1211]. It is a radically simplified form of Rust that is used for @@ -304,9 +302,9 @@ The most important rule for this representation is that every value must be uniquely represented. In other words: a specific value must only be representable in one specific way. For example: there is only one way to represent an array of two integers as a `ValTree`: -`ValTree::Branch(&[ValTree::Leaf(first_int), ValTree::Leaf(second_int)])`. +`Branch([Leaf(first_int), Leaf(second_int)])`. Even though theoretically a `[u32; 2]` could be encoded in a `u64` and thus just be a -`ValTree::Leaf(bits_of_two_u32)`, that is not a legal construction of `ValTree` +`Leaf(bits_of_two_u32)`, that is not a legal construction of `ValTree` (and is very complex to do, so it is unlikely anyone is tempted to do so). These rules also mean that some values are not representable. There can be no `union`s in type diff --git a/src/name-resolution.md b/src/name-resolution.md index 2727b8142..2e96382f7 100644 --- a/src/name-resolution.md +++ b/src/name-resolution.md @@ -1,7 +1,5 @@ # Name resolution - - In the previous chapters, we saw how the [*Abstract Syntax Tree* (`AST`)][ast] is built with all macros expanded. We saw how doing that requires doing some name resolution to resolve imports and macro names. In this chapter, we show @@ -120,9 +118,9 @@ even though they should be visible by ordinary scoping rules. An example: fn do_something(val: T) { // <- New rib in both types and values (1) // `val` is accessible, as is the helper function // `T` is accessible - let helper = || { // New rib on `helper` (2) and another on the block (3) + let helper = || { // New rib on the block (2) // `val` is accessible here - }; // End of (3) + }; // End of (2), new rib on `helper` (3) // `val` is accessible, `helper` variable shadows `helper` function fn helper() { // <- New rib in both types and values (4) // `val` is not accessible here, (4) is not transparent for locals @@ -130,7 +128,7 @@ fn do_something(val: T) { // <- New rib in both types and values (1) } // End of (4) let val = T::default(); // New rib (5) // `val` is the variable, not the parameter here -} // End of (5), (2) and (1) +} // End of (5), (3) and (1) ``` Because the rules for different namespaces are a bit different, each namespace diff --git a/src/normalization.md b/src/normalization.md new file mode 100644 index 000000000..53e20f1c0 --- /dev/null +++ b/src/normalization.md @@ -0,0 +1,310 @@ +# Aliases and Normalization + +## Aliases + +In Rust there are a number of types that are considered equal to some "underlying" type, for example inherent associated types, trait associated types, free type aliases (`type Foo = u32`), and opaque types (`-> impl RPIT`). We consider such types to be "aliases", alias types are represented by the [`TyKind::Alias`][tykind_alias] variant, with the kind of alias tracked by the [`AliasTyKind`][aliaskind] enum. + +Normalization is the process of taking these alias types and replacing them with the underlying type that they are equal to. For example given some type alias `type Foo = u32`, normalizing `Foo` would give `u32`. + +The concept of an alias is not unique to *types* and the concept also applies to constants/const generics. However, right now in the compiler we don't really treat const aliases as a "first class concept" so this chapter mostly discusses things in the context of types (even though the concepts transfer just fine). + +[tykind_alias]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_type_ir/enum.TyKind.html#variant.Alias +[aliaskind]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_type_ir/enum.AliasTyKind.html + +### Rigid, Ambiguous and Unnormalized Aliases + +Aliases can either be "rigid", "ambiguous", or simply unnormalized. + +We consider types to be rigid if their "shape" isn't going to change, for example `Box` is rigid as no amount of normalization can turn a `Box` into a `u32`, whereas ` as Iterator>::Item` is not rigid as it can be normalized to `u32`. + +Aliases are rigid when we will never be able to normalize them further. A concrete example of a *rigid* alias would be `::Item` in an environment where there is no `T: Iterator` bound, only a `T: Iterator` bound: +```rust +fn foo() { + // This alias is *rigid* + let _: ::Item; +} + +fn bar>() { + // This alias is *not* rigid as it can be normalized to `u32` + let _: ::Item; +} +``` + +When an alias can't yet be normalized but may wind up normalizable in the [current environment](./typing_parameter_envs.md), we consider it to be an "ambiguous" alias. This can occur when an alias contains inference variables which prevent being able to determine how the trait is implemented: +```rust +fn foo() { + // This alias is considered to be "ambiguous" + let _: <_ as Iterator>::Item; +} +``` + +The reason we call them "ambiguous" aliases is because its *ambiguous* whether this is a rigid alias or not. + +The source of the `_: Iterator` trait impl is *ambiguous* (i.e. unknown), it could be some `impl Iterator for u32` or it could be some `T: Iterator` trait bound, we don't know yet. Depending on why `_: Iterator` holds the alias could be an unnormalized alias or it could be a rigid alias; it's *ambiguous* what kind of alias this is. + +Finally, an alias can just be unnormalized, ` as IntoIterator>::Iter` is an unnormalized alias as it can already be normalized to `std::vec::IntoIter`, it just hasn't been done yet. + +--- + +It is worth noting that Free and Inherent aliases cannot be rigid or ambiguous as naming them also implies having resolved the definition of the alias, which specifies the underlying type of the alias. + +### Diverging Aliases + +An alias is considered to "diverge" if its definition does not specify an underlying non-alias type to normalize to. A concrete example of diverging aliases: +```rust +type Diverges = Diverges; + +trait Trait { + type DivergingAssoc; +} +impl Trait for () { + type DivergingAssoc = <() as Trait>::DivergingAssoc; +} +``` +In this example both `Diverges` and `DivergingAssoc` are "trivial" cases of diverging type aliases where they have been defined as being equal to themselves. There is no underlying type that `Diverges` can ever be normalized to. + +We generally try to error when diverging aliases are defined, but this is entirely a "best effort" check. In the previous example the definitions are "simple enough" to be detected and so errors are emitted. However, in more complex cases, or cases where only some instantiations of generic parameters would result in a diverging alias, we don't emit an error: +```rust +trait Trait { + type DivergingAssoc; +} +impl Trait for T { + // This alias always diverges but we don't emit an error because + // the compiler can't "see" that. + type DivergingAssoc = ::DivergingAssoc; +} +``` + +Ultimately this means that we have no guarantee that aliases in the type system are non-diverging. As aliases may only diverge for some specific generic arguments, it also means that we only know whether an alias diverges once it is fully concrete. This means that codegen/const-evaluation also has to handle diverging aliases: +```rust +trait Trait { + type Diverges; +} +impl Trait for T { + type Diverges = ::Diverges; +} + +fn foo() { + let a: T::Diverges; +} + +fn main() { + foo::<()>(); +} +``` +In this example we only encounter an error from the diverging alias during codegen of `foo::<()>`, if the call to `foo` is removed then no compilation error will be emitted. + +### Opaque Types + +Opaque types are a relatively special kind of alias, and are covered in their own chapter: [Opaque types](./opaque-types-type-alias-impl-trait.md). + +### Const Aliases + +Unlike type aliases, const aliases are not represented directly in the type system, instead const aliases are always an anonymous body containing a path expression to a const item. This means that the only "const alias" in the type system is an anonymous unevaluated const body. + +As such there is no `ConstKind::Alias(AliasCtKind::Projection/Inherent/Free, _)`, instead we only have `ConstKind::Unevaluated` which is used for representing anonymous constants. + +```rust +fn foo() {} + +const FREE_CONST: usize = 1 + 1; + +fn bar() { + foo::<{ FREE_CONST }>(); + // The const arg is represented with some anonymous constant: + // ```pseudo-rust + // const ANON: usize = FREE_CONST; + // foo::(); + // ``` +} +``` + +This is likely to change as const generics functionality is improved, for example `feature(associated_const_equality)` and `feature(min_generic_const_args)` both require handling const aliases similarly to types (without an anonymous constant wrapping all const args). + +## What is Normalization + +### Structural vs Deep normalization + +There are two forms of normalization, structural (sometimes called *shallow*) and deep. Structural normalization should be thought of as only normalizing the "outermost" part of a type. On the other hand deep normalization will normalize *all* aliases in a type. + +In practice structural normalization can result in more than just the outer layer of the type being normalized, but this behaviour should not be relied upon. Unnormalizable non-rigid aliases making use of bound variables (`for<'a>`) cannot be normalized by either kind of normalization. + +As an example: conceptually, structurally normalizing the type `Vec<::Assoc>` would be a no-op, whereas deeply normalizing would give `Vec`. In practice even structural normalization would give `Vec`, though, again, this should not be relied upon. + +Changing the alias to use bound variables will result in different behaviour; `Vec fn(<&'a u8 as Identity>::Assoc)>` would result in no change when structurally normalized, but would result in `Vec fn(&'a u8)>` when deeply normalized. + +### Core normalization logic + +Structurally normalizing aliases is a little bit more nuanced than replacing the alias with whatever it is defined as being equal to in its definition; the result of normalizing an alias should either be a rigid type or an inference variable (which will later be inferred to a rigid type). To accomplish this we do two things: + +First, when normalizing an ambiguous alias it is normalized to an inference variable instead of leaving it as-is, this has two main effects: +- Even though an inference variable is not a rigid type, it will always wind up inferred *to* a rigid type so we ensure that the result of normalization will not need to be normalized again +- Inference variables are used in all cases where a type is non-rigid, allowing the rest of the compiler to not have to deal with *both* ambiguous aliases *and* inference variables + +Secondly, instead of having normalization directly return the type specified in the definition of the alias, we normalize the type first before returning it[^1]. We do this so that normalization is idempotent/callers do not need to run it in a loop. + +```rust +#![feature(lazy_type_alias)] + +type Foo = Bar; +type Bar = ::Item; + +fn foo() { + let a_: Foo<_>; +} +``` + +In this example: +- Normalizing `Foo` would result in `Bar`, except we want to normalize aliases in the type `Foo` is defined as equal to +- Normalizing `Bar` would result in `::Item`, except, again, we want to normalize aliases in the type `Bar` is defined as equal to +- Normalizing `::Item` results in some new inference variable `?y`, as `::Item` is an ambiguous alias +- The final result is that normalizing `Foo` results in `?y` + +## How to normalize + +When interfacing with the type system it will often be the case that it's necessary to request a type be normalized. There are a number of different entry points to the underlying normalization logic and each entry point should only be used in specific parts of the compiler. + + +An additional complication is that the compiler is currently undergoing a transition from the old trait solver to the new trait solver. +As part of this transition our approach to normalization in the compiler has changed somewhat significantly, resulting in some normalization entry points being "old solver only" slated for removal in the long-term once the new solver has stabilized. +The transition can be tracked via the [WG-trait-system-refactor](https://github.com/rust-lang/rust/labels/WG-trait-system-refactor) label in Github. + +Here is a rough overview of the different entry points to normalization in the compiler: +- `infcx.at.structurally_normalize` +- `infcx.at.(deeply_)?normalize` +- `infcx.query_normalize` +- `tcx.normalize_erasing_regions` +- `traits::normalize_with_depth(_to)` +- `EvalCtxt::structurally_normalize` + +### Outside of the trait solver + +The [`InferCtxt`][infcx] type exposes the "main" ways to normalize during analysis: [`normalize`][normalize], [`deeply_normalize`][deeply_normalize] and [`structurally_normalize`][structurally_normalize]. These functions are often wrapped and re-exposed on various `InferCtxt` wrapper types, such as [`FnCtxt`][fcx] or [`ObligationCtxt`][ocx] with minor API tweaks to handle some arguments or parts of the return type automatically. + +#### Structural `InferCtxt` normalization + +[`infcx.at.structurally_normalize`][structurally_normalize] exposes structural normalization that is able to handle inference variables and regions. It should generally be used whenever inspecting the kind of a type. + +Inside of HIR Typeck there is a related method of normalization- [`fcx.structurally_resolve`][structurally_resolve], which will error if the type being resolved is an unresolved inference variable. When the new solver is enabled it will also attempt to structurally normalize the type. + +Due to this there is a pattern in HIR typeck where a type is first normalized via `normalize` (only normalizing in the old solver), and then `structurally_resolve`'d (only normalizing in the new solver). This pattern should be preferred over calling `structurally_normalize` during HIR typeck as `structurally_resolve` will attempt to make inference progress by evaluating goals whereas `structurally_normalize` does not. + +#### Deep `InferCtxt` normalization + +##### `infcx.at.(deeply_)?normalize` + +There are two ways to deeply normalize with an `InferCtxt`, `normalize` and `deeply_normalize`. The reason for this is that `normalize` is a "legacy" normalization entry point used only by the old solver, whereas `deeply_normalize` is intended to be the long term way to deeply normalize. Both of these methods can handle regions. + +When the new solver is stabilized the `infcx.at.normalize` function will be removed and everything will have been migrated to the new deep or structural normalization methods. For this reason the `normalize` function is a no-op under the new solver, making it suitable only when the old solver needs normalization but the new solver does not. + +Using `deeply_normalize` will result in errors being emitted when encountering ambiguous aliases[^2] as it is not possible to support normalizing *all* ambiguous aliases to inference variables[^3]. `deeply_normalize` should generally only be used in cases where we do not expect to encounter ambiguous aliases, for example when working with types from item signatures. + +##### `infcx.query_normalize` + +[`infcx.query_normalize`][query_norm] is very rarely used, it has almost all the same restrictions as `normalize_erasing_regions` (cannot handle inference variables, no diagnostics support) with the main difference being that it retains lifetime information. For this reason `normalize_erasing_regions` is the better choice in almost all circumstances as it is more efficient due to caching lifetime-erased queries. + +In practice `query_normalize` is used for normalization in the borrow checker, and elsewhere as a performance optimization over `infcx.normalize`. Once the new solver is stabilized it is expected that `query_normalize` can be removed from the compiler as the new solvers normalization implementation should be performant enough for it to not be a performance regression. + +##### `tcx.normalize_erasing_regions` + +[`normalize_erasing_regions`][norm_erasing_regions] is generally used by parts of the compiler that are not doing type system analysis. This normalization entry point does not handle inference variables, lifetimes, or any diagnostics. Lints and codegen make heavy use of this entry point as they typically are working with fully inferred aliases that can be assumed to be well formed (or at least, are not responsible for erroring on). + +[query_norm]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_trait_selection/infer/at/struct.At.html#method.query_normalize +[norm_erasing_regions]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/struct.TyCtxt.html#method.normalize_erasing_regions +[normalize]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_trait_selection/infer/at/struct.At.html#method.normalize +[deeply_normalize]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_trait_selection/traits/normalize/trait.NormalizeExt.html#tymethod.deeply_normalize +[structurally_normalize]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_trait_selection/traits/trait.StructurallyNormalizeExt.html#tymethod.structurally_normalize_ty +[infcx]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_trait_selection/infer/struct.InferCtxt.html +[fcx]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir_typeck/fn_ctxt/struct.FnCtxt.html +[ocx]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_trait_selection/traits/struct.ObligationCtxt.html +[structurally_resolve]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir_typeck/fn_ctxt/struct.FnCtxt.html#method.structurally_resolve_type + +### Inside of the trait solver + +[`traits::normalize_with_depth(_to)`][norm_with_depth] and [`EvalCtxt::structurally_normalize`][eval_ctxt_structural_norm] are only used by the internals of the trait solvers (old and new respectively). It is effectively a raw entry point to the internals of how normalization is implemented by each trait solver. Other normalization entry points cannot be used from within the internals of trait solving as it wouldn't handle goal cycles and recursion depth correctly. + +[norm_with_depth]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_trait_selection/traits/normalize/fn.normalize_with_depth.html +[eval_ctxt_structural_norm]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_next_trait_solver/solve/struct.EvalCtxt.html#method.structurally_normalize_term + +## When/Where to normalize (Old vs New solver) + +One of the big changes between the old and new solver is our approach to when we expect aliases to be normalized. + +### Old solver + +All types are expected to be normalized as soon as possible, so that all types encountered in the type system are either rigid or an inference variable (which will later be inferred to a rigid term). + +As a concrete example: equality of aliases is implemented by assuming they're rigid and recursively equating the generic arguments of the alias. + +### New solver + +It's expected that all types potentially contain ambiguous or unnormalized aliases. Whenever an operation is performed that requires aliases to be normalized, it's the responsibility of that logic to normalize the alias (this means that matching on `ty.kind()` pretty much always has to structurally normalize first). + +As a concrete example: equality of aliases is implemented by a custom goal kind ([`PredicateKind::AliasRelate`][aliasrelate]) so that it can handle normalization of the aliases itself instead of assuming all alias types being equated are rigid. + +Despite this approach we still deeply normalize during [writeback][writeback] for performance/simplicity, so that types in the MIR can still be assumed to have been deeply normalized. + +[aliasrelate]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/type.PredicateKind.html#variant.AliasRelate +[writeback]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir_typeck/writeback/index.html + +--- + +There were a few main issues with the old solver's approach to normalization that motivated changing things in the new solver: + +### Missing normalization calls + +It was a frequent occurrence that normalization calls would be missing, resulting in passing unnormalized types to APIs expecting everything to already be normalized. Treating ambiguous or unnormalized aliases as rigid would result in all sorts of weird errors from aliases not being considered equal to one another, or surprising inference guidance from equating unnormalized aliases' generic arguments. + +### Normalizing parameter environments + +Another problem was that it was not possible to normalize `ParamEnv`s correctly in the old solver as normalization itself would expect a normalized `ParamEnv` in order to give correct results. See the chapter on `ParamEnv`s for more information: [`Typing/ParamEnv`s: Normalizing all bounds](./typing_parameter_envs.md#normalizing-all-bounds) + +### Unnormalizable non-rigid aliases in higher ranked types + +Given a type such as `for<'a> fn(::Assoc>)`, it is not possible to correctly handle this with the old solver's approach to normalization. + +If we were to normalize it to `for<'a> fn(?y)` and register a goal to normalize `for<'a> >::Assoc -> ?y`, this would result in errors in cases where `>::Assoc` normalized to `&'a u32`. The inference variable `?y` would be in a lower [universe] than the placeholders made when instantiating the `for<'a>` binder. + +Leaving the alias unnormalized would also be wrong as the old solver expects all aliases to be rigid. This was a soundness bug before the new solver was stabilized in coherence: [relating projection substs is unsound during coherence](https://github.com/rust-lang/rust/issues/102048). + +Ultimately this means that it is not always possible to ensure all aliases inside of a value are rigid. + +[universe]: borrow_check/region_inference/placeholders_and_universes.md#what-is-a-universe +[deeply_normalize]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_trait_selection/traits/normalize/trait.NormalizeExt.html#tymethod.deeply_normalize + +## Handling uses of diverging aliases + +Diverging aliases, like ambiguous aliases, are normalized to inference variables. As normalizing diverging aliases results in trait solver cycles, it always results in an error in the old solver. In the new solver it only results in an error if we wind up requiring all goals to hold in the current context. E.g. normalizing diverging aliases during HIR typeck will result in an error in both solvers. + +Alias well formedness doesn't require that the alias doesn't diverge[^4], this means that checking an alias is well formed isn't sufficient to cause an error to be emitted for diverging aliases; actually attempting to normalize the alias is required. + +Erroring on diverging aliases being a side effect of normalization means that it is very *arbitrary* whether we actually emit an error, it also differs between the old and new solver as we now normalize in less places. + +An example of the ad-hoc nature of erroring on diverging aliases causing "problems": +```rust +trait Trait { + type Diverges; +} + +impl Trait for T { + type Diverges = D::Diverges; +} + +struct Bar::Diverges>(Box); +``` + +In this example a diverging alias is used but we happen to not emit an error as we never explicitly normalize the defaults of generic parameters. If the `?Sized` opt out is removed then an error is emitted because we wind up happening to normalize a `::Diverges: Sized` goal which as a side effect results in erroring about the diverging alias. + +Const aliases differ from type aliases a bit here; well formedness of const aliases requires that they can be successfully evaluated (via [`ConstEvaluatable`][const_evaluatable] goals). This means that simply checking well formedness of const arguments is sufficient to error if they would fail to evaluate. It is somewhat unclear whether it would make sense to adopt this for type aliases too or if const aliases should stop requiring this for well formedness[^5]. + +[^1]: In the new solver this is done implicitly + +[^2]: There is a subtle difference in how ambiguous aliases in binders are handled between old and new solver. In the old solver we fail to error on some ambiguous aliases inside of higher ranked types whereas the new solver correctly errors. + +[^3]: Ambiguous aliases inside of binders cannot be normalized to inference variables, this will be covered more later. + +[^4]: As checking aliases are non-diverging cannot be done until they are fully concrete, this would either imply that we cant check aliases are well formed before codegen/const-evaluation or that aliases would go from being well-formed to not well-formed after monomorphization. + +[^5]: Const aliases certainly wouldn't be *less* sound than type aliases if we stopped doing this + +[const_evaluatable]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/type.ClauseKind.html#variant.ConstEvaluatable diff --git a/src/notification-groups/about.md b/src/notification-groups/about.md index 74629aa08..d75891ecf 100644 --- a/src/notification-groups/about.md +++ b/src/notification-groups/about.md @@ -21,9 +21,7 @@ search for existing issues that haven't been claimed yet. Here's the list of the notification groups: - [Apple](./apple.md) - [ARM](./arm.md) -- [Cleanup Crew](./cleanup-crew.md) - [Emscripten](./emscripten.md) -- [LLVM](./llvm.md) - [RISC-V](./risc-v.md) - [WASI](./wasi.md) - [WebAssembly](./wasm.md) @@ -64,9 +62,7 @@ Example PRs: * [Example of adding yourself to the Apple group.](https://github.com/rust-lang/team/pull/1434) * [Example of adding yourself to the ARM group.](https://github.com/rust-lang/team/pull/358) -* [Example of adding yourself to the Cleanup Crew.](https://github.com/rust-lang/team/pull/221) * [Example of adding yourself to the Emscripten group.](https://github.com/rust-lang/team/pull/1579) -* [Example of adding yourself to the LLVM group.](https://github.com/rust-lang/team/pull/140) * [Example of adding yourself to the RISC-V group.](https://github.com/rust-lang/team/pull/394) * [Example of adding yourself to the WASI group.](https://github.com/rust-lang/team/pull/1580) * [Example of adding yourself to the WebAssembly group.](https://github.com/rust-lang/team/pull/1581) @@ -81,9 +77,7 @@ group. For example: ```text @rustbot ping apple @rustbot ping arm -@rustbot ping cleanup-crew @rustbot ping emscripten -@rustbot ping llvm @rustbot ping risc-v @rustbot ping wasi @rustbot ping wasm @@ -92,12 +86,12 @@ group. For example: To make some commands shorter and easier to remember, there are aliases, defined in the [`triagebot.toml`] file. For example, all of these commands -are equivalent and will ping the Cleanup Crew: +are equivalent and will ping the Apple group: ```text -@rustbot ping cleanup -@rustbot ping bisect -@rustbot ping reduce +@rustbot ping apple +@rustbot ping macos +@rustbot ping ios ``` Keep in mind that these aliases are meant to make humans' life easier. diff --git a/src/notification-groups/cleanup-crew.md b/src/notification-groups/cleanup-crew.md deleted file mode 100644 index 9cf4e512c..000000000 --- a/src/notification-groups/cleanup-crew.md +++ /dev/null @@ -1,90 +0,0 @@ -# Cleanup Crew - -**Github Label:** [ICEBreaker-Cleanup-Crew]
-**Ping command:** `@rustbot ping cleanup-crew` - -[ICEBreaker-Cleanup-Crew]: https://github.com/rust-lang/rust/labels/ICEBreaker-Cleanup-Crew - -The "Cleanup Crew" are focused on improving bug reports. Specifically, -the goal is to try to ensure that every bug report has all the -information that will be needed for someone to fix it: - -* a minimal, standalone example that shows the problem -* links to duplicates or related bugs -* if the bug is a regression (something that used to work, but no longer does), - then a bisection to the PR or nightly that caused the regression - -This kind of cleanup is invaluable in getting bugs fixed. Better -still, it can be done by anybody who knows Rust, without any -particularly deep knowledge of the compiler. - -Let's look a bit at the workflow for doing "cleanup crew" actions. - -## Finding a minimal, standalone example - -Here the ultimate goal is to produce an example that reproduces the same -problem but without relying on any external crates. Such a test ought to contain -as little code as possible, as well. This will make it much easier to isolate the problem. - -However, even if the "ultimate minimal test" cannot be achieved, it's -still useful to post incremental minimizations. For example, if you -can eliminate some of the external dependencies, that is helpful, and -so forth. - -It's particularly useful to reduce to an example that works -in the [Rust playground](https://play.rust-lang.org/), rather than -requiring people to checkout a cargo build. - -There are many resources for how to produce minimized test cases. Here -are a few: - -* The [rust-reduce](https://github.com/jethrogb/rust-reduce) tool can try to reduce - code automatically. - * The [C-reduce](https://github.com/csmith-project/creduce) tool also works - on Rust code, though it requires that you start from a single - file. (A post explaining how to do it can be found [here](https://insaneinside.net/2017/09/12/whole-crate-bug-reduction-with-creduce.html).) -* pnkfelix's [Rust Bug Minimization Patterns] blog post - * This post focuses on "heavy bore" techniques, where you are - starting with a large, complex cargo project that you wish to - narrow down to something standalone. - -[Rust Bug Minimization Patterns]: http://blog.pnkfx.org/blog/2019/11/18/rust-bug-minimization-patterns/ - -## Links to duplicate or related bugs - -If you are on the "Cleanup Crew", you will sometimes see multiple bug -reports that seem very similar. You can link one to the other just by -mentioning the other bug number in a Github comment. Sometimes it is -useful to close duplicate bugs. But if you do so, you should always -copy any test case from the bug you are closing to the other bug that -remains open, as sometimes duplicate-looking bugs will expose -different facets of the same problem. - -## Bisecting regressions - -For regressions (something that used to work, but no longer does), it -is super useful if we can figure out precisely when the code stopped -working. The gold standard is to be able to identify the precise -**PR** that broke the code, so we can ping the author, but even -narrowing it down to a nightly build is helpful, especially as that -then gives us a range of PRs. (One other challenge is that we -sometimes land "rollup" PRs, which combine multiple PRs into one.) - -### cargo-bisect-rustc - -To help in figuring out the cause of a regression we have a tool -called [cargo-bisect-rustc]. It will automatically download and test -various builds of rustc. For recent regressions, it is even able to -use the builds from our CI to track down the regression to a specific -PR; for older regressions, it will simply identify a nightly. - -To learn to use [cargo-bisect-rustc], check out [this blog post][learn], which -gives a quick introduction to how it works. Additionally, there is a [Guide] -which goes into more detail on how to use it. You can also ask questions at -the Zulip stream [`#t-compiler/cargo-bisect-rustc`][zcbr], or help in -improving the tool. - -[cargo-bisect-rustc]: https://github.com/rust-lang/cargo-bisect-rustc/ -[learn]: https://blog.rust-lang.org/inside-rust/2019/12/18/bisecting-rust-compiler.html -[zcbr]: https://rust-lang.zulipchat.com/#narrow/stream/217417-t-compiler.2Fcargo-bisect-rustc -[Guide]: https://rust-lang.github.io/cargo-bisect-rustc/ diff --git a/src/notification-groups/fuchsia.md b/src/notification-groups/fuchsia.md new file mode 100644 index 000000000..e3c1a7148 --- /dev/null +++ b/src/notification-groups/fuchsia.md @@ -0,0 +1,12 @@ +# Fuchsia notification group + +**Github Label:** [O-fuchsia]
+**Ping command:** `@rustbot ping fuchsia` + +[O-fuchsia]: https://github.com/rust-lang/rust/labels/O-fuchsia + +This list will be used to notify [Fuchsia][fuchsia] maintainers +when the compiler or the standard library changes in a way that would +break the Fuchsia integration. + +[fuchsia]: ../tests/ecosystem-test-jobs/fuchsia.md diff --git a/src/notification-groups/llvm.md b/src/notification-groups/llvm.md deleted file mode 100644 index 2eff63713..000000000 --- a/src/notification-groups/llvm.md +++ /dev/null @@ -1,35 +0,0 @@ -# LLVM Notification group - -**Github Label:** [A-LLVM]
-**Ping command:** `@rustbot ping llvm` - -[A-LLVM]: https://github.com/rust-lang/rust/labels/A-LLVM - -The "LLVM Notification Group" are focused on bugs that center around LLVM. -These bugs often arise because of LLVM optimizations gone awry, or as -the result of an LLVM upgrade. The goal here is: - -- to determine whether the bug is a result of us generating invalid LLVM IR, - or LLVM misoptimizing; -- if the former, to fix our IR; -- if the latter, to try and file a bug on LLVM (or identify an existing bug). - -The group may also be asked to weigh in on other sorts of LLVM-focused -questions. - -## Helpful tips and options - -The ["Debugging LLVM"][d] section of the -rustc-dev-guide gives a step-by-step process for how to help debug bugs -caused by LLVM. In particular, it discusses how to emit LLVM IR, run -the LLVM IR optimization pipelines, and so forth. You may also find -it useful to look at the various codegen options listed under `-C help` -and the internal options under `-Z help` -- there are a number that -pertain to LLVM (just search for LLVM). - -[d]: ../backend/debugging.md - -## If you do narrow to an LLVM bug - -The ["Debugging LLVM"][d] section also describes what to do once -you've identified the bug. diff --git a/src/notification-groups/rust-for-linux.md b/src/notification-groups/rust-for-linux.md index 9ba4eff62..696f2038e 100644 --- a/src/notification-groups/rust-for-linux.md +++ b/src/notification-groups/rust-for-linux.md @@ -1,9 +1,9 @@ # Rust for Linux notification group -**Github Label:** [O-rfl]
+**Github Label:** [A-rust-for-linux]
**Ping command:** `@rustbot ping rfl` -[O-rfl]: https://github.com/rust-lang/rust/labels/O-rfl +[A-rust-for-linux]: https://github.com/rust-lang/rust/labels/A-rust-for-linux This list will be used to notify [Rust for Linux (RfL)][rfl] maintainers when the compiler or the standard library changes in a way that would diff --git a/src/offload/installation.md b/src/offload/installation.md new file mode 100644 index 000000000..b376e962f --- /dev/null +++ b/src/offload/installation.md @@ -0,0 +1,71 @@ +# Installation + +In the future, `std::offload` should become available in nightly builds for users. For now, everyone still needs to build rustc from source. + +## Build instructions + +First you need to clone and configure the Rust repository: +```bash +git clone git@github.com:rust-lang/rust +cd rust +./configure --enable-llvm-link-shared --release-channel=nightly --enable-llvm-assertions --enable-llvm-offload --enable-llvm-enzyme --enable-clang --enable-lld --enable-option-checking --enable-ninja --disable-docs +``` + +Afterwards you can build rustc using: +```bash +./x build --stage 1 library +``` + +Afterwards rustc toolchain link will allow you to use it through cargo: +``` +rustup toolchain link offload build/host/stage1 +rustup toolchain install nightly # enables -Z unstable-options +``` + + + +## Build instruction for LLVM itself +```bash +git clone git@github.com:llvm/llvm-project +cd llvm-project +mkdir build +cd build +cmake -G Ninja ../llvm -DLLVM_TARGETS_TO_BUILD="host,AMDGPU,NVPTX" -DLLVM_ENABLE_ASSERTIONS=ON -DLLVM_ENABLE_PROJECTS="clang;lld" -DLLVM_ENABLE_RUNTIMES="offload,openmp" -DLLVM_ENABLE_PLUGINS=ON -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=. +ninja +ninja install +``` +This gives you a working LLVM build. + + +## Testing +run +``` +./x test --stage 1 tests/codegen-llvm/gpu_offload +``` + +## Usage +It is important to use a clang compiler build on the same llvm as rustc. Just calling clang without the full path will likely use your system clang, which probably will be incompatible. +``` +/absolute/path/to/rust/build/x86_64-unknown-linux-gnu/stage1/bin/rustc --edition=2024 --crate-type cdylib src/main.rs --emit=llvm-ir -O -C lto=fat -Cpanic=abort -Zoffload=Enable +/absolute/path/to/rust/build/x86_64-unknown-linux-gnu/llvm/bin/clang++ -fopenmp --offload-arch=native -g -O3 main.ll -o main -save-temps +LIBOMPTARGET_INFO=-1 ./main +``` +The first step will generate a `main.ll` file, which has enough instructions to cause the offload runtime to move data to and from a gpu. +The second step will use clang as the compilation driver to compile our IR file down to a working binary. Only a very small Rust subset will work out of the box here, unless +you use features like build-std, which are not covered by this guide. Look at the codegen test to get a feeling for how to write a working example. +In the last step you can run your binary, if all went well you will see a data transfer being reported: +``` +omptarget device 0 info: Entering OpenMP data region with being_mapper at unknown:0:0 with 1 arguments: +omptarget device 0 info: tofrom(unknown)[1024] +omptarget device 0 info: Creating new map entry with HstPtrBase=0x00007fffffff9540, HstPtrBegin=0x00007fffffff9540, TgtAllocBegin=0x0000155547200000, TgtPtrBegin=0x0000155547200000, Size=1024, DynRefCount=1, HoldRefCount=0, Name=unknown +omptarget device 0 info: Copying data from host to device, HstPtr=0x00007fffffff9540, TgtPtr=0x0000155547200000, Size=1024, Name=unknown +omptarget device 0 info: OpenMP Host-Device pointer mappings after block at unknown:0:0: +omptarget device 0 info: Host Ptr Target Ptr Size (B) DynRefCount HoldRefCount Declaration +omptarget device 0 info: 0x00007fffffff9540 0x0000155547200000 1024 1 0 unknown at unknown:0:0 +// some other output +omptarget device 0 info: Exiting OpenMP data region with end_mapper at unknown:0:0 with 1 arguments: +omptarget device 0 info: tofrom(unknown)[1024] +omptarget device 0 info: Mapping exists with HstPtrBegin=0x00007fffffff9540, TgtPtrBegin=0x0000155547200000, Size=1024, DynRefCount=0 (decremented, delayed deletion), HoldRefCount=0 +omptarget device 0 info: Copying data from device to host, TgtPtr=0x0000155547200000, HstPtr=0x00007fffffff9540, Size=1024, Name=unknown +omptarget device 0 info: Removing map entry with HstPtrBegin=0x00007fffffff9540, TgtPtrBegin=0x0000155547200000, Size=1024, Name=unknown +``` diff --git a/src/offload/internals.md b/src/offload/internals.md new file mode 100644 index 000000000..28857a6e7 --- /dev/null +++ b/src/offload/internals.md @@ -0,0 +1,9 @@ +# std::offload + +This module is under active development. Once upstream, it should allow Rust developers to run Rust code on GPUs. +We aim to develop a `rusty` GPU programming interface, which is safe, convenient and sufficiently fast by default. +This includes automatic data movement to and from the GPU, in a efficient way. We will (later) +also offer more advanced, possibly unsafe, interfaces which allow a higher degree of control. + +The implementation is based on LLVM's "offload" project, which is already used by OpenMP to run Fortran or C++ code on GPUs. +While the project is under development, users will need to call other compilers like clang to finish the compilation process. diff --git a/src/opaque-types-impl-trait-inference.md b/src/opaque-types-impl-trait-inference.md index c56d51a4b..42600ad87 100644 --- a/src/opaque-types-impl-trait-inference.md +++ b/src/opaque-types-impl-trait-inference.md @@ -13,13 +13,16 @@ it can work across functions and function bodies. To help explain how it works, let's consider an example. ```rust +#![feature(type_alias_impl_trait)] mod m { pub type Seq = impl IntoIterator; + #[define_opaque(Seq)] pub fn produce_singleton(t: T) -> Seq { vec![t] } + #[define_opaque(Seq)] pub fn produce_doubleton(t: T, u: T) -> Seq { vec![t, u] } @@ -78,7 +81,7 @@ If that fails, we reveal the hidden type of the opaque type, but only to prove this specific trait bound, not in general. Revealing is done by invoking the `type_of` query on the `DefId` of the opaque type. The query will internally request the hidden types from the defining function(s) -and return that (see [the section on `type_of`](#Within-the-type_of-query) for more details). +and return that (see [the section on `type_of`](#within-the-type_of-query) for more details). #### Flowchart of type checking steps diff --git a/src/overview.md b/src/overview.md index cc17eaa9e..378d8c445 100644 --- a/src/overview.md +++ b/src/overview.md @@ -1,7 +1,5 @@ # Overview of the compiler - - This chapter is about the overall process of compiling a program -- how everything fits together. @@ -38,7 +36,7 @@ Unicode character encoding. The token stream passes through a higher-level lexer located in [`rustc_parse`] to prepare for the next stage of the compile process. The -[`StringReader`] `struct` is used at this stage to perform a set of validations +[`Lexer`] `struct` is used at this stage to perform a set of validations and turn strings into interned symbols (_interning_ is discussed later). [String interning] is a way of storing only one immutable copy of each distinct string value. @@ -153,7 +151,7 @@ the final binary. [`rustc_parse::parser::Parser`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_parse/parser/struct.Parser.html [`rustc_parse`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_parse/index.html [`simplify_try`]: https://github.com/rust-lang/rust/pull/66282 -[`StringReader`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_parse/lexer/struct.StringReader.html +[`Lexer`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_parse/lexer/struct.Lexer.html [`Ty<'tcx>`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/struct.Ty.html [borrow checking]: borrow_check.md [codegen]: backend/codegen.md @@ -323,6 +321,10 @@ the name `'tcx`, which means that something is tied to the lifetime of the [`TyCtxt`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/struct.TyCtxt.html +For more information about queries in the compiler, see [the queries chapter][queries]. + +[queries]: ./query.md + ### `ty::Ty` Types are really important in Rust, and they form the core of a lot of compiler @@ -351,7 +353,7 @@ approach is to turn [`RefCell`]s into [`Mutex`]s -- that is, we switch to thread-safe internal mutability. However, there are ongoing challenges with lock contention, maintaining query-system invariants under concurrency, and the complexity of the code base. One can try out the current -work by enabling parallel compilation in `config.toml`. It's still early days, +work by enabling parallel compilation in `bootstrap.toml`. It's still early days, but there are already some promising performance improvements. [`RefCell`]: https://doc.rust-lang.org/std/cell/struct.RefCell.html @@ -410,7 +412,7 @@ For more details on bootstrapping, see - Guide: [The HIR](hir.md) - Guide: [Identifiers in the HIR](hir.md#identifiers-in-the-hir) - Guide: [The `HIR` Map](hir.md#the-hir-map) - - Guide: [Lowering `AST` to `HIR`](ast-lowering.md) + - Guide: [Lowering `AST` to `HIR`](./hir/lowering.md) - How to view `HIR` representation for your code `cargo rustc -- -Z unpretty=hir-tree` - Rustc `HIR` definition: [`rustc_hir`](https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir/index.html) - Main entry point: **TODO** diff --git a/src/panic-implementation.md b/src/panic-implementation.md index f35874286..dba3f2146 100644 --- a/src/panic-implementation.md +++ b/src/panic-implementation.md @@ -1,6 +1,4 @@ -# Panicking in rust - - +# Panicking in Rust ## Step 1: Invocation of the `panic!` macro. diff --git a/src/parallel-rustc.md b/src/parallel-rustc.md index 2dae95a34..ce69b66c2 100644 --- a/src/parallel-rustc.md +++ b/src/parallel-rustc.md @@ -1,4 +1,4 @@ -# Parallel Compilation +# Parallel compilation
As of November 2024, @@ -28,7 +28,7 @@ The following sections are kept for now but are quite outdated. [codegen]: backend/codegen.md -## Code Generation +## Code generation During monomorphization the compiler splits up all the code to be generated into smaller chunks called _codegen units_. These are then generated by @@ -38,7 +38,7 @@ occurs in the [`rustc_codegen_ssa::base`] module. [`rustc_codegen_ssa::base`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_codegen_ssa/base/index.html -## Data Structures +## Data structures The underlying thread-safe data-structures used in the parallel compiler can be found in the [`rustc_data_structures::sync`] module. These data structures @@ -46,20 +46,14 @@ are implemented differently depending on whether `parallel-compiler` is true. | data structure | parallel | non-parallel | | -------------------------------- | --------------------------------------------------- | ------------ | -| Lrc | std::sync::Arc | std::rc::Rc | -| Weak | std::sync::Weak | std::rc::Weak | -| Atomic{Bool}/{Usize}/{U32}/{U64} | std::sync::atomic::Atomic{Bool}/{Usize}/{U32}/{U64} | (std::cell::Cell) | -| OnceCell | std::sync::OnceLock | std::cell::OnceCell | | Lock\ | (parking_lot::Mutex\) | (std::cell::RefCell) | | RwLock\ | (parking_lot::RwLock\) | (std::cell::RefCell) | -| MTRef<'a, T> | &'a T | &'a mut T | | MTLock\ | (Lock\) | (T) | | ReadGuard | parking_lot::RwLockReadGuard | std::cell::Ref | | MappedReadGuard | parking_lot::MappedRwLockReadGuard | std::cell::Ref | | WriteGuard | parking_lot::RwLockWriteGuard | std::cell::RefMut | | MappedWriteGuard | parking_lot::MappedRwLockWriteGuard | std::cell::RefMut | | LockGuard | parking_lot::MutexGuard | std::cell::RefMut | -| MappedLockGuard | parking_lot::MappedMutexGuard | std::cell::RefMut | - These thread-safe data structures are interspersed during compilation which can cause lock contention resulting in degraded performance as the number of @@ -89,7 +83,7 @@ can be accessed directly through `Deref::deref`. [`rustc_data_structures::sync::worker_local`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_data_structures/sync/worker_local/index.html [`WorkerLocal`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_data_structures/sync/worker_local/struct.WorkerLocal.html -## Parallel Iterator +## Parallel iterator The parallel iterators provided by the [`rayon`] crate are easy ways to implement parallelism. In the current implementation of the parallel compiler @@ -130,7 +124,7 @@ the parallel iterator function has been used are as follows: There are still many loops that have the potential to use parallel iterators. -## Query System +## Query system The query model has some properties that make it actually feasible to evaluate multiple queries in parallel without too much effort: @@ -174,12 +168,10 @@ Here are some resources that can be used to learn more: - [This list of interior mutability in the compiler by nikomatsakis][imlist] [`rayon`]: https://crates.io/crates/rayon -[Arc]: https://doc.rust-lang.org/std/sync/struct.Arc.html [imlist]: https://github.com/nikomatsakis/rustc-parallelization/blob/master/interior-mutability-list.md [irlo0]: https://internals.rust-lang.org/t/parallelizing-rustc-using-rayon/6606 [irlo1]: https://internals.rust-lang.org/t/help-test-parallel-rustc/11503 [monomorphization]: backend/monomorph.md [parallel-rustdoc]: https://github.com/rust-lang/rust/issues/82741 -[Rc]: https://doc.rust-lang.org/std/rc/struct.Rc.html [rustc-rayon]: https://github.com/rust-lang/rustc-rayon [tracking]: https://github.com/rust-lang/rust/issues/48685 diff --git a/src/param_env/param_env_acquisition.md b/src/param_env/param_env_acquisition.md deleted file mode 100644 index 391e56291..000000000 --- a/src/param_env/param_env_acquisition.md +++ /dev/null @@ -1,44 +0,0 @@ - -# Which `ParamEnv` do I use? - -When needing a [`ParamEnv`][pe] in the compiler there are a few options for obtaining one: -- The correct env is already in scope simply use it (or pass it down the call stack to where you are). -- The [`tcx.param_env(def_id)` query][param_env_query] -- Use [`ParamEnv::new`][param_env_new] to construct an env with an arbitrary set of where clauses. Then call [`traits::normalize_param_env_or_error`][normalize_env_or_error] which will handle normalizing and elaborating all the where clauses in the env for you. -- Creating an empty environment via [`ParamEnv::reveal_all`][env_reveal_all] or [`ParamEnv::empty`][env_empty] - -In the large majority of cases a `ParamEnv` when required already exists somewhere in scope or above in the call stack and should be passed down. A non exhaustive list of places where you might find an existing `ParamEnv`: -- During typeck `FnCtxt` has a [`param_env` field][fnctxt_param_env] -- When writing late lints the `LateContext` has a [`param_env` field][latectxt_param_env] -- During well formedness checking the `WfCheckingCtxt` has a [`param_env` field][wfckctxt_param_env] -- The `TypeChecker` used by Mir Typeck has a [`param_env` field][mirtypeck_param_env] -- In the next-gen trait solver all `Goal`s have a [`param_env` field][goal_param_env] specifying what environment to prove the goal in -- When editing an existing [`TypeRelation`][typerelation] if it implements `PredicateEmittingRelation` then a [`param_env` method][typerelation_param_env] will be available. - -Using the `param_env` query to obtain an env is generally done at the start of some kind of analysis and then passed everywhere that a `ParamEnv` is required. For example the type checker will create a `ParamEnv` for the item it is type checking and then pass it around everywhere. - -Creating an env from an arbitrary set of where clauses is usually unnecessary and should only be done if the environment you need does not correspond to an actual item in the source code (i.e. [`compare_method_predicate_entailment`][method_pred_entailment] as mentioned earlier). - -Creating an empty environment via `ParamEnv::empty` is almost always wrong. There are very few places where we actually know that the environment should be empty. One of the only places where we do actually know this is after monomorphization, however the `ParamEnv` there should be constructed via `ParamEnv::reveal_all` instead as at this point we should be able to determine the hidden type of opaque types. Codegen/Post-mono is one of the only places that should be using `ParamEnv::reveal_all`. - -An additional piece of complexity here is specifying the [`Reveal`][reveal] (see linked docs for explanation of what reveal does) used for the `ParamEnv`. When constructing a param env using the `param_env` query it will have `Reveal::UserFacing`, if `Reveal::All` is desired then the [`tcx.param_env_reveal_all_normalized`][env_reveal_all_normalized] query can be used instead. - -The `ParamEnv` type has a method [`ParamEnv::with_reveal_all_normalized`][with_reveal_all] which converts an existing `ParamEnv` into one with `Reveal::All` specified. Where possible the previously mentioned query should be preferred as it is more efficient. - -[param_env_new]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/struct.ParamEnv.html#method.new -[normalize_env_or_error]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_trait_selection/traits/fn.normalize_param_env_or_error.html -[fnctxt_param_env]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir_typeck/fn_ctxt/struct.FnCtxt.html#structfield.param_env -[latectxt_param_env]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_lint/context/struct.LateContext.html#structfield.param_env -[wfckctxt_param_env]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir_analysis/check/wfcheck/struct.WfCheckingCtxt.html#structfield.param_env -[goal_param_env]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_infer/infer/canonical/ir/solve/struct.Goal.html#structfield.param_env -[typerelation_param_env]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_infer/infer/trait.PredicateEmittingRelation.html#tymethod.param_env -[typerelation]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/relate/trait.TypeRelation.html -[mirtypeck_param_env]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_borrowck/type_check/struct.TypeChecker.html#structfield.param_env -[env_reveal_all_normalized]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/context/struct.TyCtxt.html#method.param_env_reveal_all_normalized -[with_reveal_all]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/struct.ParamEnv.html#method.with_reveal_all_normalized -[env_reveal_all]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/struct.ParamEnv.html#method.reveal_all -[env_empty]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/struct.ParamEnv.html#method.empty -[reveal]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_infer/traits/enum.Reveal.html -[pe]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/struct.ParamEnv.html -[param_env_query]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir_typeck/fn_ctxt/struct.FnCtxt.html#structfield.param_env -[method_pred_entailment]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir_analysis/check/compare_impl_item/fn.compare_method_predicate_entailment.html diff --git a/src/param_env/param_env_construction_internals.md b/src/param_env/param_env_construction_internals.md deleted file mode 100644 index 69a262a17..000000000 --- a/src/param_env/param_env_construction_internals.md +++ /dev/null @@ -1,83 +0,0 @@ - -# How are `ParamEnv`'s constructed internally? - -Creating a [`ParamEnv`][pe] is more complicated than simply using the list of where clauses defined on an item as written by the user. We need to both elaborate supertraits into the env and fully normalize all aliases. This logic is handled by [`traits::normalize_param_env_or_error`][normalize_env_or_error] (even though it does not mention anything about elaboration). - -## Elaborating supertraits - -When we have a function such as `fn foo()` we would like to be able to prove `T: Clone` inside of the function as the `Copy` trait has a `Clone` supertrait. Constructing a `ParamEnv` looks at all of the trait bounds in the env and explicitly adds new where clauses to the `ParamEnv` for any supertraits found on the traits. - -A concrete example would be the following function: -```rust -trait Trait: SuperTrait {} -trait SuperTrait: SuperSuperTrait {} - -// `bar`'s unelaborated `ParamEnv` would be: -// `[T: Sized, T: Copy, T: Trait]` -fn bar(a: T) { - requires_impl(a); -} - -fn requires_impl(a: T) {} -``` - -If we did not elaborate the env then the `requires_impl` call would fail to typecheck as we would not be able to prove `T: Clone` or `T: SuperSuperTrait`. In practice we elaborate the env which means that `bar`'s `ParamEnv` is actually: -`[T: Sized, T: Copy, T: Clone, T: Trait, T: SuperTrait, T: SuperSuperTrait]` -This allows us to prove `T: Clone` and `T: SuperSuperTrait` when type checking `bar`. - -The `Clone` trait has a `Sized` supertrait however we do not end up with two `T: Sized` bounds in the env (one for the supertrait and one for the implicitly added `T: Sized` bound). This is because the elaboration process (implemented via [`util::elaborate`][elaborate]) deduplicates the where clauses to avoid this. - -As a side effect this also means that even if no actual elaboration of supertraits takes place, the existing where clauses in the env are _also_ deduplicated. See the following example: -```rust -trait Trait {} -// The unelaborated `ParamEnv` would be: -// `[T: Sized, T: Trait, T: Trait]` -// but after elaboration it would be: -// `[T: Sized, T: Trait]` -fn foo() {} -``` - -The [next-gen trait solver][next-gen-solver] also requires this elaboration to take place. - -[elaborate]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_infer/traits/util/fn.elaborate.html -[next-gen-solver]: ../solve/trait-solving.md - -## Normalizing all bounds - -In the old trait solver the where clauses stored in `ParamEnv` are required to be fully normalized or else the trait solver will not function correctly. A concrete example of needing to normalize the `ParamEnv` is the following: -```rust -trait Trait { - type Assoc; -} - -trait Other { - type Bar; -} - -impl Other for T { - type Bar = u32; -} - -// `foo`'s unnormalized `ParamEnv` would be: -// `[T: Sized, U: Sized, U: Trait]` -fn foo(a: U) -where - U: Trait<::Bar>, -{ - requires_impl(a); -} - -fn requires_impl>(_: U) {} -``` - -As humans we can tell that `::Bar` is equal to `u32` so the trait bound on `U` is equivalent to `U: Trait`. In practice trying to prove `U: Trait` in the old solver in this environment would fail as it is unable to determine that `::Bar` is equal to `u32`. - -To work around this we normalize `ParamEnv`'s after constructing them so that `foo`'s `ParamEnv` is actually: `[T: Sized, U: Sized, U: Trait]` which means the trait solver is now able to use the `U: Trait` in the `ParamEnv` to determine that the trait bound `U: Trait` holds. - -This workaround does not work in all cases as normalizing associated types requires a `ParamEnv` which introduces a bootstrapping problem. We need a normalized `ParamEnv` in order for normalization to give correct results, but we need to normalize to get that `ParamEnv`. Currently we normalize the `ParamEnv` once using the unnormalized param env and it tends to give okay results in practice even though there are some examples where this breaks ([example]). - -In the next-gen trait solver the requirement for all where clauses in the `ParamEnv` to be fully normalized is not present and so we do not normalize when constructing `ParamEnv`s. - -[example]: https://play.rust-lang.org/?version=stable&mode=debug&edition=2021&gist=e6933265ea3e84eaa47019465739992c -[pe]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/struct.ParamEnv.html -[normalize_env_or_error]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_trait_selection/traits/fn.normalize_param_env_or_error.html diff --git a/src/param_env/param_env_summary.md b/src/param_env/param_env_summary.md deleted file mode 100644 index 0ff6d8fc3..000000000 --- a/src/param_env/param_env_summary.md +++ /dev/null @@ -1,18 +0,0 @@ -# The `ParamEnv` type - -## Summary - -The [`ParamEnv`][pe] is used to store information about the environment that we are interacting with the type system from. For example the set of in-scope where-clauses is stored in `ParamEnv` as it differs between each item whereas the list of user written impls is not stored in the `ParamEnv` as this does not change for each item. - -This chapter of the dev guide covers: -- A high level summary of what a `ParamEnv` is and what it is used for -- Technical details about what the process of constructing a `ParamEnv` involves -- Guidance about how to acquire a `ParamEnv` when one is required - -## Bundling - -A useful API on `ParamEnv` is the [`and`][and] method which allows bundling a value with the `ParamEnv`. The `and` method produces a [`ParamEnvAnd`][pea] making it clearer that using the inner value is intended to be done in that specific environment. - -[and]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/struct.ParamEnv.html#method.and -[pe]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/struct.ParamEnv.html -[pea]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/struct.ParamEnvAnd.html \ No newline at end of file diff --git a/src/param_env/param_env_what_is_it.md b/src/param_env/param_env_what_is_it.md deleted file mode 100644 index ca09518d9..000000000 --- a/src/param_env/param_env_what_is_it.md +++ /dev/null @@ -1,60 +0,0 @@ - -# What is a `ParamEnv`? - -The type system relies on information in the environment in order for it to function correctly. This information is stored in the [`ParamEnv`][pe] type and it is important to use the correct `ParamEnv` when interacting with the type system. - -The information represented by `ParamEnv` is a list of in-scope where-clauses, and a [`Reveal`][reveal] (see linked docs for more information). A `ParamEnv` typically corresponds to a specific item's where clauses, some clauses are not explicitly written bounds and instead are implicitly added in [`predicates_of`][predicates_of] such as `ConstArgHasType` or some implied bounds. - -A `ParamEnv` can also be created with arbitrary data that is not derived from a specific item such as in [`compare_method_predicate_entailment`][method_pred_entailment] which creates a hybrid `ParamEnv` consisting of the impl's where clauses and the trait definition's function's where clauses. In most cases `ParamEnv`s are initially created via the [`param_env` query][query] which returns a `ParamEnv` derived from the provided item's where clauses. - -If we have a function such as: -```rust -// `foo` would have a `ParamEnv` of: -// `[T: Sized, T: Trait, ::Assoc: Clone]` -fn foo() -where - ::Assoc: Clone, -{} -``` -If we were conceptually inside of `foo` (for example, type-checking or linting it) we would use this `ParamEnv` everywhere that we interact with the type system. This would allow things such as normalization (TODO: write a chapter about normalization and link it), evaluating generic constants, and proving where clauses/goals, to rely on `T` being sized, implementing `Trait`, etc. - -A more concrete example: -```rust -// `foo` would have a `ParamEnv` of: -// `[T: Sized, T: Clone]` -fn foo(a: T) { - // when typechecking `foo` we require all the where clauses on `bar` - // to hold in order for it to be legal to call. This means we have to - // prove `T: Clone`. As we are type checking `foo` we use `foo`'s - // environment when trying to check that `T: Clone` holds. - // - // Trying to prove `T: Clone` with a `ParamEnv` of `[T: Sized, T: Clone]` - // will trivially succeed as bound we want to prove is in our environment. - requires_clone(a); -} -``` - -Or alternatively an example that would not compile: -```rust -// `foo2` would have a `ParamEnv` of: -// `[T: Sized]` -fn foo2(a: T) { - // When typechecking `foo2` we attempt to prove `T: Clone`. - // As we are type checking `foo2` we use `foo2`'s environment - // when trying to prove `T: Clone`. - // - // Trying to prove `T: Clone` with a `ParamEnv` of `[T: Sized]` will - // fail as there is nothing in the environment telling the trait solver - // that `T` implements `Clone` and there exists no user written impl - // that could apply. - requires_clone(a); -} -``` - -It's very important to use the correct `ParamEnv` when interacting with the type system as otherwise it can lead to ICEs or things compiling when they shouldn't (or vice versa). See [#82159](https://github.com/rust-lang/rust/pull/82159) and [#82067](https://github.com/rust-lang/rust/pull/82067) as examples of PRs that changed rustc to use the correct param env to avoid ICE. Determining how to acquire the correct `ParamEnv` is explained later in this chapter. - -[predicates_of]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir_analysis/collect/predicates_of/fn.predicates_of.html -[method_pred_entailment]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir_analysis/check/compare_impl_item/fn.compare_method_predicate_entailment.html -[pe]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/struct.ParamEnv.html -[query]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/context/struct.TyCtxt.html#method.param_env -[reveal]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_infer/traits/enum.Reveal.html \ No newline at end of file diff --git a/src/part-5-intro.md b/src/part-5-intro.md index f32508d27..a44fff1e1 100644 --- a/src/part-5-intro.md +++ b/src/part-5-intro.md @@ -1,4 +1,4 @@ -# From MIR to Binaries +# From MIR to binaries All of the preceding chapters of this guide have one thing in common: we never generated any executable machine code at all! diff --git a/src/pat-exhaustive-checking.md b/src/pat-exhaustive-checking.md index 4a796ac95..e953931aa 100644 --- a/src/pat-exhaustive-checking.md +++ b/src/pat-exhaustive-checking.md @@ -1,4 +1,4 @@ -# Pattern and Exhaustiveness Checking +# Pattern and exhaustiveness checking In Rust, pattern matching and bindings have a few very helpful properties. The compiler will check that bindings are irrefutable when made and that match arms diff --git a/src/profile-guided-optimization.md b/src/profile-guided-optimization.md index 995752b0b..4e3dadd40 100644 --- a/src/profile-guided-optimization.md +++ b/src/profile-guided-optimization.md @@ -1,12 +1,10 @@ -# Profile Guided Optimization - - +# Profile-guided optimization `rustc` supports doing profile-guided optimization (PGO). This chapter describes what PGO is and how the support for it is implemented in `rustc`. -## What Is Profiled-Guided Optimization? +## What is profiled-guided optimization? The basic concept of PGO is to collect data about the typical execution of a program (e.g. which branches it is likely to take) and then use this data @@ -52,7 +50,7 @@ instrumentation, via the experimental option [`-C instrument-coverage`](./llvm-coverage-instrumentation.md), but using these coverage results for PGO has not been attempted at this time. -### Overall Workflow +### Overall workflow Generating a PGO-optimized program involves the following four steps: @@ -62,12 +60,12 @@ Generating a PGO-optimized program involves the following four steps: 4. Compile the program again, this time making use of the profiling data (e.g. `rustc -C profile-use=merged.profdata main.rs`) -### Compile-Time Aspects +### Compile-time aspects Depending on which step in the above workflow we are in, two different things can happen at compile time: -#### Create Binaries with Instrumentation +#### Create binaries with instrumentation As mentioned above, the profiling instrumentation is added by LLVM. `rustc` instructs LLVM to do so [by setting the appropriate][pgo-gen-passmanager] @@ -88,7 +86,7 @@ runtime are not removed [by marking the with the right export level][pgo-gen-sym [pgo-gen-symbols]:https://github.com/rust-lang/rust/blob/1.34.1/src/librustc_codegen_ssa/back/symbol_export.rs#L212-L225 -#### Compile Binaries Where Optimizations Make Use Of Profiling Data +#### Compile binaries where optimizations make use of profiling data In the final step of the workflow described above, the program is compiled again, with the compiler using the gathered profiling data in order to drive @@ -106,7 +104,7 @@ LLVM does the rest (e.g. setting branch weights, marking functions with `cold` or `inlinehint`, etc). -### Runtime Aspects +### Runtime aspects Instrumentation-based approaches always also have a runtime component, i.e. once we have an instrumented program, that program needs to be run in order @@ -120,7 +118,7 @@ The `rustc` version of this can be found in `library/profiler_builtins` which basically packs the C code from `compiler-rt` into a Rust crate. In order for `profiler_builtins` to be built, `profiler = true` must be set -in `rustc`'s `config.toml`. +in `rustc`'s `bootstrap.toml`. [compiler-rt-profile]: https://github.com/llvm/llvm-project/tree/main/compiler-rt/lib/profile @@ -132,9 +130,9 @@ There is also a [codegen test][codegen-test] that checks that some expected instrumentation artifacts show up in LLVM IR. [rmake-tests]: https://github.com/rust-lang/rust/tree/master/tests/run-make -[codegen-test]: https://github.com/rust-lang/rust/blob/master/tests/codegen/pgo-instrumentation.rs +[codegen-test]: https://github.com/rust-lang/rust/blob/master/tests/codegen-llvm/pgo-instrumentation.rs -## Additional Information +## Additional information Clang's documentation contains a good overview on [PGO in LLVM][llvm-pgo]. diff --git a/src/profiling.md b/src/profiling.md index df987e00a..de06bd7cd 100644 --- a/src/profiling.md +++ b/src/profiling.md @@ -87,7 +87,7 @@ Example output for the compiler: Since this doesn't seem to work with incremental compilation or `./x check`, you will be compiling rustc _a lot_. -I recommend changing a few settings in `config.toml` to make it bearable: +I recommend changing a few settings in `bootstrap.toml` to make it bearable: ``` [rust] # A debug build takes _a third_ as long on my machine, diff --git a/src/profiling/with_perf.md b/src/profiling/with_perf.md index 6cd98f886..0d4f23bcd 100644 --- a/src/profiling/with_perf.md +++ b/src/profiling/with_perf.md @@ -6,9 +6,9 @@ This is a guide for how to profile rustc with [perf](https://perf.wiki.kernel.or - Get a clean checkout of rust-lang/master, or whatever it is you want to profile. -- Set the following settings in your `config.toml`: - - `debuginfo-level = 1` - enables line debuginfo - - `jemalloc = false` - lets you do memory use profiling with valgrind +- Set the following settings in your `bootstrap.toml`: + - `rust.debuginfo-level = 1` - enables line debuginfo + - `rust.jemalloc = false` - lets you do memory use profiling with valgrind - leave everything else the defaults - Run `./x build` to get a full build - Make a rustup toolchain pointing to that result @@ -52,6 +52,13 @@ you made in the beginning. But there are some things to be aware of: - You probably don't want incremental messing about with your profile. So something like `CARGO_INCREMENTAL=0` can be helpful. +In case to avoid the issue of `addr2line xxx/elf: could not read first record` when reading +collected data from `cargo`, you may need use the latest version of `addr2line`: + +```bash +cargo install addr2line --features="bin" +``` + ### Gathering a perf profile from a `perf.rust-lang.org` test Often we want to analyze a specific test from `perf.rust-lang.org`. diff --git a/src/profiling/with_rustc_perf.md b/src/profiling/with_rustc_perf.md index 87d205d9e..c47fed24e 100644 --- a/src/profiling/with_rustc_perf.md +++ b/src/profiling/with_rustc_perf.md @@ -7,11 +7,9 @@ However, using the suite manually can be a bit cumbersome. To make this easier f the compiler build system (`bootstrap`) also provides built-in integration with the benchmarking suite, which will download and build the suite for you, build a local compiler toolchain and let you profile it using a simplified command-line interface. -You can use the `./x perf -- [options]` command to use this integration. +You can use the `./x perf [options]` command to use this integration. -> Note that you need to specify arguments after `--` in the `x perf` command! You will not be able to pass arguments without the double dashes. - -You can use normal bootstrap flags for this command, such as `--stage 1` or `--stage 2`, for example to modify the stage of the created sysroot. It might also be useful to configure `config.toml` to better support profiling, e.g. set `rust.debuginfo-level = 1` to add source line information to the built compiler. +You can use normal bootstrap flags for this command, such as `--stage 1` or `--stage 2`, for example to modify the stage of the created sysroot. It might also be useful to configure `bootstrap.toml` to better support profiling, e.g. set `rust.debuginfo-level = 1` to add source line information to the built compiler. `x perf` currently supports the following commands: - `benchmark `: Benchmark the compiler and store the results under the passed `id`. diff --git a/src/profiling/wpa_profiling.md b/src/profiling/wpa_profiling.md index a800c5717..d2680d408 100644 --- a/src/profiling/wpa_profiling.md +++ b/src/profiling/wpa_profiling.md @@ -44,7 +44,7 @@ compiler we're using to build rustc will aid our analysis greatly by allowing WP symbols correctly. Unfortunately, the stage 0 compiler does not have symbols turned on which is why we'll need to build a stage 1 compiler and then a stage 2 compiler ourselves. -To do this, make sure you have set `debuginfo-level = 1` in your `config.toml` file. This tells +To do this, make sure you have set `debuginfo-level = 1` in your `bootstrap.toml` file. This tells rustc to generate debug information which includes stack frames when bootstrapping. Now you can build the stage 1 compiler: `x build --stage 1 -i library` or however diff --git a/src/queries/incremental-compilation-in-detail.md b/src/queries/incremental-compilation-in-detail.md index 4133b196c..46e38832e 100644 --- a/src/queries/incremental-compilation-in-detail.md +++ b/src/queries/incremental-compilation-in-detail.md @@ -1,6 +1,4 @@ -# Incremental Compilation In Detail - - +# Incremental compilation in detail The incremental compilation scheme is, in essence, a surprisingly simple extension to the overall query system. It relies on the fact that: @@ -66,7 +64,7 @@ because it reads the up-to-date version of `Hir(bar)`. Also, we re-run `type_check_item(bar)` because result of `type_of(bar)` might have changed. -## The Problem With The Basic Algorithm: False Positives +## The problem with the basic algorithm: false positives If you read the previous paragraph carefully you'll notice that it says that `type_of(bar)` *might* have changed because one of its inputs has changed. @@ -93,7 +91,7 @@ of examples like this and small changes to the input often potentially affect very large parts of the output binaries. As a consequence, we had to make the change detection system smarter and more accurate. -## Improving Accuracy: The red-green Algorithm +## Improving accuracy: the red-green algorithm The "false positives" problem can be solved by interleaving change detection and query re-evaluation. Instead of walking the graph all the way to the @@ -191,7 +189,7 @@ then itself involve recursively invoking more queries, which can mean we come ba to the `try_mark_green()` algorithm for the dependencies recursively. -## The Real World: How Persistence Makes Everything Complicated +## The real world: how persistence makes everything complicated The sections above described the underlying algorithm for incremental compilation but because the compiler process exits after being finished and @@ -258,7 +256,7 @@ the `LocalId`s within it are still the same. -### Checking Query Results For Changes: HashStable And Fingerprints +### Checking query results for changes: `HashStable` and `Fingerprint`s In order to do red-green-marking we often need to check if the result of a query has changed compared to the result it had during the previous @@ -306,7 +304,7 @@ This approach works rather well but it's not without flaws: their stable equivalents while doing the hashing. -### A Tale Of Two DepGraphs: The Old And The New +### A tale of two `DepGraph`s: the old and the new The initial description of dependency tracking glosses over a few details that quickly become a head scratcher when actually trying to implement things. @@ -344,7 +342,7 @@ new graph is serialized out to disk, alongside the query result cache, and can act as the previous dep-graph in a subsequent compilation session. -### Didn't You Forget Something?: Cache Promotion +### Didn't you forget something?: cache promotion The system described so far has a somewhat subtle property: If all inputs of a dep-node are green then the dep-node itself can be marked as green without @@ -374,7 +372,7 @@ the result cache doesn't unnecessarily shrink again. -# Incremental Compilation and the Compiler Backend +# Incremental compilation and the compiler backend The compiler backend, the part involving LLVM, is using the query system but it is not implemented in terms of queries itself. As a consequence it does not @@ -406,7 +404,7 @@ would save. -## Query Modifiers +## Query modifiers The query system allows for applying [modifiers][mod] to queries. These modifiers affect certain aspects of how the system treats the query with @@ -472,7 +470,7 @@ respect to incremental compilation: [mod]: ../query.html#adding-a-new-kind-of-query -## The Projection Query Pattern +## The projection query pattern It's interesting to note that `eval_always` and `no_hash` can be used together in the so-called "projection query" pattern. It is often the case that there is @@ -516,7 +514,7 @@ because we have the projections to take care of keeping things green as much as possible. -# Shortcomings of the Current System +# Shortcomings of the current system There are many things that still can be improved. diff --git a/src/queries/incremental-compilation.md b/src/queries/incremental-compilation.md index 6e5b4e8cc..731ff3287 100644 --- a/src/queries/incremental-compilation.md +++ b/src/queries/incremental-compilation.md @@ -1,7 +1,5 @@ # Incremental compilation - - The incremental compilation scheme is, in essence, a surprisingly simple extension to the overall query system. We'll start by describing a slightly simplified variant of the real thing – the "basic algorithm" – diff --git a/src/queries/query-evaluation-model-in-detail.md b/src/queries/query-evaluation-model-in-detail.md index f7f204bf7..c1a4373f7 100644 --- a/src/queries/query-evaluation-model-in-detail.md +++ b/src/queries/query-evaluation-model-in-detail.md @@ -1,6 +1,4 @@ -# The Query Evaluation Model in Detail - - +# The Query Evaluation Model in detail This chapter provides a deeper dive into the abstract model queries are built on. It does not go into implementation details but tries to explain diff --git a/src/queries/salsa.md b/src/queries/salsa.md index 1a7b7fa9a..dc7160edc 100644 --- a/src/queries/salsa.md +++ b/src/queries/salsa.md @@ -1,7 +1,5 @@ # How Salsa works - - This chapter is based on the explanation given by Niko Matsakis in this [video](https://www.youtube.com/watch?v=_muY4HjSqVw) about [Salsa](https://github.com/salsa-rs/salsa). To find out more you may diff --git a/src/query.md b/src/query.md index 782c5b4b3..8377a7b2f 100644 --- a/src/query.md +++ b/src/query.md @@ -1,8 +1,6 @@ # Queries: demand-driven compilation - - -As described in [the high-level overview of the compiler][hl], the Rust compiler +As described in [Overview of the compiler], the Rust compiler is still (as of July 2021) transitioning from a traditional "pass-based" setup to a "demand-driven" system. The compiler query system is the key to rustc's demand-driven organization. @@ -13,7 +11,7 @@ there is a query called `type_of` that, given the [`DefId`] of some item, will compute the type of that item and return it to you. [`DefId`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_span/def_id/struct.DefId.html -[hl]: ./compiler-src.md +[Overview of the compiler]: overview.md#queries Query execution is *memoized*. The first time you invoke a query, it will go do the computation, but the next time, the result is @@ -37,12 +35,15 @@ will in turn demand information about that crate, starting from the actual parsing. Although this vision is not fully realized, large sections of the -compiler (for example, generating [MIR](./mir/index.md)) currently work exactly like this. +compiler (for example, generating [MIR]) currently work exactly like this. -[^incr-comp-detail]: The ["Incremental Compilation in Detail](queries/incremental-compilation-in-detail.md) chapter gives a more +[^incr-comp-detail]: The [Incremental compilation in detail] chapter gives a more in-depth description of what queries are and how they work. If you intend to write a query of your own, this is a good read. +[Incremental compilation in detail]: queries/incremental-compilation-in-detail.md +[MIR]: mir/index.md + ## Invoking queries Invoking a query is simple. The [`TyCtxt`] ("type context") struct offers a method @@ -67,9 +68,15 @@ are cheaply cloneable; insert an `Rc` if necessary). ### Providers If, however, the query is *not* in the cache, then the compiler will -try to find a suitable **provider**. A provider is a function that has -been defined and linked into the compiler somewhere that contains the -code to compute the result of the query. +call the corresponding **provider** function. A provider is a function +implemented in a specific module and **manually registered** into the +[`Providers`][providers_struct] struct during compiler initialization. +The macro system generates the [`Providers`][providers_struct] struct, +which acts as a function table for all query implementations, where each +field is a function pointer to the actual provider. + +**Note:** The `Providers` struct is generated by macros and acts as a function table for all query implementations. +It is **not** a Rust trait, but a plain struct with function pointer fields. **Providers are defined per-crate.** The compiler maintains, internally, a table of providers for every crate, at least @@ -97,7 +104,18 @@ fn provider<'tcx>( Providers take two arguments: the `tcx` and the query key. They return the result of the query. -### How providers are setup +N.B. Most of the `rustc_*` crates only provide **local +providers**. Almost all **extern providers** wind up going through the +[`rustc_metadata` crate][rustc_metadata], which loads the information +from the crate metadata. But in some cases there are crates that +provide queries for *both* local and external crates, in which case +they define both a `provide` and a `provide_extern` function, through +[`wasm_import_module_map`][wasm_import_module_map], that `rustc_driver` can invoke. + +[rustc_metadata]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_metadata/index.html +[wasm_import_module_map]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_codegen_ssa/back/symbol_export/fn.wasm_import_module_map.html + +### How providers are set up When the tcx is created, it is given the providers by its creator using the [`Providers`][providers_struct] struct. This struct is generated by @@ -108,19 +126,16 @@ the macros here, but it is basically a big list of function pointers: ```rust,ignore struct Providers { type_of: for<'tcx> fn(TyCtxt<'tcx>, DefId) -> Ty<'tcx>, - ... + // ... one field for each query } ``` -At present, we have one copy of the struct for local crates, and one -for external crates, though the plan is that we may eventually have -one per crate. +#### How are providers registered? -These `Providers` structs are ultimately created and populated by -`rustc_driver`, but it does this by distributing the work -throughout the other `rustc_*` crates. This is done by invoking -various [`provide`][provide_fn] functions. These functions tend to look -something like this: +The `Providers` struct is filled in during compiler initialization, mainly by the `rustc_driver` crate. +But the actual provider functions are implemented in various `rustc_*` crates (like `rustc_middle`, `rustc_hir_analysis`, etc). + +To register providers, each crate exposes a [`provide`][provide_fn] function that looks like this: [provide_fn]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/hir/fn.provide.html @@ -128,41 +143,34 @@ something like this: pub fn provide(providers: &mut Providers) { *providers = Providers { type_of, + // ... add more providers here ..*providers }; } ``` -That is, they take an `&mut Providers` and mutate it in place. Usually -we use the formulation above just because it looks nice, but you could -as well do `providers.type_of = type_of`, which would be equivalent. -(Here, `type_of` would be a top-level function, defined as we saw -before.) So, if we want to add a provider for some other query, -let's call it `fubar`, into the crate above, we might modify the `provide()` -function like so: +- This function takes a mutable reference to the `Providers` struct and sets the fields to point to the correct provider functions. +- You can also assign fields individually, e.g. `providers.type_of = type_of;`. -```rust,ignore -pub fn provide(providers: &mut Providers) { - *providers = Providers { - type_of, - fubar, - ..*providers - }; -} +#### Adding a new provider -fn fubar<'tcx>(tcx: TyCtxt<'tcx>, key: DefId) -> Fubar<'tcx> { ... } -``` +Suppose you want to add a new query called `fubar`. You would: -N.B. Most of the `rustc_*` crates only provide **local -providers**. Almost all **extern providers** wind up going through the -[`rustc_metadata` crate][rustc_metadata], which loads the information -from the crate metadata. But in some cases there are crates that -provide queries for *both* local and external crates, in which case -they define both a `provide` and a `provide_extern` function, through -[`wasm_import_module_map`][wasm_import_module_map], that `rustc_driver` can invoke. +1. Implement the provider function: + ```rust,ignore + fn fubar<'tcx>(tcx: TyCtxt<'tcx>, key: DefId) -> Fubar<'tcx> { ... } + ``` +2. Register it in the `provide` function: + ```rust,ignore + pub fn provide(providers: &mut Providers) { + *providers = Providers { + fubar, + ..*providers + }; + } + ``` -[rustc_metadata]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_metadata/index.html -[wasm_import_module_map]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_codegen_ssa/back/symbol_export/fn.wasm_import_module_map.html +--- ## Adding a new query diff --git a/src/return-position-impl-trait-in-trait.md b/src/return-position-impl-trait-in-trait.md index 5f358819c..85cece2ac 100644 --- a/src/return-position-impl-trait-in-trait.md +++ b/src/return-position-impl-trait-in-trait.md @@ -356,7 +356,7 @@ trait Foo { Failing because a down-stream impl could theoretically provide an implementation for `RPITIT` without providing an implementation of -`foo`: +`bar`: ```text error[E0308]: mismatched types diff --git a/src/rustc-driver/getting-diagnostics.md b/src/rustc-driver/getting-diagnostics.md index e3ca32305..518cf4e82 100644 --- a/src/rustc-driver/getting-diagnostics.md +++ b/src/rustc-driver/getting-diagnostics.md @@ -7,8 +7,7 @@ otherwise be printed to stderr. To get diagnostics from the compiler, configure [`rustc_interface::Config`] to output diagnostic to a buffer, -and run [`TyCtxt.analysis`]. -The following was tested with `nightly-2024-09-16`: +and run [`rustc_hir_typeck::typeck`] for each item. ```rust {{#include ../../examples/rustc-interface-getting-diagnostics.rs}} @@ -17,3 +16,4 @@ The following was tested with `nightly-2024- [`rustc_interface`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_interface/index.html [`rustc_interface::Config`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_interface/interface/struct.Config.html [`TyCtxt.analysis`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_interface/passes/fn.analysis.html +[`rustc_hir_typeck::typeck`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir_typeck/fn.typeck.html diff --git a/src/rustc-driver/interacting-with-the-ast.md b/src/rustc-driver/interacting-with-the-ast.md index 5eaa0c82c..f46418701 100644 --- a/src/rustc-driver/interacting-with-the-ast.md +++ b/src/rustc-driver/interacting-with-the-ast.md @@ -5,7 +5,6 @@ ## Getting the type of an expression To get the type of an expression, use the [`after_analysis`] callback to get a [`TyCtxt`]. -The following was tested with `nightly-2024-12-15`: ```rust {{#include ../../examples/rustc-driver-interacting-with-the-ast.rs}} diff --git a/src/rustc-driver/intro.md b/src/rustc-driver/intro.md index a6234dc12..a3684397b 100644 --- a/src/rustc-driver/intro.md +++ b/src/rustc-driver/intro.md @@ -6,9 +6,9 @@ The [`rustc_driver`] is essentially `rustc`'s `main` function. It acts as the glue for running the various phases of the compiler in the correct order, using the interface defined in the [`rustc_interface`] crate. Where possible, using [`rustc_driver`] rather than [`rustc_interface`] is recommended. -The main entry point of [`rustc_driver`] is [`rustc_driver::RunCompiler`][rd_rc]. -This builder accepts the same command-line args as rustc as well as an implementation of [`Callbacks`][cb] and a couple of other optional options. -[`Callbacks`][cb] is a `trait` that allows for custom compiler configuration, +The main entry point of [`rustc_driver`] is [`rustc_driver::run_compiler`][rd_rc]. +This builder accepts the same command-line args as rustc as well as an implementation of [`Callbacks`] and a couple of other optional options. +[`Callbacks`] is a `trait` that allows for custom compiler configuration, as well as allowing custom code to run after different phases of the compilation. ## `rustc_interface` @@ -33,14 +33,8 @@ specifically [`rustc_driver_impl::run_compiler`][rdi_rc] [`Compiler`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_interface/interface/struct.Compiler.html [`rustc_driver`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_driver/ [`rustc_interface`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_interface/index.html -[`Session`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_session/struct.Session.html -[`SourceMap`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_span/source_map/struct.SourceMap.html -[`TyCtxt`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/struct.TyCtxt.html -[Appendix A]: appendix/stupid-stats.html -[cb]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_driver/trait.Callbacks.html +[`Callbacks`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_driver/trait.Callbacks.html [example]: https://github.com/rust-lang/rustc-dev-guide/blob/master/examples/rustc-interface-example.rs [i_rc]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_interface/interface/fn.run_compiler.html -[rd_rc]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_driver/struct.RunCompiler.html +[rd_rc]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_driver/fn.run_compiler.html [rdi_rc]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_driver_impl/fn.run_compiler.html -[stupid-stats]: https://github.com/nrc/stupid-stats -[`nightly-rustc`]: https://doc.rust-lang.org/nightly/nightly-rustc/ diff --git a/src/rustc-driver/remarks-on-perma-unstable-features.md b/src/rustc-driver/remarks-on-perma-unstable-features.md new file mode 100644 index 000000000..b434cfc9c --- /dev/null +++ b/src/rustc-driver/remarks-on-perma-unstable-features.md @@ -0,0 +1,54 @@ +# Remarks on perma unstable features + +## `rustc_private` + +### Overview + +The `rustc_private` feature allows external crates to use compiler internals. + +### Using `rustc-private` with Official Toolchains + +When using the `rustc_private` feature with official Rust toolchains distributed via rustup, you need to install two additional components: + +1. **`rustc-dev`**: Provides compiler libraries +2. **`llvm-tools`**: Provides LLVM libraries required for linking + +#### Installation Steps + +Install both components using rustup: + +```text +rustup component add rustc-dev llvm-tools +``` + +#### Common Error + +Without the `llvm-tools` component, you'll encounter linking errors like: + +```text +error: linking with `cc` failed: exit status: 1 + | + = note: rust-lld: error: unable to find library -lLLVM-{version} +``` + +### Using `rustc-private` with Custom Toolchains + +For custom-built toolchains or environments not using rustup, additional configuration is typically required: + +#### Requirements + +- LLVM libraries must be available in your system's library search paths +- The LLVM version must match the one used to build your Rust toolchain + +#### Troubleshooting Steps + +1. **Check LLVM installation**: Verify LLVM is installed and accessible +2. **Configure library paths**: You may need to set environment variables: + ```text + export LD_LIBRARY_PATH=/path/to/llvm/lib:$LD_LIBRARY_PATH + ``` +3. **Check version compatibility**: Ensure your LLVM version is compatible with your Rust toolchain + +### Additional Resources + +- [GitHub Issue #137421](https://github.com/rust-lang/rust/issues/137421): Explains that `rustc_private` linker failures often occur because `llvm-tools` is not installed diff --git a/src/rustdoc-internals.md b/src/rustdoc-internals.md index 7f1c83e00..4affbafe4 100644 --- a/src/rustdoc-internals.md +++ b/src/rustdoc-internals.md @@ -1,7 +1,5 @@ # Rustdoc Internals - - This page describes [`rustdoc`]'s passes and modes. For an overview of `rustdoc`, see the ["Rustdoc overview" chapter](./rustdoc.md). @@ -55,8 +53,8 @@ The first step in [`clean::utils::krate`][ck1] is to invoke * inlining public `use` exports of private items, or showing a "Reexport" line in the module page * inlining items with `#[doc(hidden)]` if the base item is hidden but the - * showing `#[macro_export]`-ed macros at the crate root, regardless of where - they're defined reexport is not + * showing `#[macro_export]`-ed macros at the crate root, regardless of whether + they're defined as a reexport or not After this step, `clean::krate` invokes [`clean_doc_module`], which actually converts the `HIR` items to the cleaned [`AST`][ast]. This is also the step where cross- @@ -270,35 +268,6 @@ in `test.rs` is the function `make_test`, which is where hand-written Some extra reading about `make_test` can be found [here](https://quietmisdreavus.net/code/2018/02/23/how-the-doctests-get-made/). -## Dotting i's And Crossing t's - -So that's `rustdoc`'s code in a nutshell, but there's more things in the -compiler that deal with it. Since we have the full `compiletest` suite at hand, -there's a set of tests in `tests/rustdoc` that make sure the final `HTML` is -what we expect in various situations. These tests also use a supplementary -script, `src/etc/htmldocck.py`, that allows it to look through the final `HTML` -using `XPath` notation to get a precise look at the output. The full -description of all the commands available to `rustdoc` tests (e.g. [`@has`] and -[`@matches`]) is in [`htmldocck.py`]. - -To use multiple crates in a `rustdoc` test, add `// aux-build:filename.rs` -to the top of the test file. `filename.rs` should be placed in an `auxiliary` -directory relative to the test file with the comment. If you need to build -docs for the auxiliary file, use `// build-aux-docs`. - -In addition, there are separate tests for the search index and `rustdoc`'s -ability to query it. The files in `tests/rustdoc-js` each contain a -different search query and the expected results, broken out by search tab. -These files are processed by a script in `src/tools/rustdoc-js` and the `Node.js` -runtime. These tests don't have as thorough of a writeup, but a broad example -that features results in all tabs can be found in `basic.js`. The basic idea is -that you match a given `QUERY` with a set of `EXPECTED` results, complete with -the full item path of each item. - -[`@has`]: https://github.com/rust-lang/rust/blob/master/src/etc/htmldocck.py#L39 -[`@matches`]: https://github.com/rust-lang/rust/blob/master/src/etc/htmldocck.py#L44 -[`htmldocck.py`]: https://github.com/rust-lang/rust/blob/master/src/etc/htmldocck.py - ## Testing Locally Some features of the generated `HTML` documentation might require local diff --git a/src/rustdoc-internals/rustdoc-gui-test-suite.md b/src/rustdoc-internals/rustdoc-gui-test-suite.md new file mode 100644 index 000000000..e155f960e --- /dev/null +++ b/src/rustdoc-internals/rustdoc-gui-test-suite.md @@ -0,0 +1,14 @@ +# The `rustdoc-gui` test suite + +> **FIXME**: This section is a stub. Please help us flesh it out! + +This page is about the test suite named `rustdoc-gui` used to test the "GUI" of `rustdoc` (i.e., the HTML/JS/CSS as rendered in a browser). +For other rustdoc-specific test suites, see [Rustdoc test suites]. + +These use a NodeJS-based tool called [`browser-UI-test`] that uses [puppeteer] to run tests in a headless browser and check rendering and interactivity. For information on how to write this form of test, see [`tests/rustdoc-gui/README.md`][rustdoc-gui-readme] as well as [the description of the `.goml` format][goml-script] + +[Rustdoc test suites]: ../tests/compiletest.md#rustdoc-test-suites +[`browser-UI-test`]: https://github.com/GuillaumeGomez/browser-UI-test/ +[puppeteer]: https://pptr.dev/ +[rustdoc-gui-readme]: https://github.com/rust-lang/rust/blob/master/tests/rustdoc-gui/README.md +[goml-script]: https://github.com/GuillaumeGomez/browser-UI-test/blob/master/goml-script.md diff --git a/src/rustdoc-internals/rustdoc-json-test-suite.md b/src/rustdoc-internals/rustdoc-json-test-suite.md new file mode 100644 index 000000000..e08f77095 --- /dev/null +++ b/src/rustdoc-internals/rustdoc-json-test-suite.md @@ -0,0 +1,3 @@ +# The `rustdoc-json` test suite + +> **FIXME**: This section is a stub. It will be populated by [PR #2422](https://github.com/rust-lang/rustc-dev-guide/pull/2422/). diff --git a/src/rustdoc-internals/rustdoc-test-suite.md b/src/rustdoc-internals/rustdoc-test-suite.md new file mode 100644 index 000000000..3ec5ebd79 --- /dev/null +++ b/src/rustdoc-internals/rustdoc-test-suite.md @@ -0,0 +1,194 @@ +# The `rustdoc` test suite + +This page is about the test suite named `rustdoc` used to test the HTML output of `rustdoc`. +For other rustdoc-specific test suites, see [Rustdoc test suites]. + +Each test file in this test suite is simply a Rust source file `file.rs` sprinkled with +so-called *directives* located inside normal Rust code comments. +These come in two flavors: *Compiletest* and *HtmlDocCk*. + +To learn more about the former, read [Compiletest directives]. +For the latter, continue reading. + +Internally, [`compiletest`] invokes the supplementary checker script [`htmldocck.py`]. + +[Rustdoc test suites]: ../tests/compiletest.md#rustdoc-test-suites +[`compiletest`]: ../tests/compiletest.md +[`htmldocck.py`]: https://github.com/rust-lang/rust/blob/master/src/etc/htmldocck.py + +## HtmlDocCk Directives + +Directives to HtmlDocCk are assertions that place constraints on the generated HTML. +They look similar to those given to `compiletest` in that they take the form of `//@` comments +but ultimately, they are completely distinct and processed by different programs. + +[XPath] is used to query parts of the HTML document tree. + +**Introductory example**: + +```rust,ignore (illustrative) +//@ has file/type.Alias.html +//@ has - '//*[@class="rust item-decl"]//code' 'type Alias = Option;' +pub type Alias = Option; +``` + +Here, we check that documentation generated for crate `file` contains a page for the +public type alias `Alias` where the code block that is found at the top contains the +expected rendering of the item. The `//*[@class="rust item-decl"]//code` is an XPath +expression. + +Conventionally, you place these directives directly above the thing they are meant to test. +Technically speaking however, they don't need to be as HtmlDocCk only looks for the directives. + +All directives take a `PATH` argument. +To avoid repetition, `-` can be passed to it to re-use the previous `PATH` argument. +Since the path contains the name of the crate, it is conventional to add a +`#![crate_name = "foo"]` attribute to the crate root to shorten the resulting path. + +All arguments take the form of shell-style (single or double) quoted strings, +with the exception of `COUNT` and the special `-` form of `PATH`. + +All directives (except `files`) can be *negated* by putting a `!` in front of their name. +Before you add negated directives, please read about [their caveats](#caveats). + +Similar to shell commands, +directives can extend across multiple lines if their last char is `\`. +In this case, the start of the next line should be `//`, with no `@`. + +Similar to compiletest directives, besides a space you can also use a colon `:` to separate +the directive name and the arguments, however a space is preferred for HtmlDocCk directives. + +Use the special string `{{channel}}` in XPaths, `PATTERN` arguments and [snapshot files](#snapshot) +if you'd like to refer to the URL `https://doc.rust-lang.org/CHANNEL` where `CHANNEL` refers to the +current release channel (e.g, `stable` or `nightly`). + +Listed below are all possible directives: + +[XPath]: https://en.wikipedia.org/wiki/XPath + +### `has` + +> Usage 1: `//@ has PATH` + +Check that the file given by `PATH` exists. + +> Usage 2: `//@ has PATH XPATH PATTERN` + +Checks that the text of each element / attribute / text selected by `XPATH` in the +whitespace-normalized[^1] file given by `PATH` matches the +(also whitespace-normalized) string `PATTERN`. + +**Tip**: If you'd like to avoid whitespace normalization and/or if you'd like to match with a regex, +use `matches` instead. + +### `hasraw` + +> Usage: `//@ hasraw PATH PATTERN` + +Checks that the contents of the whitespace-normalized[^1] file given by `PATH` +matches the (also whitespace-normalized) string `PATTERN`. + +**Tip**: If you'd like to avoid whitespace normalization and / or if you'd like to match with a +regex, use `matchesraw` instead. + +### `matches` + +> Usage: `//@ matches PATH XPATH PATTERN` + +Checks that the text of each element / attribute / text selected by `XPATH` in the +file given by `PATH` matches the Python-flavored[^2] regex `PATTERN`. + +### `matchesraw` + +> Usage: `//@ matchesraw PATH PATTERN` + +Checks that the contents of the file given by `PATH` matches the +Python-flavored[^2] regex `PATTERN`. + +### `count` + +> Usage: `//@ count PATH XPATH COUNT` + +Checks that there are exactly `COUNT` matches for `XPATH` within the file given by `PATH`. + +### `snapshot` + +> Usage: `//@ snapshot NAME PATH XPATH` + +Checks that the element / text selected by `XPATH` in the file given by `PATH` matches the +pre-recorded subtree or text (the "snapshot") in file `FILE_STEM.NAME.html` where `FILE_STEM` +is the file stem of the test file. + +Pass the `--bless` option to `compiletest` to accept the current subtree/text as expected. +This will overwrite the aforementioned file (or create it if it doesn't exist). It will +automatically normalize the channel-dependent URL `https://doc.rust-lang.org/CHANNEL` to +the special string `{{channel}}`. + +### `has-dir` + +> Usage: `//@ has-dir PATH` + +Checks for the existence of the directory given by `PATH`. + +### `files` + +> Usage: `//@ files PATH ENTRIES` + +Checks that the directory given by `PATH` contains exactly `ENTRIES`. +`ENTRIES` is a Python-like list of strings inside a quoted string. + +**Example**: `//@ files "foo/bar" '["index.html", "sidebar-items.js"]'` + +[^1]: Whitespace normalization means that all spans of consecutive whitespace are replaced with a single space. +[^2]: They are Unicode aware (flag `UNICODE` is set), match case-sensitively and in single-line mode. + +## Compiletest Directives (Brief) + +As mentioned in the introduction, you also have access to [compiletest directives]. +Most importantly, they allow you to register auxiliary crates and +to pass flags to the `rustdoc` binary under test. +It's *strongly recommended* to read that chapter if you don't know anything about them yet. + +Here are some details that are relevant to this test suite specifically: + +* While you can use both `//@ compile-flags` and `//@ doc-flags` to pass flags to `rustdoc`, + prefer to user the latter to show intent. The former is meant for `rustc`. +* Add `//@ build-aux-docs` to the test file that has auxiliary crates to not only compile the + auxiliaries with `rustc` but to also document them with `rustdoc`. + +## Caveats + +Testing for the absence of an element or a piece of text is quite fragile and not very future proof. + +It's not unusual that the *shape* of the generated HTML document tree changes from time to time. +This includes for example renamings of CSS classes. + +Whenever that happens, *positive* checks will either continue to match the intended element / +attribute / text (if their XPath expression is general / loose enough) and +thus continue to test the correct thing or they won't in which case they would fail thereby +forcing the author of the change to look at them. + +Compare that to *negative* checks (e.g., `//@ !has PATH XPATH PATTERN`) which won't fail if their +XPath expression "no longer" matches. The author who changed "the shape" thus won't get notified and +as a result someone else can unintentionally reintroduce `PATTERN` into the generated docs without +the original negative check failing. + +**Note**: Please avoid the use of *negated* checks! + +**Tip**: If you can't avoid it, please **always** pair it with an analogous positive check in the +immediate vicinity, so people changing "the shape" have a chance to notice and to update the +negated check! + +## Limitations + +HtmlDocCk uses the XPath implementation from the Python standard library. +This leads to several limitations: + +* All `XPATH` arguments must start with `//` due to a flaw in the implementation. +* Many XPath features (functions, axies, etc.) are not supported. +* Only well-formed HTML can be parsed (hopefully rustdoc doesn't output mismatched tags). + +Furthmore, compiletest [revisions] are not supported. + +[revisions]: ../tests/compiletest.md#revisions +[compiletest directives]: ../tests/directives.md diff --git a/src/rustdoc-internals/search.md b/src/rustdoc-internals/search.md index ddf8ec405..beff0a94c 100644 --- a/src/rustdoc-internals/search.md +++ b/src/rustdoc-internals/search.md @@ -7,8 +7,6 @@ in the crates in the doc bundle, and the second reads it, turns it into some in-memory structures, and scans them linearly to search. - - ## Search index format `search.js` calls this Raw, because it turns it into @@ -46,8 +44,8 @@ For space savings, it's also written without newlines or spaces. ] ``` -[`src/librustdoc/html/static/js/externs.js`] -defines an actual schema in a Closure `@typedef`. +[`src/librustdoc/html/static/js/rustdoc.d.ts`] +defines an actual schema in a TypeScript `type`. | Key | Name | Description | | --- | -------------------- | ------------ | @@ -68,7 +66,7 @@ with a free function called `function_name` and a struct called `Data`, with the type signature `Data, i32 -> str`, and an alias, `get_name`, that equivalently refers to `function_name`. -[`src/librustdoc/html/static/js/externs.js`]: https://github.com/rust-lang/rust/blob/79b710c13968a1a48d94431d024d2b1677940866/src/librustdoc/html/static/js/externs.js#L204-L258 +[`src/librustdoc/html/static/js/rustdoc.d.ts`]: https://github.com/rust-lang/rust/blob/2f92f050e83bf3312ce4ba73c31fe843ad3cbc60/src/librustdoc/html/static/js/rustdoc.d.ts#L344-L390 The search index needs to fit the needs of the `rustdoc` compiler, the `search.js` frontend, @@ -469,7 +467,7 @@ want the libs team to be able to add new items without causing unrelated tests to fail, but standalone tests will use it more often. The `ResultsTable` and `ParsedQuery` types are specified in -[`externs.js`](https://github.com/rust-lang/rust/blob/master/src/librustdoc/html/static/js/externs.js). +[`rustdoc.d.ts`](https://github.com/rust-lang/rust/blob/master/src/librustdoc/html/static/js/rustdoc.d.ts). For example, imagine we needed to fix a bug where a function named `constructor` couldn't be found. To do this, write two files: diff --git a/src/rustdoc.md b/src/rustdoc.md index 3867d2489..9290fcd3b 100644 --- a/src/rustdoc.md +++ b/src/rustdoc.md @@ -9,8 +9,6 @@ For more details about how rustdoc works, see the [Rustdoc internals]: ./rustdoc-internals.md - - `rustdoc` uses `rustc` internals (and, of course, the standard library), so you will have to build the compiler and `std` once before you can build `rustdoc`. @@ -58,39 +56,38 @@ does is call the `main()` that's in this crate's `lib.rs`, though.) * If you want to copy those docs to a webserver, copy all of `build/host/doc`, since that's where the CSS, JS, fonts, and landing page are. + * For frontend debugging, disable the `rust.docs-minification` option in [`bootstrap.toml`]. * Use `./x test tests/rustdoc*` to run the tests using a stage1 rustdoc. * See [Rustdoc internals] for more information about tests. +[`bootstrap.toml`]: ./building/how-to-build-and-run.md + ## Code structure -* All paths in this section are relative to `src/librustdoc` in the rust-lang/rust repository. +All paths in this section are relative to `src/librustdoc/` in the rust-lang/rust repository. + * Most of the HTML printing code is in `html/format.rs` and `html/render/mod.rs`. - It's in a bunch of `fmt::Display` implementations and supplementary - functions. -* The types that got `Display` impls above are defined in `clean/mod.rs`, right - next to the custom `Clean` trait used to process them out of the rustc HIR. + It's in a bunch of functions returning `impl std::fmt::Display`. +* The data types that get rendered by the functions mentioned above are defined in `clean/types.rs`. + The functions responsible for creating them from the `HIR` and the `rustc_middle::ty` IR + live in `clean/mod.rs`. * The bits specific to using rustdoc as a test harness are in `doctest.rs`. * The Markdown renderer is loaded up in `html/markdown.rs`, including functions for extracting doctests from a given block of Markdown. -* The tests on the structure of rustdoc HTML output are located in `tests/rustdoc`, where - they're handled by the test runner of bootstrap and the supplementary script - `src/etc/htmldocck.py`. +* Frontend CSS and JavaScript are stored in `html/static/`. + * Re. JavaScript, type annotations are written using [TypeScript-flavored JSDoc] +comments and an external `.d.ts` file. + This way, the code itself remains plain, valid JavaScript. + We only use `tsc` as a linter. + +[TypeScript-flavored JSDoc]: https://www.typescriptlang.org/docs/handbook/jsdoc-supported-types.html ## Tests -* All paths in this section are relative to `tests` in the rust-lang/rust repository. -* Tests on search engine and index are located in `rustdoc-js` and `rustdoc-js-std`. - The format is specified - [in the search guide](rustdoc-internals/search.md#testing-the-search-engine). -* Tests on the "UI" of rustdoc (the terminal output it produces when run) are in - `rustdoc-ui` -* Tests on the "GUI" of rustdoc (the HTML, JS, and CSS as rendered in a browser) - are in `rustdoc-gui`. These use a [NodeJS tool called - browser-UI-test](https://github.com/GuillaumeGomez/browser-UI-test/) that uses - puppeteer to run tests in a headless browser and check rendering and - interactivity. +`rustdoc`'s integration tests are split across several test suites. +See [Rustdoc tests suites](tests/compiletest.md#rustdoc-test-suites) for more details. ## Constraints @@ -103,6 +100,28 @@ Certain browser features that require secure origins, like `localStorage` and Service Workers, don't work reliably. We can still use such features but we should make sure pages are still usable without them. +Rustdoc [does not type-check function bodies][platform-specific docs]. +This works by [overriding the built-in queries for typeck][override queries], +by [silencing name resolution errors], and by [not resolving opaque types]. +This comes with several caveats: in particular, rustdoc *cannot* run any parts of the compiler that +require type-checking bodies; for example it cannot generate `.rlib` files or run most lints. +We want to move away from this model eventually, but we need some alternative for +[the people using it][async-std]; see [various][zulip stop accepting broken code] +[previous][rustdoc meeting 2024-07-08] [zulip][compiler meeting 2023-01-26] [discussion][notriddle rfc]. +For examples of code that breaks if this hack is removed, see +[`tests/rustdoc-ui/error-in-impl-trait`]. + +[platform-specific docs]: https://doc.rust-lang.org/rustdoc/advanced-features.html#interactions-between-platform-specific-docs +[override queries]: https://github.com/rust-lang/rust/blob/52bf0cf795dfecc8b929ebb1c1e2545c3f41d4c9/src/librustdoc/core.rs#L299-L323 +[silencing name resolution errors]: https://github.com/rust-lang/rust/blob/52bf0cf795dfecc8b929ebb1c1e2545c3f41d4c9/compiler/rustc_resolve/src/late.rs#L4517 +[not resolving opaque types]: https://github.com/rust-lang/rust/blob/52bf0cf795dfecc8b929ebb1c1e2545c3f41d4c9/compiler/rustc_hir_analysis/src/check/check.rs#L188-L194 +[async-std]: https://github.com/rust-lang/rust/issues/75100 +[rustdoc meeting 2024-07-08]: https://rust-lang.zulipchat.com/#narrow/channel/393423-t-rustdoc.2Fmeetings/topic/meeting.202024-07-08/near/449969836 +[compiler meeting 2023-01-26]: https://rust-lang.zulipchat.com/#narrow/channel/238009-t-compiler.2Fmeetings/topic/.5Bweekly.5D.202023-01-26/near/323755789 +[zulip stop accepting broken code]: https://rust-lang.zulipchat.com/#narrow/stream/266220-rustdoc/topic/stop.20accepting.20broken.20code +[notriddle rfc]: https://rust-lang.zulipchat.com/#narrow/channel/266220-t-rustdoc/topic/Pre-RFC.3A.20stop.20accepting.20broken.20code +[`tests/rustdoc-ui/error-in-impl-trait`]: https://github.com/rust-lang/rust/tree/163cb4ea3f0ae3bc7921cc259a08a7bf92e73ee6/tests/rustdoc-ui/error-in-impl-trait + ## Multiple runs, same output directory Rustdoc can be run multiple times for varying inputs, with its output set to the diff --git a/src/sanitizers.md b/src/sanitizers.md index 73e888eae..29d9056c1 100644 --- a/src/sanitizers.md +++ b/src/sanitizers.md @@ -1,4 +1,4 @@ -# Sanitizers Support +# Sanitizers support The rustc compiler contains support for following sanitizers: @@ -32,7 +32,7 @@ implementation: * The sanitizer runtime libraries are part of the [compiler-rt] project, and [will be built][sanitizer-build] on [supported targets][sanitizer-targets] - when enabled in `config.toml`: + when enabled in `bootstrap.toml`: ```toml [build] @@ -76,16 +76,16 @@ implementation: ## Testing sanitizers Sanitizers are validated by code generation tests in -[`tests/codegen/sanitize*.rs`][test-cg] and end-to-end functional tests in +[`tests/codegen-llvm/sanitize*.rs`][test-cg] and end-to-end functional tests in [`tests/ui/sanitizer/`][test-ui] directory. Testing sanitizer functionality requires the sanitizer runtimes (built when -`sanitizer = true` in `config.toml`) and target providing support for particular +`sanitizer = true` in `bootstrap.toml`) and target providing support for particular sanitizer. When sanitizer is unsupported on given target, sanitizers tests will be ignored. This behaviour is controlled by compiletest `needs-sanitizer-*` directives. -[test-cg]: https://github.com/rust-lang/rust/tree/master/tests/codegen +[test-cg]: https://github.com/rust-lang/rust/tree/master/tests/codegen-llvm [test-ui]: https://github.com/rust-lang/rust/tree/master/tests/ui/sanitizer ## Enabling sanitizer on a new target diff --git a/src/serialization.md b/src/serialization.md index 0ff049901..8eb37bbe2 100644 --- a/src/serialization.md +++ b/src/serialization.md @@ -1,4 +1,4 @@ -# Serialization in Rustc +# Serialization in rustc rustc has to [serialize] and deserialize various data during compilation. Specifically: @@ -75,7 +75,7 @@ impl Decodable for MyStruct { rustc has a lot of [arena allocated types]. Deserializing these types isn't possible without access to the arena that they need to be allocated on. -The [`TyDecoder`] and [`TyEncoder`] traits are supertraits of [`Decoder`] and [`Encoder`] that allow access to a [`TyCtxt`]. +The [`TyDecoder`] and [`TyEncoder`] traits are subtraits of [`Decoder`] and [`Encoder`] that allow access to a [`TyCtxt`]. Types which contain `arena` allocated types can then bound the type parameter of their [`Encodable`] and [`Decodable`] implementations with these traits. @@ -169,7 +169,7 @@ The `LazyArray<[T]>` and `LazyTable` types provide some functionality over than the one being read. **note**: `LazyValue` does not cache its value after being deserialized the -first time. Instead the query system its self is the main way of caching these +first time. Instead the query system itself is the main way of caching these results. [`LazyArray`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_metadata/rmeta/struct.LazyValue.html diff --git a/src/solve/caching.md b/src/solve/caching.md index 72b90c59b..e568d47ca 100644 --- a/src/solve/caching.md +++ b/src/solve/caching.md @@ -98,8 +98,8 @@ TODO: write this :3 [req-depth-ck]: https://github.com/rust-lang/rust/blob/7606c13961ddc1174b70638e934df0439b7dc515/compiler/rustc_middle/src/traits/solve/cache.rs#L76-L86 [update-depth]: https://github.com/rust-lang/rust/blob/7606c13961ddc1174b70638e934df0439b7dc515/compiler/rustc_trait_selection/src/solve/search_graph.rs#L308 [rem-depth]: https://github.com/rust-lang/rust/blob/7606c13961ddc1174b70638e934df0439b7dc515/compiler/rustc_middle/src/traits/solve/cache.rs#L124 -[^1]: This is overly restrictive: if all nested goal return the overflow response with some -availabledepth `n`, then their result should be the same for any depths smaller than `n`. +[^1]: This is overly restrictive: if all nested goals return the overflow response with some +available depth `n`, then their result should be the same for any depths smaller than `n`. We can implement this optimization in the future. [chapter on coinduction]: ./coinduction.md diff --git a/src/solve/candidate-preference.md b/src/solve/candidate-preference.md new file mode 100644 index 000000000..896052947 --- /dev/null +++ b/src/solve/candidate-preference.md @@ -0,0 +1,427 @@ +# Candidate preference + +There are multiple ways to prove `Trait` and `NormalizesTo` goals. Each such option is called a [`Candidate`]. If there are multiple applicable candidates, we prefer some candidates over others. We store the relevant information in their [`CandidateSource`]. + +This preference may result in incorrect inference or region constraints and would therefore be unsound during coherence. Because of this, we simply try to merge all candidates in coherence. + +## `Trait` goals + +Trait goals merge their applicable candidates in [`fn merge_trait_candidates`]. This document provides additional details and references to explain *why* we've got the current preference rules. + +### `CandidateSource::BuiltinImpl(BuiltinImplSource::Trivial))` + +Trivial builtin impls are builtin impls which are known to be always applicable for well-formed types. This means that if one exists, using another candidate should never have fewer constraints. We currently only consider `Sized` - and `MetaSized` - impls to be trivial. + +This is necessary to prevent a lifetime error for the following pattern + +```rust +trait Trait: Sized {} +impl<'a> Trait for &'a str {} +impl<'a> Trait for &'a str {} +fn is_sized(_: T) {} +fn foo<'a, 'b, T>(x: &'b str) +where + &'a str: Trait, +{ + // Elaborating the `&'a str: Trait` where-bound results in a + // `&'a str: Sized` where-bound. We do not want to prefer this + // over the builtin impl. + is_sized(x); +} +``` + +This preference is incorrect in case the builtin impl has a nested goal which relies on a non-param where-clause +```rust +struct MyType<'a, T: ?Sized>(&'a (), T); +fn is_sized() {} +fn foo<'a, T: ?Sized>() +where + (MyType<'a, T>,): Sized, + MyType<'static, T>: Sized, +{ + // The where-bound is trivial while the builtin `Sized` impl for tuples + // requires proving `MyType<'a, T>: Sized` which can only be proven by + // using the where-clause, adding an unnecessary `'static` constraint. + is_sized::<(MyType<'a, T>,)>(); + //~^ ERROR lifetime may not live long enough +} +``` + +### `CandidateSource::ParamEnv` + +Once there's at least one *non-global* `ParamEnv` candidate, we prefer *all* `ParamEnv` candidates over other candidate kinds. +A where-bound is global if it is not higher-ranked and doesn't contain any generic parameters. It may contain `'static`. + +We try to apply where-bounds over other candidates as users tends to have the most control over them, so they can most easily +adjust them in case our candidate preference is incorrect. + +#### Preference over `Impl` candidates + +This is necessary to avoid region errors in the following example + +```rust +trait Trait<'a> {} +impl Trait<'static> for T {} +fn impls_trait<'a, T: Trait<'a>>() {} +fn foo<'a, T: Trait<'a>>() { + impls_trait::<'a, T>(); +} +``` + +We also need this as shadowed impls can result in currently ambiguous solver cycles: [trait-system-refactor-initiative#76]. Without preference we'd be forced to fail with ambiguity +errors if the where-bound results in region constraints to avoid incompleteness. +```rust +trait Super { + type SuperAssoc; +} + +trait Trait: Super { + type TraitAssoc; +} + +impl Trait for T +where + T: Super, +{ + type TraitAssoc = U; +} + +fn overflow() { + // We can use the elaborated `Super` where-bound + // to prove the where-bound of the `T: Trait` implementation. This currently results in + // overflow. + let x: ::TraitAssoc; +} +``` + +This preference causes a lot of issues. See [#24066]. Most of the +issues are caused by prefering where-bounds over impls even if the where-bound guides type inference: +```rust +trait Trait { + fn call_me(&self, x: T) {} +} +impl Trait for T {} +impl Trait for T {} +fn bug, U>(x: T) { + x.call_me(1u32); + //~^ ERROR mismatched types +} +``` +However, even if we only apply this preference if the where-bound doesn't guide inference, it may still result +in incorrect lifetime constraints: +```rust +trait Trait<'a> {} +impl<'a> Trait<'a> for &'a str {} +fn impls_trait<'a, T: Trait<'a>>(_: T) {} +fn foo<'a, 'b>(x: &'b str) +where + &'a str: Trait<'b> +{ + // Need to prove `&'x str: Trait<'b>` with `'b: 'x`. + impls_trait::<'b, _>(x); + //~^ ERROR lifetime may not live long enough +} +``` + +#### Preference over `AliasBound` candidates + +This is necessary to avoid region errors in the following example +```rust +trait Bound<'a> {} +trait Trait<'a> { + type Assoc: Bound<'a>; +} + +fn impls_bound<'b, T: Bound<'b>>() {} +fn foo<'a, 'b, 'c, T>() +where + T: Trait<'a>, + for<'hr> T::Assoc: Bound<'hr>, +{ + impls_bound::<'b, T::Assoc>(); + impls_bound::<'c, T::Assoc>(); +} +``` +It can also result in unnecessary constraints +```rust +trait Bound<'a> {} +trait Trait<'a> { + type Assoc: Bound<'a>; +} + +fn impls_bound<'b, T: Bound<'b>>() {} +fn foo<'a, 'b, T>() +where + T: for<'hr> Trait<'hr>, + >::Assoc: Bound<'a>, +{ + // Using the where-bound for `>::Assoc: Bound<'a>` + // unnecessarily equates `>::Assoc` with the + // `>::Assoc` from the env. + impls_bound::<'a, >::Assoc>(); + // For a `>::Assoc: Bound<'b>` the self type of the + // where-bound matches, but the arguments of the trait bound don't. + impls_bound::<'b, >::Assoc>(); +} +``` + +#### Why no preference for global where-bounds + +Global where-bounds are either fully implied by an impl or unsatisfiable. If they are unsatisfiable, we don't really care what happens. If a where-bound is fully implied then using the impl to prove the trait goal cannot result in additional constraints. For trait goals this is only useful for where-bounds which use `'static`: + +```rust +trait A { + fn test(&self); +} + +fn foo(x: &dyn A) +where + dyn A + 'static: A, // Using this bound would lead to a lifetime error. +{ + x.test(); +} +``` +More importantly, by using impls here we prevent global where-bounds from shadowing impls when normalizing associated types. There are no known issues from preferring impls over global where-bounds. + +#### Why still consider global where-bounds + +Given that we just use impls even if there exists a global where-bounds, you may ask why we don't just ignore these global where-bounds entirely: we use them to weaken the inference guidance from non-global where-bounds. + +Without a global where-bound, we currently prefer non-global where bounds even though there would be an applicable impl as well. By adding a non-global where-bound, this unnecessary inference guidance is disabled, allowing the following to compile: +```rust +fn check(color: Color) +where + Vec: Into + Into, +{ + let _: f32 = Vec.into(); + // Without the global `Vec: Into` bound we'd + // eagerly use the non-global `Vec: Into` bound + // here, causing this to fail. +} + +struct Vec; +impl From for f32 { + fn from(_: Vec) -> Self { + loop {} + } +} +``` + +### `CandidateSource::AliasBound` + +We prefer alias-bound candidates over impls. We currently use this preference to guide type inference, causing the following to compile. I personally don't think this preference is desirable 🤷 +```rust +pub trait Dyn { + type Word: Into; + fn d_tag(&self) -> Self::Word; + fn tag32(&self) -> Option { + self.d_tag().into().try_into().ok() + // prove `Self::Word: Into` and then select a method + // on `?0`, needs eager inference. + } +} +``` +```rust +fn impl_trait() -> impl Into { + 0u16 +} + +fn main() { + // There are two possible types for `x`: + // - `u32` by using the "alias bound" of `impl Into` + // - `impl Into`, i.e. `u16`, by using `impl From for T` + // + // We infer the type of `x` to be `u32` even though this is not + // strictly necessary and can even lead to surprising errors. + let x = impl_trait().into(); + println!("{}", std::mem::size_of_val(&x)); +} +``` +This preference also avoids ambiguity due to region constraints, I don't know whether people rely on this in practice. +```rust +trait Bound<'a> {} +impl Bound<'static> for T {} +trait Trait<'a> { + type Assoc: Bound<'a>; +} + +fn impls_bound<'b, T: Bound<'b>>() {} +fn foo<'a, T: Trait<'a>>() { + // Should we infer this to `'a` or `'static`. + impls_bound::<'_, T::Assoc>(); +} +``` + +### `CandidateSource::BuiltinImpl(BuiltinImplSource::Object(_))` + +We prefer builtin trait object impls over user-written impls. This is **unsound** and should be remoed in the future. See [#57893](https://github.com/rust-lang/rust/issues/57893) and [#141347](https://github.com/rust-lang/rust/pull/141347) for more details. + +## `NormalizesTo` goals + +The candidate preference behavior during normalization is implemented in [`fn assemble_and_merge_candidates`]. + +### Where-bounds shadow impls + +Normalization of associated items does not consider impls if the corresponding trait goal has been proven via a `ParamEnv` or `AliasBound` candidate. +This means that for where-bounds which do not constrain associated types, the associated types remain *rigid*. + +This is necessary to avoid unnecessary region constraints from applying impls. +```rust +trait Trait<'a> { + type Assoc; +} +impl Trait<'static> for u32 { + type Assoc = u32; +} + +fn bar<'b, T: Trait<'b>>() -> T::Assoc { todo!() } +fn foo<'a>() +where + u32: Trait<'a>, +{ + // Normalizing the return type would use the impl, proving + // the `T: Trait` where-bound would use the where-bound, resulting + // in different region constraints. + bar::<'_, u32>(); +} +``` + +### We always consider `AliasBound` candidates + +In case the where-bound does not specify the associated item, we consider `AliasBound` candidates instead of treating the alias as rigid, even though the trait goal was proven via a `ParamEnv` candidate. + +```rust +trait Super { + type Assoc; +} +trait Bound { + type Assoc: Super; +} +trait Trait: Super {} + +// Elaborating the environment results in a `T::Assoc: Super` where-bound. +// This where-bound must not prevent normalization via the `Super` +// item bound. +fn heck>(x: ::Assoc) -> u32 { + x +} +``` +Using such an alias can result in additional region constraints, cc [#133044]. +```rust +trait Bound<'a> { + type Assoc; +} +trait Trait { + type Assoc: Bound<'static, Assoc = u32>; +} + +fn heck<'a, T: Trait>>(x: >::Assoc) { + // Normalizing the associated type requires `T::Assoc: Bound<'static>` as it + // uses the `Bound<'static>` alias-bound instead of keeping the alias rigid. + drop(x); +} +``` + +### We prefer `ParamEnv` candidates over `AliasBound` + +While we use `AliasBound` candidates if the where-bound does not specify the associated type, in case it does, we prefer the where-bound. +This is necessary for the following example: +```rust +// Make sure we prefer the `I::IntoIterator: Iterator` +// where-bound over the `I::Intoiterator: Iterator` +// alias-bound. + +trait Iterator { + type Item; +} + +trait IntoIterator { + type Item; + type IntoIter: Iterator; +} + +fn normalize>() {} + +fn foo() +where + I: IntoIterator, + I::IntoIter: Iterator, +{ + // We need to prefer the `I::IntoIterator: Iterator` + // where-bound over the `I::Intoiterator: Iterator` + // alias-bound. + normalize::(); +} +``` + +### We always consider where-bounds + +Even if the trait goal was proven via an impl, we still prefer `ParamEnv` candidates, if any exist. + +#### We prefer "orphaned" where-bounds + +We add "orphaned" `Projection` clauses into the `ParamEnv` when normalizing item bounds of GATs and RPITIT in `fn check_type_bounds`. +We need to prefer these `ParamEnv` candidates over impls and other where-bounds. +```rust +#![feature(associated_type_defaults)] +trait Foo { + // We should be able to prove that `i32: Baz` because of + // the impl below, which requires that `Self::Bar<()>: Eq` + // which is true, because we assume `for Self::Bar = i32`. + type Bar: Baz = i32; +} +trait Baz {} +impl Baz for i32 where T::Bar<()>: Eq {} +trait Eq {} +impl Eq for T {} +``` + +I don't fully understand the cases where this preference is actually necessary and haven't been able to exploit this in fun ways yet, but 🤷 + +#### We prefer global where-bounds over impls + +This is necessary for the following to compile. I don't know whether anything relies on it in practice 🤷 +```rust +trait Id { + type This; +} +impl Id for T { + type This = T; +} + +fn foo(x: T) -> ::This +where + u32: Id, +{ + x +} +``` +This means normalization can result in additional region constraints, cc [#133044]. +```rust +trait Trait { + type Assoc; +} + +impl Trait for &u32 { + type Assoc = u32; +} + +fn trait_bound() {} +fn normalize>() {} + +fn foo<'a>() +where + &'static u32: Trait, +{ + trait_bound::<&'a u32>(); // ok, proven via impl + normalize::<&'a u32>(); // error, proven via where-bound +} +``` + +[`Candidate`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_next_trait_solver/solve/assembly/struct.Candidate.html +[`CandidateSource`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_next_trait_solver/solve/enum.CandidateSource.html +[`fn merge_trait_candidates`]: https://github.com/rust-lang/rust/blob/e3ee7f7aea5b45af3b42b5e4713da43876a65ac9/compiler/rustc_next_trait_solver/src/solve/trait_goals.rs#L1342-L1424 +[`fn assemble_and_merge_candidates`]: https://github.com/rust-lang/rust/blob/e3ee7f7aea5b45af3b42b5e4713da43876a65ac9/compiler/rustc_next_trait_solver/src/solve/assembly/mod.rs#L920-L1003 +[trait-system-refactor-initiative#76]: https://github.com/rust-lang/trait-system-refactor-initiative/issues/76 +[#24066]: https://github.com/rust-lang/rust/issues/24066 +[#133044]: https://github.com/rust-lang/rust/issues/133044 \ No newline at end of file diff --git a/src/solve/coinduction.md b/src/solve/coinduction.md index c682e002d..9753f7539 100644 --- a/src/solve/coinduction.md +++ b/src/solve/coinduction.md @@ -237,14 +237,14 @@ Alternatively, we could simply always treat the equate branch of `normalizes_to` Any cycles should result in infinite types, which aren't supported anyways and would only result in overflow when deeply normalizing for codegen. -experimentation and examples: https://hackmd.io/-8p0AHnzSq2VAE6HE_wX-w?view +experimentation and examples: Another attempt at a summary. - in projection eq, we must make progress with constraining the rhs - a cycle is only ok if while equating we have a rigid ty on the lhs after norm at least once - cycles outside of the recursive `eq` call of `normalizes_to` are always fine -[^1]: related: https://coq.inria.fr/refman/language/core/coinductive.html#top-level-definitions-of-corecursive-functions +[^1]: related: [perfect derive]: https://smallcultfollowing.com/babysteps/blog/2022/04/12/implied-bounds-and-perfect-derive [ex1]: https://play.rust-lang.org/?version=stable&mode=debug&edition=2021&gist=0a9c3830b93a2380e6978d6328df8f72 diff --git a/src/solve/invariants.md b/src/solve/invariants.md index c16a3aeb2..8ec15f339 100644 --- a/src/solve/invariants.md +++ b/src/solve/invariants.md @@ -4,17 +4,15 @@ FIXME: This file talks about invariants of the type system as a whole, not only There are a lot of invariants - things the type system guarantees to be true at all times - which are desirable or expected from other languages and type systems. Unfortunately, quite -a few of them do not hold in Rust right now. This is either a fundamental to its design or -caused by bugs and something that may change in the future. +a few of them do not hold in Rust right now. This is either fundamental to its design or +caused by bugs, and something that may change in the future. -It is important to know about the things you can assume while working on - and with - the +It is important to know about the things you can assume while working on, and with, the type system, so here's an incomplete and unofficial list of invariants of the core type system: -- ✅: this invariant mostly holds, with some weird exceptions, you can rely on it outside -of these cases -- ❌: this invariant does not hold, either due to bugs or by design, you must not rely on -it for soundness or have to be incredibly careful when doing so +- ✅: this invariant mostly holds, with some weird exceptions or current bugs +- ❌: this invariant does not hold, and is unlikely to do so in the future; do not rely on it for soundness or have to be incredibly careful when doing so ### `wf(X)` implies `wf(normalize(X))` ✅ @@ -23,8 +21,7 @@ well-formed after normalizing said aliases. We rely on this as otherwise we would have to re-check for well-formedness for these types. -This is unfortunately broken for `>::Output` due to implied bounds, -resulting in [#114936]. +This currently does not hold due to a type system unsoundness: [#84533](https://github.com/rust-lang/rust/issues/84533). ### Structural equality modulo regions implies semantic equality ✅ @@ -32,14 +29,15 @@ If you have a some type and equate it to itself after replacing any regions with inference variables in both the lhs and rhs, the now potentially structurally different types should still be equal to each other. -Needed to prevent goals from succeeding in HIR typeck and then failing in MIR borrowck. -If this does invariant is broken MIR typeck ends up failing with an ICE. +This is needed to prevent goals from succeeding in HIR typeck and then failing in MIR borrowck. +If this invariant is broken, MIR typeck ends up failing with an ICE. ### Applying inference results from a goal does not change its result ❌ TODO: this invariant is formulated in a weird way and needs to be elaborated. Pretty much: I would like this check to only fail if there's a solver bug: -https://github.com/rust-lang/rust/blob/2ffeb4636b4ae376f716dc4378a7efb37632dc2d/compiler/rustc_trait_selection/src/solve/eval_ctxt.rs#L391-L407 +. +We should readd this check and see where it breaks :3 If we prove some goal/equate types/whatever, apply the resulting inference constraints, and then redo the original action, the result should be the same. @@ -76,82 +74,99 @@ Many of the currently known unsound issues end up relying on this invariant bein It is however very difficult to imagine a sound type system without this invariant, so the issue is that the invariant is broken, not that we incorrectly rely on it. -### Generic goals and their instantiations have the same result ✅ +### The type system is complete ❌ + +The type system is not complete. +It often adds unnecessary inference constraints, and errors even though the goal could hold. + +- method selection +- opaque type inference +- handling type outlives constraints +- preferring `ParamEnv` candidates over `Impl` candidates during candidate selection +in the trait solver + +### Goals keep their result from HIR typeck afterwards ✅ + +Having a goal which succeeds during HIR typeck but fails when being reevaluated during MIR borrowck causes ICE, e.g. [#140211](https://github.com/rust-lang/rust/issues/140211). + +Having a goal which succeeds during HIR typeck but fails after being instantiated is unsound, e.g. [#140212](https://github.com/rust-lang/rust/issues/140212). -Pretty much: If we successfully typecheck a generic function concrete instantiations -of that function should also typeck. We should not get errors post-monomorphization. -We can however get overflow errors at that point. +It is interesting that we allow some incompleteness in the trait solver while still maintaining this limitation. It would be nice if there was a clear way to separate the "allowed incompleteness" from behavior which would break this invariant. -TODO: example for overflow error post-monomorphization +#### Normalization must not change results This invariant is relied on to allow the normalization of generic aliases. Breaking -it can easily result in unsoundness, e.g. [#57893](https://github.com/rust-lang/rust/issues/57893) +it can easily result in unsoundness, e.g. [#57893](https://github.com/rust-lang/rust/issues/57893). + +#### Goals may still overflow after instantiation + +This happens they start to hit the recursion limit. +We also have diverging aliases which are scuffed. +It's unclear how these should be handled :3 ### Trait goals in empty environments are proven by a unique impl ✅ If a trait goal holds with an empty environment, there should be a unique `impl`, either user-defined or builtin, which is used to prove that goal. This is -necessary to select a unique method. It +necessary to select unique methods and associated items. -We do however break this invariant in few cases, some of which are due to bugs, -some by design: +We do however break this invariant in a few cases, some of which are due to bugs, some by design: - *marker traits* are allowed to overlap as they do not have associated items - *specialization* allows specializing impls to overlap with their parent - the builtin trait object trait implementation can overlap with a user-defined impl: -[#57893] +[#57893](https://github.com/rust-lang/rust/issues/57893) -### The type system is complete ❌ - -The type system is not complete, it often adds unnecessary inference constraints, and errors -even though the goal could hold. - -- method selection -- opaque type inference -- handling type outlives constraints -- preferring `ParamEnv` candidates over `Impl` candidates during candidate selection -in the trait solver #### The type system is complete during the implicit negative overlap check in coherence ✅ -For more on overlap checking: [coherence] +For more on overlap checking, see [Coherence chapter]. -During the implicit negative overlap check in coherence we must never return *error* for -goals which can be proven. This would allow for overlapping impls with potentially different -associated items, breaking a bunch of other invariants. +During the implicit negative overlap check in coherence, +we must never return *error* for goals which can be proven. +This would allow for overlapping impls with potentially different associated items, +breaking a bunch of other invariants. This invariant is currently broken in many different ways while actually something we rely on. We have to be careful as it is quite easy to break: - generalization of aliases - generalization during subtyping binders (luckily not exploitable in coherence) -### Trait solving must be (free) lifetime agnostic ✅ +### Trait solving must not depend on lifetimes being different ✅ -Trait solving during codegen should have the same result as during typeck. As we erase -all free regions during codegen we must not rely on them during typeck. A noteworthy example -is special behavior for `'static`. +If a goal holds with lifetimes being different, it must also hold with these lifetimes being the same. We otherwise get post-monomorphization errors during codegen or unsoundness due to invalid vtables. + +We could also just get inconsistent behavior when first proving a goal with different lifetimes which are later constrained to be equal. + +### Trait solving in bodies must not depend on lifetimes being equal ✅ We also have to be careful with relying on equality of regions in the trait solver. This is fine for codegen, as we treat all erased regions as equal. We can however lose equality information from HIR to MIR typeck. -The new solver "uniquifies regions" during canonicalization, canonicalizing `u32: Trait<'x, 'x>` -as `exists<'0, '1> u32: Trait<'0, '1>`, to make it harder to rely on this property. +This currently does not hold with the new solver: [trait-system-refactor-initiative#27](https://github.com/rust-lang/trait-system-refactor-initiative/issues/27). ### Removing ambiguity makes strictly more things compile ❌ Ideally we *should* not rely on ambiguity for things to compile. Not doing that will cause future improvements to be breaking changes. -Due to *incompleteness* this is not the case and improving inference can result in inference -changes, breaking existing projects. +Due to *incompleteness* this is not the case, +and improving inference can result in inference changes, breaking existing projects. ### Semantic equality implies structural equality ✅ Two types being equal in the type system must mean that they have the same `TypeId` after instantiating their generic parameters with concrete -arguments. This currently does not hold: [#97156]. +arguments. We can otherwise use their different `TypeId`s to impact trait selection. + +We lookup types using structural equality during codegen, but this shouldn't necessarily be unsound +- may result in redundant method codegen or backend type check errors? +- we also rely on it in CTFE assertions + +### Semantically different types have different `TypeId`s ✅ + +Semantically different `'static` types need different `TypeId`s to avoid transmutes, +for example `for<'a> fn(&'a str)` vs `fn(&'static str)` must have a different `TypeId`. + -[#57893]: https://github.com/rust-lang/rust/issues/57893 -[#97156]: https://github.com/rust-lang/rust/issues/97156 -[#114936]: https://github.com/rust-lang/rust/issues/114936 -[coherence]: ../coherence.md +[coherence chapter]: ../coherence.md diff --git a/src/solve/normalization.md b/src/solve/normalization.md deleted file mode 100644 index 99dc20c46..000000000 --- a/src/solve/normalization.md +++ /dev/null @@ -1,127 +0,0 @@ -# Normalization in the new solver - -> FIXME: Normalization has been changed significantly since this chapter was written. - -With the new solver we've made some fairly significant changes to normalization when compared -to the existing implementation. - -We now differentiate between "one-step normalization", "structural normalization" and -"deep normalization". - -## One-step normalization - -One-step normalization is implemented via `NormalizesTo` goals. Unlike other goals -in the trait solver, `NormalizesTo` always expects the term to be an unconstrained -inference variable[^opaques]. Think of it as a function, taking an alias as input -and returning its underlying value. If the alias is rigid, `NormalizesTo` fails and -returns `NoSolution`. This is the case for `::Assoc` if there's a `T: Trait` -where-bound and for opaque types with `Reveal::UserFacing` unless they are in the -defining scope. We must not treat any aliases as rigid in coherence. - -The underlying value may itself be an unnormalized alias, e.g. -`NormalizesTo(<<() as Id>::This as Id>::This)` only returns `<() as Id>::This`, -even though that alias can be further normalized to `()`. As the term is -always an unconstrained inference variable, the expected term cannot influence -normalization, see [trait-system-refactor-initiative#22] for more. - -Only ever computing `NormalizesTo` goals with an unconstrained inference variable -requires special solver support. It is only used by `AliasRelate` goals and pending -`NormalizesTo` goals are tracked separately from other goals: [source][try-eval-norm]. -As the expected term is always erased in `NormalizesTo`, we have to return its -ambiguous nested goals to its caller as not doing so weakens inference. See -[#122687] for more details. - -[trait-system-refactor-initiative#22]: https://github.com/rust-lang/trait-system-refactor-initiative/issues/22 -[try-eval-norm]: https://github.com/rust-lang/rust/blob/2627e9f3012a97d3136b3e11bf6bd0853c38a534/compiler/rustc_trait_selection/src/solve/eval_ctxt/mod.rs#L523-L537 -[#122687]: https://github.com/rust-lang/rust/pull/122687 - -## `AliasRelate` and structural normalization - -We structurally normalize an alias by applying one-step normalization until -we end up with a rigid alias, ambiguity, or overflow. This is done by repeatedly -evaluating `NormalizesTo` goals inside of a snapshot: [source][structural_norm]. - -`AliasRelate(lhs, rhs)` is implemented by first structurally normalizing both the -`lhs` and the `rhs` and then relating the resulting rigid types (or inference -variables). Importantly, if `lhs` or `rhs` ends up as an alias, this alias can -now be treated as rigid and gets unified without emitting a nested `AliasRelate` -goal: [source][structural-relate]. - -This means that `AliasRelate` with an unconstrained `rhs` ends up functioning -similar to `NormalizesTo`, acting as a function which fully normalizes `lhs` -before assigning the resulting rigid type to an inference variable. This is used by -`fn structurally_normalize_ty` both [inside] and [outside] of the trait solver. -This has to be used whenever we match on the value of some type, both inside -and outside of the trait solver. - - - -[structural_norm]: https://github.com/rust-lang/rust/blob/2627e9f3012a97d3136b3e11bf6bd0853c38a534/compiler/rustc_trait_selection/src/solve/alias_relate.rs#L140-L175 -[structural-relate]: https://github.com/rust-lang/rust/blob/a0569fa8f91b5271e92d2f73fd252de7d3d05b9c/compiler/rustc_trait_selection/src/solve/alias_relate.rs#L88-L107 -[inside]: https://github.com/rust-lang/rust/blob/a0569fa8f91b5271e92d2f73fd252de7d3d05b9c/compiler/rustc_trait_selection/src/solve/mod.rs#L278-L299 -[outside]: https://github.com/rust-lang/rust/blob/a0569fa8f91b5271e92d2f73fd252de7d3d05b9c/compiler/rustc_trait_selection/src/traits/structural_normalize.rs#L17-L48 - -## Deep normalization - -By walking over a type, and using `fn structurally_normalize_ty` for each encountered -alias, it is possible to deeply normalize a type, normalizing all aliases as much as -possible. However, this only works for aliases referencing bound variables if they are -not ambiguous as we're unable to replace the alias with a corresponding inference -variable without leaking universes. - - - -[generalize-no-alias]: https://github.com/rust-lang/rust/blob/a0569fa8f91b5271e92d2f73fd252de7d3d05b9c/compiler/rustc_infer/src/infer/relate/generalize.rs#L353-L358 - -## Outside of the trait solver - -The core type system - relating types and trait solving - will not need deep -normalization with the new solver. There are still some areas which depend on it. -For these areas there is the function `At::deeply_normalize`. Without additional -trait solver support deep normalization does not always work in case of ambiguity. -Luckily deep normalization is currently only necessary in places where there is no ambiguity. -`At::deeply_normalize` immediately fails if there's ambiguity. - -If we only care about the outermost layer of types, we instead use -`At::structurally_normalize` or `FnCtxt::(try_)structurally_resolve_type`. -Unlike `At::deeply_normalize`, structural normalization is also used in cases where we -have to handle ambiguity. - -Because this may result in behavior changes depending on how the trait solver handles -ambiguity, it is safer to also require full normalization there. This happens in -`FnCtxt::structurally_resolve_type` which always emits a hard error if the self type ends -up as an inference variable. There are some existing places which have a fallback for -inference variables instead. These places use `try_structurally_resolve_type` instead. - -## Why deep normalization with ambiguity is hard - -Fully correct deep normalization is very challenging, especially with the new solver -given that we do not want to deeply normalize inside of the solver. Mostly deeply normalizing -but sometimes failing to do so is bound to cause very hard to minimize and understand bugs. -If possible, avoiding any reliance on deep normalization entirely therefore feels preferable. - -If the solver itself does not deeply normalize, any inference constraints returned by the -solver would require normalization. Handling this correctly is ugly. This also means that -we change goals we provide to the trait solver by "normalizing away" some projections. - -The way we (mostly) guarantee deep normalization with the old solver is by eagerly replacing -the projection with an inference variable and emitting a nested `Projection` goal. This works -as `Projection` goals in the old solver deeply normalize. Unless we add another `PredicateKind` -for deep normalization to the new solver we cannot emulate this behavior. This does not work -for projections with bound variables, sometimes leaving them unnormalized. An approach which -also supports projections with bound variables will be even more involved. - -[^opaques]: opaque types are currently handled a bit differently. this may change in the future diff --git a/src/solve/opaque-types.md b/src/solve/opaque-types.md index 672aab770..8880962d6 100644 --- a/src/solve/opaque-types.md +++ b/src/solve/opaque-types.md @@ -1,8 +1,10 @@ # Opaque types in the new solver -The way opaque types are handled in the new solver differs from the old implementation. +The way [opaque types] are handled in the new solver differs from the old implementation. This should be a self-contained explanation of the behavior in the new solver. +[opaque types]: ../opaque-types-type-alias-impl-trait.md + ## opaques are alias types Opaque types are treated the same as other aliases, most notabily associated types, @@ -33,7 +35,7 @@ For opaque types in the defining scope and in the implicit-negative coherence mo always done in two steps. Outside of the defining scope `normalizes-to` for opaques always returns `Err(NoSolution)`. -We start by trying to to assign the expected type as a hidden type. +We start by trying to assign the expected type as a hidden type. In the implicit-negative coherence mode, this currently always results in ambiguity without interacting with the opaque types storage. We could instead add allow 'defining' all opaque types, @@ -54,7 +56,7 @@ Finally, we check whether the item bounds of the opaque hold for the expected ty [source][item-bounds-ck]. [norm]: https://github.com/rust-lang/rust/blob/384d26fc7e3bdd7687cc17b2662b091f6017ec2a/compiler/rustc_trait_selection/src/solve/normalizes_to/opaque_types.rs#L13 -[coherence-example]: https://github.com/rust-lang/rust/blob/master/tests/ui/type-alias-impl-trait/coherence_different_hidden_ty.rs +[coherence-example]: https://github.com/rust-lang/rust/blob/master/tests/ui/type-alias-impl-trait/coherence/coherence_different_hidden_ty.rs [placeholder-ck]: https://github.com/rust-lang/rust/blob/384d26fc7e3bdd7687cc17b2662b091f6017ec2a/compiler/rustc_trait_selection/src/solve/normalizes_to/opaque_types.rs#L33 [check-storage]: https://github.com/rust-lang/rust/blob/384d26fc7e3bdd7687cc17b2662b091f6017ec2a/compiler/rustc_trait_selection/src/solve/normalizes_to/opaque_types.rs#L51-L52 [eq-prev]: https://github.com/rust-lang/rust/blob/384d26fc7e3bdd7687cc17b2662b091f6017ec2a/compiler/rustc_trait_selection/src/solve/normalizes_to/opaque_types.rs#L51-L59 diff --git a/src/solve/significant-changes.md b/src/solve/significant-changes.md index c5bb8a01b..eac8f0318 100644 --- a/src/solve/significant-changes.md +++ b/src/solve/significant-changes.md @@ -42,7 +42,7 @@ old implementation structurally relates the aliases instead. This enables the new solver to stall equality until it is able to normalize the related aliases. The behavior of the old solver is incomplete and relies on eager normalization -which replaces ambiguous aliases with inference variables. As this is not +which replaces ambiguous aliases with inference variables. As this is not possible for aliases containing bound variables, the old implementation does not handle aliases inside of binders correctly, e.g. [#102048]. See the chapter on [normalization] for more details. @@ -106,4 +106,4 @@ their ambiguous nested goals are returned to the caller which then evaluates the See [#122687] for more details. [#122687]: https://github.com/rust-lang/rust/pull/122687 -[normalization]: ./normalization.md +[normalization]: ../normalization.md diff --git a/src/solve/the-solver.md b/src/solve/the-solver.md index f7d82d117..006fb649d 100644 --- a/src/solve/the-solver.md +++ b/src/solve/the-solver.md @@ -25,7 +25,7 @@ a separate "probe", to not leak inference constraints to the other candidates. We then try to merge the assembled candidates via `EvalCtxt::merge_candidates`. -## Important concepts and design pattern +## Important concepts and design patterns ### `EvalCtxt::add_goal` @@ -64,7 +64,7 @@ eagerly instantiates `'a` with a placeholder and then recursively proves Some goals can be proven in multiple ways. In these cases we try each option in a separate "probe" and then attempt to merge the resulting responses by using `EvalCtxt::try_merge_responses`. If merging the responses fails, we use -`EvalCtxt::flounder` instead, returning ambiguity. For some goals, we try +`EvalCtxt::flounder` instead, returning ambiguity. For some goals, we try to incompletely prefer some choices over others in case `EvalCtxt::try_merge_responses` fails. diff --git a/src/solve/trait-solving.md b/src/solve/trait-solving.md index 345ee0b09..c1eb1a94b 100644 --- a/src/solve/trait-solving.md +++ b/src/solve/trait-solving.md @@ -2,8 +2,7 @@ This chapter describes how trait solving works with the new WIP solver located in [`rustc_trait_selection/solve`][solve]. Feel free to also look at the docs for -[the current solver](../traits/resolution.md) and [the chalk solver](../traits/chalk.md) -can be found separately. +[the current solver](../traits/resolution.md) and [the chalk solver](../traits/chalk.md). ## Core concepts diff --git a/src/stability.md b/src/stability.md index 1bfe911c9..c26d34273 100644 --- a/src/stability.md +++ b/src/stability.md @@ -6,8 +6,6 @@ APIs to use unstable APIs internally in the rustc standard library. **NOTE**: this section is for *library* features, not *language* features. For instructions on stabilizing a language feature see [Stabilizing Features](./stabilization_guide.md). - - ## unstable The `#[unstable(feature = "foo", issue = "1234", reason = "lorem ipsum")]` @@ -34,7 +32,8 @@ Previously, due to a [rustc bug], stable items inside unstable modules were available to stable code in that location. As of September 2024, items with [accidentally stabilized paths] are marked with the `#[rustc_allowed_through_unstable_modules]` attribute -to prevent code dependent on those paths from breaking. +to prevent code dependent on those paths from breaking. Do *not* add this attribute +to any more items unless that is needed to avoid breaking changes. The `unstable` attribute may also have the `soft` value, which makes it a future-incompatible deny-by-default lint instead of a hard error. This is used @@ -182,4 +181,7 @@ the `deprecated_in_future` lint is triggered which is default `allow`, but most of the standard library raises it to a warning with `#![warn(deprecated_in_future)]`. +## unstable_feature_bound +The `#[unstable_feature_bound(foo)]` attribute can be used together with `#[unstable]` attribute to mark an `impl` of stable type and stable trait as unstable. In std/core, an item annotated with `#[unstable_feature_bound(foo)]` can only be used by another item that is also annotated with `#[unstable_feature_bound(foo)]`. Outside of std/core, using an item with `#[unstable_feature_bound(foo)]` requires the feature to be enabled with `#![feature(foo)]` attribute on the crate. Currently, only `impl`s and free functions can be annotated with `#[unstable_feature_bound]`. + [blog]: https://www.ralfj.de/blog/2018/07/19/const.html diff --git a/src/stabilization_guide.md b/src/stabilization_guide.md index f875c6874..e399930fc 100644 --- a/src/stabilization_guide.md +++ b/src/stabilization_guide.md @@ -1,120 +1,64 @@ # Request for stabilization -**NOTE**: this page is about stabilizing *language* features. -For stabilizing *library* features, see [Stabilizing a library feature]. +**NOTE**: This page is about stabilizing *language* features. For stabilizing *library* features, see [Stabilizing a library feature]. [Stabilizing a library feature]: ./stability.md#stabilizing-a-library-feature -Once an unstable feature has been well-tested with no outstanding -concern, anyone may push for its stabilization. It involves the -following steps: +Once an unstable feature has been well-tested with no outstanding concerns, anyone may push for its stabilization, though involving the people who have worked on it is prudent. Follow these steps: - +## Write an RFC, if needed -## Documentation PRs +If the feature was part of a [lang experiment], the lang team generally will want to first accept an RFC before stabilization. - +[lang experiment]: https://lang-team.rust-lang.org/how_to/experiment.html -If any documentation for this feature exists, it should be -in the [`Unstable Book`], located at [`src/doc/unstable-book`]. -If it exists, the page for the feature gate should be removed. +## Documentation PRs -If there was documentation there, integrating it into the -existing documentation is needed. + -If there wasn't documentation there, it needs to be added. +The feature might be documented in the [`Unstable Book`], located at [`src/doc/unstable-book`]. Remove the page for the feature gate if it exists. Integrate any useful parts of that documentation in other places. -Places that may need updated documentation: +Places that may need updated documentation include: -- [The Reference]: This must be updated, in full detail. -- [The Book]: This may or may not need updating, depends. - If you're not sure, please open an issue on this repository - and it can be discussed. -- standard library documentation: As needed. Language features - often don't need this, but if it's a feature that changes - how good examples are written, such as when `?` was added - to the language, updating examples is important. -- [Rust by Example]: As needed. +- [The Reference]: This must be updated, in full detail, and a member of the lang-docs team must review and approve the PR before the stabilization can be merged. +- [The Book]: This is updated as needed. If you're not sure, please open an issue on this repository and it can be discussed. +- Standard library documentation: This is updated as needed. Language features often don't need this, but if it's a feature that changes how idiomatic examples are written, such as when `?` was added to the language, updating these in the library documentation is important. Review also the keyword documentation and ABI documentation in the standard library, as these sometimes needs updates for language changes. +- [Rust by Example]: This is updated as needed. -Prepare PRs to update documentation involving this new feature -for repositories mentioned above. Maintainers of these repositories -will keep these PRs open until the whole stabilization process -has completed. Meanwhile, we can proceed to the next step. +Prepare PRs to update documentation involving this new feature for the repositories mentioned above. Maintainers of these repositories will keep these PRs open until the whole stabilization process has completed. Meanwhile, we can proceed to the next step. ## Write a stabilization report -Find the tracking issue of the feature, and create a short -stabilization report. Essentially this would be a brief summary -of the feature plus some links to test cases showing it works -as expected, along with a list of edge cases that came up -and were considered. This is a minimal "due diligence" that -we do before stabilizing. +Author a stabilization report using the [template found in this repository][srt]. -The report should contain: +The stabilization reports summarizes: -- A summary, showing examples (e.g. code snippets) what is - enabled by this feature. -- Links to test cases in our test suite regarding this feature - and describe the feature's behavior on encountering edge cases. -- Links to the documentations (the PRs we have made in the - previous steps). -- Any other relevant information. -- The resolutions of any unresolved questions if the stabilization - is for an RFC. +- The main design decisions and deviations since the RFC was accepted, including both decisions that were FCP'd or otherwise accepted by the language team as well as those being presented to the lang team for the first time. + - Often, the final stabilized language feature has significant design deviations from the original RFC. That's OK, but these deviations must be highlighted and explained carefully. +- The work that has been done since the RFC was accepted, acknowledging the main contributors that helped drive the language feature forward. -Examples of stabilization reports can be found in -[rust-lang/rust#44494][report1] and [rust-lang/rust#28237][report2] (these links -will bring you directly to the comment containing the stabilization report). +The [*Stabilization Template*][srt] includes a series of questions that aim to surface connections between this feature and lang's subteams (e.g. types, opsem, lang-docs, etc.) and to identify items that are commonly overlooked. -[report1]: https://github.com/rust-lang/rust/issues/44494#issuecomment-360191474 -[report2]: https://github.com/rust-lang/rust/issues/28237#issuecomment-363374130 +[srt]: ./stabilization_report_template.md -## FCP - -If any member of the team responsible for tracking this -feature agrees with stabilizing this feature, they will -start the FCP (final-comment-period) process by commenting - -```text -@rfcbot fcp merge -``` - -The rest of the team members will review the proposal. If the final -decision is to stabilize, we proceed to do the actual code modification. +The stabilization report is typically posted as the main comment on the stabilization PR (see the next section). ## Stabilization PR -*This is for stabilizing language features. If you are stabilizing a library -feature, see [the stabilization chapter of the std dev guide][std-guide-stabilization] instead.* - -Once we have decided to stabilize a feature, we need to have -a PR that actually makes that stabilization happen. These kinds -of PRs are a great way to get involved in Rust, as they take -you on a little tour through the source code. - -Here is a general guide to how to stabilize a feature -- -every feature is different, of course, so some features may -require steps beyond what this guide talks about. +Every feature is different, and some may require steps beyond what this guide discusses. -Note: Before we stabilize any feature, it's the rule that it -should appear in the documentation. +Before the stabilization will be considered by the lang team, there must be a complete PR to the Reference describing the feature, and before the stabilization PR will be merged, this PR must have been reviewed and approved by the lang-docs team. ### Updating the feature-gate listing -There is a central listing of unstable feature-gates in -[`compiler/rustc_feature/src/unstable.rs`]. Search for the `declare_features!` -macro. There should be an entry for the feature you are aiming -to stabilize, something like (this example is taken from -[rust-lang/rust#32409]: +There is a central listing of unstable feature-gates in [`compiler/rustc_feature/src/unstable.rs`]. Search for the `declare_features!` macro. There should be an entry for the feature you are aiming to stabilize, something like (this example is taken from [rust-lang/rust#32409]: ```rust,ignore // pub(restricted) visibilities (RFC 1422) (unstable, pub_restricted, "CURRENT_RUSTC_VERSION", Some(32409)), ``` -The above line should be moved to [`compiler/rustc_feature/src/accepted.rs`]. -Entries in the `declare_features!` call are sorted, so find the correct place. -When it is done, it should look like: +The above line should be moved to [`compiler/rustc_feature/src/accepted.rs`]. Entries in the `declare_features!` call are sorted, so find the correct place. When it is done, it should look like: ```rust,ignore // pub(restricted) visibilities (RFC 1422) @@ -122,54 +66,31 @@ When it is done, it should look like: // note that we changed this ``` -(Even though you will encounter version numbers in the file of past changes, -you should not put the rustc version you expect your stabilization to happen in, -but instead `CURRENT_RUSTC_VERSION`) +(Even though you will encounter version numbers in the file of past changes, you should not put the rustc version you expect your stabilization to happen in, but instead use `CURRENT_RUSTC_VERSION`.) ### Removing existing uses of the feature-gate -Next search for the feature string (in this case, `pub_restricted`) -in the codebase to find where it appears. Change uses of -`#![feature(XXX)]` from the `std` and any rustc crates (this includes test folders -under `library/` and `compiler/` but not the toplevel `tests/` one) to be -`#![cfg_attr(bootstrap, feature(XXX))]`. This includes the feature-gate -only for stage0, which is built using the current beta (this is -needed because the feature is still unstable in the current beta). +Next, search for the feature string (in this case, `pub_restricted`) in the codebase to find where it appears. Change uses of `#![feature(XXX)]` from the `std` and any rustc crates (this includes test folders under `library/` and `compiler/` but not the toplevel `tests/` one) to be `#![cfg_attr(bootstrap, feature(XXX))]`. This includes the feature-gate only for stage0, which is built using the current beta (this is needed because the feature is still unstable in the current beta). -Also, remove those strings from any tests (e.g. under `tests/`). If there are tests -specifically targeting the feature-gate (i.e., testing that the -feature-gate is required to use the feature, but nothing else), -simply remove the test. +Also, remove those strings from any tests (e.g. under `tests/`). If there are tests specifically targeting the feature-gate (i.e., testing that the feature-gate is required to use the feature, but nothing else), simply remove the test. ### Do not require the feature-gate to use the feature -Most importantly, remove the code which flags an error if the -feature-gate is not present (since the feature is now considered -stable). If the feature can be detected because it employs some -new syntax, then a common place for that code to be is in the -same `compiler/rustc_ast_passes/src/feature_gate.rs`. -For example, you might see code like this: +Most importantly, remove the code which flags an error if the feature-gate is not present (since the feature is now considered stable). If the feature can be detected because it employs some new syntax, then a common place for that code to be is in `compiler/rustc_ast_passes/src/feature_gate.rs`. For example, you might see code like this: ```rust,ignore -gate_feature_post!(&self, pub_restricted, span, - "`pub(restricted)` syntax is experimental"); +gate_all!(pub_restricted, "`pub(restricted)` syntax is experimental"); ``` -This `gate_feature_post!` macro prints an error if the -`pub_restricted` feature is not enabled. It is not needed -now that `#[pub_restricted]` is stable. +This `gate_feature_post!` macro prints an error if the `pub_restricted` feature is not enabled. It is not needed now that `#[pub_restricted]` is stable. For more subtle features, you may find code like this: ```rust,ignore -if self.tcx.sess.features.borrow().pub_restricted { /* XXX */ } +if self.tcx.features().async_fn_in_dyn_trait() { /* XXX */ } ``` -This `pub_restricted` field (obviously named after the feature) -would ordinarily be false if the feature flag is not present -and true if it is. So transform the code to assume that the field -is true. In this case, that would mean removing the `if` and -leaving just the `/* XXX */`. +This `pub_restricted` field (named after the feature) would ordinarily be false if the feature flag is not present and true if it is. So transform the code to assume that the field is true. In this case, that would mean removing the `if` and leaving just the `/* XXX */`. ```rust,ignore if self.tcx.sess.features.borrow().pub_restricted { /* XXX */ } @@ -194,3 +115,40 @@ if something { /* XXX */ } [Rust by Example]: https://github.com/rust-lang/rust-by-example [`Unstable Book`]: https://doc.rust-lang.org/unstable-book/index.html [`src/doc/unstable-book`]: https://github.com/rust-lang/rust/tree/master/src/doc/unstable-book + +## Team nominations + +When opening the stabilization PR, CC the lang team and its advisors (`@rust-lang/lang @rust-lang/lang-advisors`) and any other teams to whom the feature is relevant, e.g.: + +- `@rust-lang/types`, for type system interactions. +- `@rust-lang/opsem`, for interactions with unsafe code. +- `@rust-lang/compiler`, for implementation robustness. +- `@rust-lang/libs-api`, for changes to the standard library API or its guarantees. +- `@rust-lang/lang-docs`, for questions about how this should be documented in the Reference. + +After the stabilization PR is opened with the stabilization report, wait a bit for any immediate comments. When such comments "simmer down" and you feel the PR is ready for consideration by the lang team, [nominate the PR](https://lang-team.rust-lang.org/how_to/nominate.html) to get it on the agenda for consideration in an upcoming lang meeting. + +If you are not a `rust-lang` organization member, you can ask your assigned reviewer to CC the relevant teams on your behalf. + +## Propose FCP on the PR + +After the lang team and other relevant teams review the stabilization, and after you have answered any questions they may have had, a member of one of the teams may propose to accept the stabilization by commenting: + +```text +@rfcbot fcp merge +``` + +Once enough team members have reviewed, the PR will move into a "final comment period" (FCP). If no new concerns are raised, this period will complete and the PR can be merged after implementation review in the usual way. + +## Reviewing and merging stabilizations + +On a stabilization, before giving it the `r+`, ensure that the PR: + +- Matches what the team proposed for stabilization and what is documented in the Reference PR. +- Includes any changes the team decided to request along the way in order to resolve or avoid concerns. +- Is otherwise exactly what is described in the stabilization report and in any relevant RFCs or prior lang FCPs. +- Does not expose on stable behaviors other than those specified, accepted for stabilization, and documented in the Reference. +- Has sufficient tests to convincingly demonstrate these things. +- Is accompanied by a PR to the Reference than has been reviewed and approved by a member of lang-docs. + +In particular, when reviewing the PR, keep an eye out for any user-visible details that the lang team failed to consider and specify. If you find one, describe it and nominate the PR for the lang team. diff --git a/src/stabilization_report_template.md b/src/stabilization_report_template.md new file mode 100644 index 000000000..793f7d7e4 --- /dev/null +++ b/src/stabilization_report_template.md @@ -0,0 +1,277 @@ +# Stabilization report template + +## What is this? + +This is a template for [stabilization reports](./stabilization_guide.md) of **language features**. The questions aim to solicit the details most often needed. These details help reviewers to identify potential problems upfront. Not all parts of the template will apply to every stabilization. If a question doesn't apply, explain briefly why. + +Copy everything after the separator and edit it as Markdown. Replace each *TODO* with your answer. + +--- + +# Stabilization report + +## Summary + +> Remind us what this feature is and what value it provides. Tell the story of what led up to this stabilization. +> +> E.g., see: +> +> - [Stabilize AFIT/RPITIT](https://web.archive.org/web/20250329190642/https://github.com/rust-lang/rust/pull/115822) +> - [Stabilize RTN](https://web.archive.org/web/20250321214601/https://github.com/rust-lang/rust/pull/138424) +> - [Stabilize ATPIT](https://web.archive.org/web/20250124214256/https://github.com/rust-lang/rust/pull/120700) +> - [Stabilize opaque type precise capturing](https://web.archive.org/web/20250312173538/https://github.com/rust-lang/rust/pull/127672) + +*TODO* + +Tracking: + +- *TODO* (Link to tracking issue.) + +Reference PRs: + +- *TODO* (Link to Reference PRs.) + +cc @rust-lang/lang @rust-lang/lang-advisors + +### What is stabilized + +> Describe each behavior being stabilized and give a short example of code that will now be accepted. + +```rust +todo!() +``` + +### What isn't stabilized + +> Describe any parts of the feature not being stabilized. Talk about what we might want to do later and what doors are being left open for that. If what we're not stabilizing might lead to surprises for users, talk about that in particular. + +## Design + +### Reference + +> What updates are needed to the Reference? Link to each PR. If the Reference is missing content needed for describing this feature, discuss that. + +- *TODO* + +### RFC history + +> What RFCs have been accepted for this feature? + +- *TODO* + +### Answers to unresolved questions + +> What questions were left unresolved by the RFC? How have they been answered? Link to any relevant lang decisions. + +*TODO* + +### Post-RFC changes + +> What other user-visible changes have occurred since the RFC was accepted? Describe both changes that the lang team accepted (and link to those decisions) as well as changes that are being presented to the team for the first time in this stabilization report. + +*TODO* + +### Key points + +> What decisions have been most difficult and what behaviors to be stabilized have proved most contentious? Summarize the major arguments on all sides and link to earlier documents and discussions. + +*TODO* + +### Nightly extensions + +> Are there extensions to this feature that remain unstable? How do we know that we are not accidentally committing to those? + +*TODO* + +### Doors closed + +> What doors does this stabilization close for later changes to the language? E.g., does this stabilization make any other RFCs, lang experiments, or known in-flight proposals more difficult or impossible to do later? + +## Feedback + +### Call for testing + +> Has a "call for testing" been done? If so, what feedback was received? + +*TODO* + +### Nightly use + +> Do any known nightly users use this feature? Counting instances of `#![feature(FEATURE_NAME)]` on GitHub with grep might be informative. + +*TODO* + +## Implementation + +### Major parts + +> Summarize the major parts of the implementation and provide links into the code and to relevant PRs. +> +> See, e.g., this breakdown of the major parts of async closures: +> +> - + +*TODO* + +### Coverage + +> Summarize the test coverage of this feature. +> +> Consider what the "edges" of this feature are. We're particularly interested in seeing tests that assure us about exactly what nearby things we're not stabilizing. Tests should of course comprehensively demonstrate that the feature works. Think too about demonstrating the diagnostics seen when common mistakes are made and the feature is used incorrectly. +> +> Within each test, include a comment at the top describing the purpose of the test and what set of invariants it intends to demonstrate. This is a great help to our review. +> +> Describe any known or intentional gaps in test coverage. +> +> Contextualize and link to test folders and individual tests. + +*TODO* + +### Outstanding bugs + +> What outstanding bugs involve this feature? List them. Should any block the stabilization? Discuss why or why not. + +*TODO* + +- *TODO* +- *TODO* +- *TODO* + +### Outstanding FIXMEs + +> What FIXMEs are still in the code for that feature and why is it OK to leave them there? + +*TODO* + +### Tool changes + +> What changes must be made to our other tools to support this feature. Has this work been done? Link to any relevant PRs and issues. + +- [ ] rustfmt + - *TODO* +- [ ] rust-analyzer + - *TODO* +- [ ] rustdoc (both JSON and HTML) + - *TODO* +- [ ] cargo + - *TODO* +- [ ] clippy + - *TODO* +- [ ] rustup + - *TODO* +- [ ] docs.rs + - *TODO* + +*TODO* + +### Breaking changes + +> If this stabilization represents a known breaking change, link to the crater report, the analysis of the crater report, and to all PRs we've made to ecosystem projects affected by this breakage. Discuss any limitations of what we're able to know about or to fix. + +*TODO* + +Crater report: + +- *TODO* + +Crater analysis: + +- *TODO* + +PRs to affected crates: + +- *TODO* +- *TODO* +- *TODO* + +## Type system, opsem + +### Compile-time checks + +> What compilation-time checks are done that are needed to prevent undefined behavior? +> +> Link to tests demonstrating that these checks are being done. + +*TODO* + +- *TODO* +- *TODO* +- *TODO* + +### Type system rules + +> What type system rules are enforced for this feature and what is the purpose of each? + +*TODO* + +### Sound by default? + +> Does the feature's implementation need specific checks to prevent UB, or is it sound by default and need specific opt-in to perform the dangerous/unsafe operations? If it is not sound by default, what is the rationale? + +*TODO* + +### Breaks the AM? + +> Can users use this feature to introduce undefined behavior, or use this feature to break the abstraction of Rust and expose the underlying assembly-level implementation? Describe this if so. + +*TODO* + +## Common interactions + +### Temporaries + +> Does this feature introduce new expressions that can produce temporaries? What are the scopes of those temporaries? + +*TODO* + +### Drop order + +> Does this feature raise questions about the order in which we should drop values? Talk about the decisions made here and how they're consistent with our earlier decisions. + +*TODO* + +### Pre-expansion / post-expansion + +> Does this feature raise questions about what should be accepted pre-expansion (e.g. in code covered by `#[cfg(false)]`) versus what should be accepted post-expansion? What decisions were made about this? + +*TODO* + +### Edition hygiene + +> If this feature is gated on an edition, how do we decide, in the context of the edition hygiene of tokens, whether to accept or reject code. E.g., what token do we use to decide? + +*TODO* + +### SemVer implications + +> Does this feature create any new ways in which library authors must take care to prevent breaking downstreams when making minor-version releases? Describe these. Are these new hazards "major" or "minor" according to [RFC 1105](https://rust-lang.github.io/rfcs/1105-api-evolution.html)? + +*TODO* + +### Exposing other features + +> Are there any other unstable features whose behavior may be exposed by this feature in any way? What features present the highest risk of that? + +*TODO* + +## History + +> List issues and PRs that are important for understanding how we got here. + +- *TODO* +- *TODO* +- *TODO* + +## Acknowledgments + +> Summarize contributors to the feature by name for recognition and so that those people are notified about the stabilization. Does anyone who worked on this *not* think it should be stabilized right now? We'd like to hear about that if so. + +*TODO* + +## Open items + +> List any known items that have not yet been completed and that should be before this is stabilized. + +- [ ] *TODO* +- [ ] *TODO* +- [ ] *TODO* diff --git a/src/test-implementation.md b/src/test-implementation.md index bee783c0f..f09d73631 100644 --- a/src/test-implementation.md +++ b/src/test-implementation.md @@ -1,7 +1,5 @@ # The `#[test]` attribute - - Many Rust programmers rely on a built-in attribute called `#[test]`. All @@ -83,7 +81,7 @@ with your hand-written one, it will not share a [Symbol][Symbol]. This technique prevents name collision during code generation and is the foundation of Rust's [`macro`] hygiene. -## Step 2: Harness Generation +## Step 2: Harness generation Now that our tests are accessible from the root of our crate, we need to do something with them using [`rustc_ast`][ast] generates a module like so: @@ -106,7 +104,7 @@ called [`test`][test] that is part of Rust core, that implements all of the runtime for testing. [`test`][test]'s interface is unstable, so the only stable way to interact with it is through the `#[test]` macro. -## Step 3: Test Object Generation +## Step 3: Test object generation If you've written tests in Rust before, you may be familiar with some of the optional attributes available on test functions. For example, a test can be diff --git a/src/tests/adding.md b/src/tests/adding.md index 895eabfbd..e5c26bef1 100644 --- a/src/tests/adding.md +++ b/src/tests/adding.md @@ -1,7 +1,5 @@ # Adding new tests - - **In general, we expect every PR that fixes a bug in rustc to come accompanied by a regression test of some kind.** This test should fail in master but pass after the PR. These tests are really useful for preventing us from repeating the diff --git a/src/tests/best-practices.md b/src/tests/best-practices.md index 6905ee132..be00207e3 100644 --- a/src/tests/best-practices.md +++ b/src/tests/best-practices.md @@ -70,6 +70,11 @@ related tests. > //! > //! Regression test for . > ``` +> +> One exception to this rule is [crashes tests]: there it is canonical that +> tests are named only after issue numbers because its purpose is to track +> snippets from which issues no longer ICE/crash, and they would either be +> removed or converted into proper ui/other tests in the fix PRs. ## Test organization @@ -175,6 +180,8 @@ See [compiletest directives] for a listing of directives. - For `ignore-*`/`needs-*`/`only-*` directives, unless extremely obvious, provide a brief remark on why the directive is needed. E.g. `"//@ ignore-wasi (wasi codegens the main symbol differently)"`. +- When using `//@ ignore-auxiliary`, specify the corresponding main test files, + e.g. ``//@ ignore-auxiliary (used by `./foo.rs`)``. ## FileCheck best practices @@ -192,3 +199,4 @@ See [LLVM FileCheck guide][FileCheck] for details. [compiletest directives]: ./directives.md [`run-make`]: ./compiletest.md#run-make-tests [FileCheck]: https://llvm.org/docs/CommandGuide/FileCheck.html +[crashes tests]: ./compiletest.md#crashes-tests diff --git a/src/tests/ci.md b/src/tests/ci.md index 5e27a2fd7..750e4fa1a 100644 --- a/src/tests/ci.md +++ b/src/tests/ci.md @@ -28,7 +28,7 @@ Our CI is primarily executed on [GitHub Actions], with a single workflow defined in [`.github/workflows/ci.yml`], which contains a bunch of steps that are unified for all CI jobs that we execute. When a commit is pushed to a corresponding branch or a PR, the workflow executes the -[`calculate-job-matrix.py`] script, which dynamically generates the specific CI +[`src/ci/citool`] crate, which dynamically generates the specific CI jobs that should be executed. This script uses the [`jobs.yml`] file as an input, which contains a declarative configuration of all our CI jobs. @@ -66,9 +66,9 @@ kinds of builds (sets of jobs). ### Pull Request builds After each push to a pull request, a set of `pr` jobs are executed. Currently, -these execute the `x86_64-gnu-llvm-X`, `x86_64-gnu-tools`, `mingw-check` and -`mingw-check-tidy` jobs, all running on Linux. These execute a relatively short -(~30 minutes) and lightweight test suite that should catch common issues. More +these execute the `x86_64-gnu-llvm-X`, `x86_64-gnu-tools`, `pr-check-1`, `pr-check-2` +and `tidy` jobs, all running on Linux. These execute a relatively short +(~40 minutes) and lightweight test suite that should catch common issues. More specifically, they run a set of lints, they try to perform a cross-compile check build to Windows mingw (without producing any artifacts) and they test the compiler using a *system* version of LLVM. Unfortunately, it would take too many @@ -100,8 +100,8 @@ Most platforms only run the build steps, some run a restricted set of tests, only a subset run the full suite of tests (see Rust's [platform tiers]). Auto jobs are defined in the `auto` section of [`jobs.yml`]. They are executed -on the `auto` branch under the `rust-lang-ci/rust` repository[^rust-lang-ci] and -their results can be seen [here](https://github.com/rust-lang-ci/rust/actions), +on the `auto` branch under the `rust-lang/rust` repository and +their results can be seen [here](https://github.com/rust-lang/rust/actions), although usually you will be notified of the result by a comment made by bors on the corresponding PR. @@ -110,9 +110,6 @@ more [here](#merging-prs-serially-with-bors). [platform tiers]: https://forge.rust-lang.org/release/platform-support.html#rust-platform-support -[^rust-lang-ci]: The `auto` and `try` jobs run under the `rust-lang-ci` fork for - historical reasons. This may change in the future. - ### Try builds Sometimes we want to run a subset of the test suite on CI for a given PR, or @@ -133,29 +130,48 @@ There are several use-cases for try builds: Again, a working compiler build is needed for this, which can be produced by the [dist-x86_64-linux] CI job. - Run a specific CI job (e.g. Windows tests) on a PR, to quickly test if it - passes the test suite executed by that job. You can select which CI jobs will - be executed in the try build by adding up to 10 lines containing `try-job: - ` to the PR description. All such specified jobs will be executed - in the try build once the `@bors try` command is used on the PR. If no try - jobs are specified in this way, the jobs defined in the `try` section of - [`jobs.yml`] will be executed by default. + passes the test suite executed by that job. + +By default, if you send a comment with `@bors try`, the jobs defined in the `try` section of +[`jobs.yml`] will be executed. We call this mode a "fast try build". Such a try build +will not execute any tests, and it will allow compilation warnings. It is useful when you want to +get an optimized toolchain as fast as possible, for a crater run or performance benchmarks, +even if it might not be working fully correctly. + +If you want to run a custom CI job in a try build and make sure that it passes all tests and does +not produce any compilation warnings, you can select CI jobs to be executed by adding lines +containing `try-job: ` to the PR description. All such specified jobs will be executed +in the try build once the `@bors try` command is used on the PR. + +Each pattern can either be an exact name of a job or a glob pattern that matches multiple jobs, +for example `*msvc*` or `*-alt`. You can start at most 20 jobs in a single try build. When using +glob patterns, you might want to wrap them in backticks (`` ` ``) to avoid GitHub rendering +the pattern as Markdown. + +The job pattern needs to match one or more jobs defined in the `auto` or `optional` sections +of [`jobs.yml`]: + +- `auto` jobs are executed before a commit is merged into the `master` branch. +- `optional` jobs are executed only when explicitly requested via a try build. + They are typically used for tier 2 and tier 3 targets. > **Using `try-job` PR description directives** > -> 1. Identify which set of try-jobs (max 10) you would like to exercise. You can +> 1. Identify which set of try-jobs you would like to exercise. You can > find the name of the CI jobs in [`jobs.yml`]. > -> 2. Amend PR description to include (usually at the end of the PR description) -> e.g. +> 2. Amend PR description to include a set of patterns (usually at the end +> of the PR description), for example: > > ```text > This PR fixes #123456. > > try-job: x86_64-msvc > try-job: test-various +> try-job: `*-alt` > ``` > -> Each `try-job` directive must be on its own line. +> Each `try-job` pattern must be on its own line. > > 3. Run the prescribed try jobs with `@bors try`. As aforementioned, this > requires the user to either (1) have `try` permissions or (2) be delegated @@ -167,17 +183,21 @@ There are several use-cases for try builds: > that are exercised this way. Try jobs are defined in the `try` section of [`jobs.yml`]. They are executed on -the `try` branch under the `rust-lang-ci/rust` repository[^rust-lang-ci] and -their results can be seen [here](https://github.com/rust-lang-ci/rust/actions), +the `try` branch under the `rust-lang/rust` repository and +their results can be seen [here](https://github.com/rust-lang/rust/actions), although usually you will be notified of the result by a comment made by bors on the corresponding PR. +Note that if you start the default try job using `@bors try`, it will skip building several `dist` components and running post-optimization tests, to make the build duration shorter. If you want to execute the full build as it would happen before a merge, add an explicit `try-job` pattern with the name of the default try job (currently `dist-x86_64-linux`). + Multiple try builds can execute concurrently across different PRs.
-bors identify try jobs by commit hash. This means that if you have two PRs + +Bors identifies try jobs by commit hash. This means that if you have two PRs containing the same (latest) commits, running `@bors try` will result in the *same* try job and it really confuses `bors`. Please refrain from doing so. +
[rustc-perf]: https://github.com/rust-lang/rustc-perf @@ -299,8 +319,7 @@ platform’s custom [Docker container]. This has a lot of advantages for us: - We can avoid reinstalling tools (like QEMU or the Android emulator) every time thanks to Docker image caching. - Users can run the same tests in the same environment locally by just running - `src/ci/docker/run.sh image-name`, which is awesome to debug failures. Note - that there are only linux docker images available locally due to licensing and + `cargo run --manifest-path src/ci/citool/Cargo.toml run-local `, which is awesome to debug failures. Note that there are only linux docker images available locally due to licensing and other restrictions. The docker images prefixed with `dist-` are used for building artifacts while @@ -323,7 +342,7 @@ Our CI workflow uses various caching mechanisms, mainly for two things: ### Docker images caching The Docker images we use to run most of the Linux-based builders take a *long* -time to fully build. To speed up the build, we cache it using [Docker registry +time to fully build. To speed up the build, we cache them using [Docker registry caching], with the intermediate artifacts being stored on [ghcr.io]. We also push the built Docker images to ghcr, so that they can be reused by other tools (rustup) or by developers running the Docker build locally (to speed up their @@ -335,16 +354,32 @@ override the cache for the others. Instead, we store the images under different tags, identifying them with a custom hash made from the contents of all the Dockerfiles and related scripts. -[ghcr.io]: https://github.com/rust-lang-ci/rust/pkgs/container/rust-ci +The CI calculates a hash key, so that the cache of a Docker image is +invalidated if one of the following changes: + +- Dockerfile +- Files copied into the Docker image in the Dockerfile +- The architecture of the GitHub runner (x86 or ARM) + +[ghcr.io]: https://github.com/rust-lang/rust/pkgs/container/rust-ci [Docker registry caching]: https://docs.docker.com/build/cache/backends/registry/ ### LLVM caching with sccache We build some C/C++ stuff in various CI jobs, and we rely on [sccache] to cache the intermediate LLVM artifacts. Sccache is a distributed ccache developed by -Mozilla, which can use an object storage bucket as the storage backend. In our -case, the artefacts are uploaded to an S3 bucket that we control -(`rust-lang-ci-sccache2`). +Mozilla, which can use an object storage bucket as the storage backend. + +With sccache there's no need to calculate the hash key ourselves. Sccache +invalidates the cache automatically when it detects changes to relevant inputs, +such as the source code, the version of the compiler, and important environment +variables. +So we just pass the sccache wrapper on top of cargo and sccache does the rest. + +We store the persistent artifacts on the S3 bucket `rust-lang-ci-sccache2`. So +when the CI runs, if sccache sees that LLVM is being compiled with the same C/C++ +compiler and the LLVM source code is the same, sccache retrieves the individual +compiled translation units from S3. [sccache]: https://github.com/mozilla/sccache @@ -410,11 +445,25 @@ To learn more about the dashboard, see the [Datadog CI docs]. [Datadog CI docs]: https://docs.datadoghq.com/continuous_integration/ [public dashboard]: https://p.datadoghq.com/sb/3a172e20-e9e1-11ed-80e3-da7ad0900002-b5f7bb7e08b664a06b08527da85f7e30 +## Determining the CI configuration + +If you want to determine which `bootstrap.toml` settings are used in CI for a +particular job, it is probably easiest to just look at the build log. To do +this: + +1. Go to + + to find the most recently successful build, and click on it. +2. Choose the job you are interested in on the left-hand side. +3. Click on the gear icon and choose "View raw logs" +4. Search for the string "Configure the build" +5. All of the build settings are listed below that starting with the + `configure:` prefix. + [GitHub Actions]: https://github.com/rust-lang/rust/actions [`jobs.yml`]: https://github.com/rust-lang/rust/blob/master/src/ci/github-actions/jobs.yml [`.github/workflows/ci.yml`]: https://github.com/rust-lang/rust/blob/master/.github/workflows/ci.yml -[`calculate-job-matrix.py`]: https://github.com/rust-lang/rust/blob/master/src/ci/github-actions/calculate-job-matrix.py -[rust-lang-ci]: https://github.com/rust-lang-ci/rust/actions +[`src/ci/citool`]: https://github.com/rust-lang/rust/blob/master/src/ci/citool [bors]: https://github.com/bors [homu]: https://github.com/rust-lang/homu [merge queue]: https://bors.rust-lang.org/queue/rust diff --git a/src/tests/codegen-backend-tests/cg_clif.md b/src/tests/codegen-backend-tests/cg_clif.md new file mode 100644 index 000000000..030ddd7df --- /dev/null +++ b/src/tests/codegen-backend-tests/cg_clif.md @@ -0,0 +1,3 @@ +# Cranelift codegen backend tests + +TODO: please add some more information to this page. diff --git a/src/tests/codegen-backend-tests/cg_gcc.md b/src/tests/codegen-backend-tests/cg_gcc.md new file mode 100644 index 000000000..4caf4c0e0 --- /dev/null +++ b/src/tests/codegen-backend-tests/cg_gcc.md @@ -0,0 +1,3 @@ +# GCC codegen backend tests + +TODO: please add some more information to this page. diff --git a/src/tests/codegen-backend-tests/intro.md b/src/tests/codegen-backend-tests/intro.md new file mode 100644 index 000000000..6bf46ddcd --- /dev/null +++ b/src/tests/codegen-backend-tests/intro.md @@ -0,0 +1,13 @@ +# Codegen backend testing + +See also the [Code generation](../../backend/codegen.md) chapter. + +In addition to the primary LLVM codegen backend, the rust-lang/rust CI also runs tests of the [cranelift][cg_clif] and [GCC][cg_gcc] codegen backends in certain test jobs. + +For more details on the tests involved, see: + +- [Cranelift codegen backend tests](./cg_clif.md) +- [GCC codegen backend tests](./cg_gcc.md) + +[cg_clif]: https://github.com/rust-lang/rustc_codegen_cranelift +[cg_gcc]: https://github.com/rust-lang/rustc_codegen_gcc diff --git a/src/tests/compiletest.md b/src/tests/compiletest.md index 459c08290..4980ed845 100644 --- a/src/tests/compiletest.md +++ b/src/tests/compiletest.md @@ -1,7 +1,5 @@ # Compiletest - - ## Introduction `compiletest` is the main test harness of the Rust test suite. It allows test @@ -29,7 +27,7 @@ on if or how to run the test, what behavior to expect, and more. See [directives](directives.md) and the test suite documentation below for more details on these annotations. -See the [Adding new tests](adding.md) and [Best practies](best-practices.md) +See the [Adding new tests](adding.md) and [Best practices](best-practices.md) chapters for a tutorial on creating a new test and advice on writing a good test, and the [Running tests](running.md) chapter on how to run the test suite. @@ -56,6 +54,9 @@ incremental compilation. The various suites are defined in The following test suites are available, with links for more information: +[`tests`]: https://github.com/rust-lang/rust/blob/master/tests +[`src/tools/compiletest/src/common.rs`]: https://github.com/rust-lang/rust/tree/master/src/tools/compiletest/src/common.rs + ### Compiler-specific test suites | Test suite | Purpose | @@ -65,33 +66,37 @@ The following test suites are available, with links for more information: | [`pretty`](#pretty-printer-tests) | Check pretty printing | | [`incremental`](#incremental-tests) | Check incremental compilation behavior | | [`debuginfo`](#debuginfo-tests) | Check debuginfo generation running debuggers | -| [`codegen`](#codegen-tests) | Check code generation | +| [`codegen-*`](#codegen-tests) | Check code generation | | [`codegen-units`](#codegen-units-tests) | Check codegen unit partitioning | | [`assembly`](#assembly-tests) | Check assembly output | | [`mir-opt`](#mir-opt-tests) | Check MIR generation and optimizations | | [`coverage`](#coverage-tests) | Check coverage instrumentation | | [`coverage-run-rustdoc`](#coverage-tests) | `coverage` tests that also run instrumented doctests | +| [`crashes`](#crashes-tests) | Check that the compiler ICEs/panics/crashes on certain inputs to catch accidental fixes | ### General purpose test suite -[`run-make`](#run-make-tests) are general purpose tests using Rust programs (or -Makefiles (legacy)). +[`run-make`](#run-make-tests) are general purpose tests using Rust programs. ### Rustdoc test suites -See [Rustdoc tests](../rustdoc.md#tests) for more details. +| Test suite | Purpose | +|--------------------------------------|--------------------------------------------------------------------------| +| [`rustdoc`][rustdoc-html-tests] | Check HTML output of `rustdoc` | +| [`rustdoc-gui`][rustdoc-gui-tests] | Check `rustdoc`'s GUI using a web browser | +| [`rustdoc-js`][rustdoc-js-tests] | Check `rustdoc`'s search engine and index | +| [`rustdoc-js-std`][rustdoc-js-tests] | Check `rustdoc`'s search engine and index on the std library docs | +| [`rustdoc-json`][rustdoc-json-tests] | Check JSON output of `rustdoc` | +| `rustdoc-ui` | Check terminal output of `rustdoc` ([see also](ui.md)) | -| Test suite | Purpose | -|------------------|--------------------------------------------------------------------------| -| `rustdoc` | Check `rustdoc` generated files contain the expected documentation | -| `rustdoc-gui` | Check `rustdoc`'s GUI using a web browser | -| `rustdoc-js` | Check `rustdoc` search is working as expected | -| `rustdoc-js-std` | Check rustdoc search is working as expected specifically on the std docs | -| `rustdoc-json` | Check JSON output of `rustdoc` | -| `rustdoc-ui` | Check terminal output of `rustdoc` | +Some rustdoc-specific tests can also be found in `ui/rustdoc/`. +These check rustdoc-related or -specific lints that (also) run as part of `rustc`, not (only) `rustdoc`. +Run-make tests pertaining to rustdoc are typically named `run-make/rustdoc-*/`. -[`tests`]: https://github.com/rust-lang/rust/blob/master/tests -[`src/tools/compiletest/src/common.rs`]: https://github.com/rust-lang/rust/tree/master/src/tools/compiletest/src/common.rs +[rustdoc-html-tests]: ../rustdoc-internals/rustdoc-test-suite.md +[rustdoc-gui-tests]: ../rustdoc-internals/rustdoc-gui-test-suite.md +[rustdoc-js-tests]: ../rustdoc-internals/search.md#testing-the-search-engine +[rustdoc-json-tests]: ../rustdoc-internals/rustdoc-json-test-suite.md ### Pretty-printer tests @@ -108,7 +113,7 @@ default behavior without any commands is to: 2. Run `rustc -Zunpretty=normal` on the output of the previous step. 3. The output of the previous two steps should be the same. 4. Run `rustc -Zno-codegen` on the output to make sure that it can type check - (this is similar to running `cargo check`). + (similar to `cargo check`). If any of the commands above fail, then the test fails. @@ -283,7 +288,7 @@ For example, `./x test tests/debuginfo -- --debugger gdb` will only test GDB com ### Codegen tests -The tests in [`tests/codegen`] test LLVM code generation. They compile the test +The tests in [`tests/codegen-llvm`] test LLVM code generation. They compile the test with the `--emit=llvm-ir` flag to emit LLVM IR. They then run the LLVM [FileCheck] tool. The test is annotated with various `// CHECK` comments to check the generated code. See the [FileCheck] documentation for a tutorial and @@ -294,13 +299,13 @@ See also the [assembly tests](#assembly-tests) for a similar set of tests. If you need to work with `#![no_std]` cross-compiling tests, consult the [`minicore` test auxiliary](./minicore.md) chapter. -[`tests/codegen`]: https://github.com/rust-lang/rust/tree/master/tests/codegen +[`tests/codegen-llvm`]: https://github.com/rust-lang/rust/tree/master/tests/codegen-llvm [FileCheck]: https://llvm.org/docs/CommandGuide/FileCheck.html ### Assembly tests -The tests in [`tests/assembly`] test LLVM assembly output. They compile the test +The tests in [`tests/assembly-llvm`] test LLVM assembly output. They compile the test with the `--emit=asm` flag to emit a `.s` file with the assembly output. They then run the LLVM [FileCheck] tool. @@ -317,7 +322,7 @@ See also the [codegen tests](#codegen-tests) for a similar set of tests. If you need to work with `#![no_std]` cross-compiling tests, consult the [`minicore` test auxiliary](./minicore.md) chapter. -[`tests/assembly`]: https://github.com/rust-lang/rust/tree/master/tests/assembly +[`tests/assembly-llvm`]: https://github.com/rust-lang/rust/tree/master/tests/assembly-llvm ### Codegen-units tests @@ -326,12 +331,8 @@ The tests in [`tests/codegen-units`] test the [monomorphization](../backend/monomorph.md) collector and CGU partitioning. These tests work by running `rustc` with a flag to print the result of the -monomorphization collection pass, and then special annotations in the file are -used to compare against that. - -Each test should be annotated with the `//@ -compile-flags:-Zprint-mono-items=VAL` directive with the appropriate `VAL` to -instruct `rustc` to print the monomorphization information. +monomorphization collection pass, i.e., `-Zprint-mono-items`, and then special +annotations in the file are used to compare against that. Then, the test should be annotated with comments of the form `//~ MONO_ITEM name` where `name` is the monomorphized string printed by rustc like `fn **Note on phasing out `Makefile`s** -> -> We are planning to migrate all existing Makefile-based `run-make` tests -> to Rust programs. You should not be adding new Makefile-based `run-make` -> tests. -> -> See . - The tests in [`tests/run-make`] are general-purpose tests using Rust *recipes*, which are small programs (`rmake.rs`) allowing arbitrary Rust code such as `rustc` invocations, and is supported by a [`run_make_support`] library. Using @@ -424,10 +417,9 @@ Compiletest directives like `//@ only-` or `//@ ignore-` are supported in `rmake.rs`, like in UI tests. However, revisions or building auxiliary via directives are not currently supported. -Two `run-make` tests are ported over to Rust recipes as examples: - -- -- +`rmake.rs` and `run-make-support` may *not* use any nightly/unstable features, +as they must be compilable by a stage 0 rustc that may be a beta or even stable +rustc. #### Quickly check if `rmake.rs` tests can be compiled @@ -452,7 +444,9 @@ To work around this when working on a particular test, temporarily create a with these contents:
+ Be careful not to add this `Cargo.toml` or its `Cargo.lock` to your actual PR! +
```toml @@ -481,20 +475,6 @@ Then add a corresponding entry to `"rust-analyzer.linkedProjects"` ], ``` -#### Using Makefiles (legacy) - -
-You should avoid writing new Makefile-based `run-make` tests. -
- -Each test should be in a separate directory with a `Makefile` indicating the -commands to run. - -There is a [`tools.mk`] Makefile which you can include which provides a bunch of -utilities to make it easier to run commands and compare outputs. Take a look at -some of the other tests for some examples on how to get started. - -[`tools.mk`]: https://github.com/rust-lang/rust/blob/master/tests/run-make/tools.mk [`tests/run-make`]: https://github.com/rust-lang/rust/tree/master/tests/run-make [`run_make_support`]: https://github.com/rust-lang/rust/tree/master/src/tools/run-make-support @@ -549,10 +529,10 @@ data into a human-readable code coverage report. Instrumented binaries need to be linked against the LLVM profiler runtime, so `coverage-run` tests are **automatically skipped** unless the profiler runtime -is enabled in `config.toml`: +is enabled in `bootstrap.toml`: ```toml -# config.toml +# bootstrap.toml [build] profiler = true ``` @@ -574,10 +554,10 @@ only running the main `coverage` suite. [`tests/crashes`] serve as a collection of tests that are expected to cause the compiler to ICE, panic or crash in some other way, so that accidental fixes are -tracked. This was formally done at but +tracked. Formerly, this was done at but doing it inside the rust-lang/rust testsuite is more convenient. -It is imperative that a test in the suite causes rustc to ICE, panic or crash +It is imperative that a test in the suite causes rustc to ICE, panic, or crash in some other way. A test will "pass" if rustc exits with an exit status other than 1 or 0. @@ -588,9 +568,12 @@ If you want to see verbose stdout/stderr, you need to set $ COMPILETEST_VERBOSE_CRASHES=1 ./x test tests/crashes/999999.rs --stage 1 ``` -When adding crashes from , the issue -number should be noted in the file name (`12345.rs` should suffice) and also -inside the file include a `//@ known-bug: #4321` directive. +Anyone can add ["untracked" crashes] from the issue tracker. It's strongly +recommended to include test cases from several issues in a single PR. +When you do so, each issue number should be noted in the file name (`12345.rs` +should suffice) and also inside the file by means of a `//@ known-bug: #12345` +directive. Please [label][labeling] the relevant issues with `S-bug-has-test` +afterwards. If you happen to fix one of the crashes, please move it to a fitting subdirectory in `tests/ui` and give it a meaningful name. Please add a doc @@ -613,6 +596,8 @@ a subset first. The issue numbers can be found in the file name or the `//@ known-bug` directive inside the test file. [`tests/crashes`]: https://github.com/rust-lang/rust/tree/master/tests/crashes +["untracked" crashes]: https://github.com/rust-lang/rust/issues?q=is%3Aissue+state%3Aopen+label%3AI-ICE%2CI-crash+label%3AT-compiler+label%3AS-has-mcve+-label%3AS-bug-has-test +[labeling]: https://forge.rust-lang.org/release/issue-triaging.html#applying-and-removing-labels ## Building auxiliary crates @@ -642,7 +627,7 @@ file). The `-L` flag is used to find the extern crates. `aux-crate` is very similar to `aux-build`. However, it uses the `--extern` flag to link to the extern crate to make the crate be available as an extern prelude. That allows you to specify the additional syntax of the `--extern` flag, such as -renaming a dependency. For example, `// aux-crate:foo=bar.rs` will compile +renaming a dependency. For example, `//@ aux-crate:foo=bar.rs` will compile `auxiliary/bar.rs` and make it available under then name `foo` within the test. This is similar to how Cargo does dependency renaming. diff --git a/src/tests/directives.md b/src/tests/directives.md index 69f4c8641..f4ba9a044 100644 --- a/src/tests/directives.md +++ b/src/tests/directives.md @@ -1,15 +1,11 @@ # Compiletest directives - - -Directives are special comments that tell compiletest how to build and interpret -a test. They must appear before the Rust source in the test. They may also -appear in `rmake.rs` or legacy Makefiles for [run-make -tests](compiletest.md#run-make-tests). +Directives are special comments that tell compiletest how to build and interpret a test. +They may also appear in `rmake.rs` [run-make tests](compiletest.md#run-make-tests). They are normally put after the short comment that explains the point of this test. Compiletest test suites use `//@` to signal that a comment is a directive. @@ -54,6 +50,8 @@ not be exhaustive. Directives can generally be found by browsing the ### Auxiliary builds +See [Building auxiliary crates](compiletest.html#building-auxiliary-crates) + | Directive | Explanation | Supported test suites | Possible values | |-----------------------|-------------------------------------------------------------------------------------------------------|-----------------------|-----------------------------------------------| | `aux-bin` | Build a aux binary, made available in `auxiliary/bin` relative to test directory | All except `run-make` | Path to auxiliary `.rs` file | @@ -61,10 +59,9 @@ not be exhaustive. Directives can generally be found by browsing the | `aux-crate` | Like `aux-build` but makes available as extern prelude | All except `run-make` | `=` | | `aux-codegen-backend` | Similar to `aux-build` but pass the compiled dylib to `-Zcodegen-backend` when building the main file | `ui-fulldeps` | Path to codegen backend file | | `proc-macro` | Similar to `aux-build`, but for aux forces host and don't use `-Cprefer-dynamic`[^pm]. | All except `run-make` | Path to auxiliary proc-macro `.rs` file | -| `build_aux_docs` | Build docs for auxiliaries as well | All except `run-make` | N/A | +| `build-aux-docs` | Build docs for auxiliaries as well. Note that this only works with `aux-build`, not `aux-crate`. | All except `run-make` | N/A | -[^pm]: please see the Auxiliary proc-macro section in the - [compiletest](./compiletest.md) chapter for specifics. +[^pm]: please see the [Auxiliary proc-macro section](compiletest.html#auxiliary-proc-macro) in the compiletest chapter for specifics. ### Controlling outcome expectations @@ -77,8 +74,10 @@ expectations](ui.md#controlling-passfail-expectations). | `check-fail` | Building (no codegen) should fail | `ui`, `crashes` | N/A | | `build-pass` | Building should pass | `ui`, `crashes`, `codegen`, `incremental` | N/A | | `build-fail` | Building should fail | `ui`, `crashes` | N/A | -| `run-pass` | Running the test binary should pass | `ui`, `crashes`, `incremental` | N/A | -| `run-fail` | Running the test binary should fail | `ui`, `crashes` | N/A | +| `run-pass` | Program must exit with code `0` | `ui`, `crashes`, `incremental` | N/A | +| `run-fail` | Program must exit with code `1..=127` | `ui`, `crashes` | N/A | +| `run-crash` | Program must crash | `ui` | N/A | +| `run-fail-or-crash` | Program must `run-fail` or `run-crash` | `ui` | N/A | | `ignore-pass` | Ignore `--pass` flag | `ui`, `crashes`, `codegen`, `incremental` | N/A | | `dont-check-failure-status` | Don't check exact failure status (i.e. `1`) | `ui`, `incremental` | N/A | | `failure-status` | Check | `ui`, `crashes` | Any `u16` | @@ -94,7 +93,8 @@ for more details. | Directive | Explanation | Supported test suites | Possible values | |-----------------------------------|--------------------------------------------------------------------------------------------------------------------------|----------------------------------------------|-----------------------------------------------------------------------------------------| | `check-run-results` | Check run test binary `run-{pass,fail}` output snapshot | `ui`, `crashes`, `incremental` if `run-pass` | N/A | -| `error-pattern` | Check that output contains a regex pattern | `ui`, `crashes`, `incremental` if `run-pass` | Regex | +| `error-pattern` | Check that output contains a specific string | `ui`, `crashes`, `incremental` if `run-pass` | String | +| `regex-error-pattern` | Check that output contains a regex pattern | `ui`, `crashes`, `incremental` if `run-pass` | Regex | | `check-stdout` | Check `stdout` against `error-pattern`s from running test binary[^check_stdout] | `ui`, `crashes`, `incremental` | N/A | | `normalize-stderr-32bit` | Normalize actual stderr (for 32-bit platforms) with a rule `"" -> ""` before comparing against snapshot | `ui`, `incremental` | `"" -> ""`, ``/`` is regex capture and replace syntax | | `normalize-stderr-64bit` | Normalize actual stderr (for 64-bit platforms) with a rule `"" -> ""` before comparing against snapshot | `ui`, `incremental` | `"" -> ""`, ``/`` is regex capture and replace syntax | @@ -102,6 +102,7 @@ for more details. | `normalize-stdout` | Normalize actual stdout with a rule `"" -> ""` before comparing against snapshot | `ui`, `incremental` | `"" -> ""`, ``/`` is regex capture and replace syntax | | `dont-check-compiler-stderr` | Don't check actual compiler stderr vs stderr snapshot | `ui` | N/A | | `dont-check-compiler-stdout` | Don't check actual compiler stdout vs stdout snapshot | `ui` | N/A | +| `dont-require-annotations` | Don't require line annotations for the given diagnostic kind (`//~ KIND`) to be exhaustive | `ui`, `incremental` | `ERROR`, `WARN`, `NOTE`, `HELP`, `SUGGESTION` | | `run-rustfix` | Apply all suggestions via `rustfix`, snapshot fixed output, and check fixed output builds | `ui` | N/A | | `rustfix-only-machine-applicable` | `run-rustfix` but only machine-applicable suggestions | `ui` | N/A | | `exec-env` | Env var to set when executing a test | `ui`, `crashes` | `=` | @@ -121,10 +122,12 @@ for more details. These directives are used to ignore the test in some situations, which means the test won't be compiled or run. -* `ignore-X` where `X` is a target detail or stage will ignore the test - accordingly (see below) +* `ignore-X` where `X` is a target detail or other criteria on which to ignore the test (see below) * `only-X` is like `ignore-X`, but will *only* run the test on that target or stage +* `ignore-auxiliary` is intended for files that *participate* in one or more other + main test files but that `compiletest` should not try to build the file itself. + Please backlink to which main test is actually using the auxiliary file. * `ignore-test` always ignores the test. This can be used to temporarily disable a test if it is currently not working, but you want to keep it in tree to re-enable it later. @@ -141,7 +144,8 @@ Some examples of `X` in `ignore-X` or `only-X`: matches that target as well as the emscripten targets. - Pointer width: `32bit`, `64bit` - Endianness: `endian-big` -- Stage: `stage0`, `stage1`, `stage2` +- Stage: `stage1`, `stage2` +- Binary format: `elf` - Channel: `stable`, `beta` - When cross compiling: `cross-compile` - When [remote testing] is used: `remote` @@ -151,6 +155,9 @@ Some examples of `X` in `ignore-X` or `only-X`: `compare-mode-split-dwarf`, `compare-mode-split-dwarf-single` - The two different test modes used by coverage tests: `ignore-coverage-map`, `ignore-coverage-run` +- When testing a dist toolchain: `dist` + - This needs to be enabled with `COMPILETEST_ENABLE_DIST_TESTS=1` +- The `rustc_abi` of the target: e.g. `rustc_abi-x86_64-sse2` The following directives will check rustc build settings and target settings: @@ -159,9 +166,9 @@ settings: stable support for `asm!` - `needs-profiler-runtime` — ignores the test if the profiler runtime was not enabled for the target - (`build.profiler = true` in rustc's `config.toml`) + (`build.profiler = true` in rustc's `bootstrap.toml`) - `needs-sanitizer-support` — ignores if the sanitizer support was not enabled - for the target (`sanitizers = true` in rustc's `config.toml`) + for the target (`sanitizers = true` in rustc's `bootstrap.toml`) - `needs-sanitizer-{address,hwaddress,leak,memory,thread}` — ignores if the corresponding sanitizer is not enabled for the target (AddressSanitizer, hardware-assisted AddressSanitizer, LeakSanitizer, MemorySanitizer or @@ -171,8 +178,9 @@ settings: flag, or running on fuchsia. - `needs-unwind` — ignores if the target does not support unwinding - `needs-rust-lld` — ignores if the rust lld support is not enabled (`rust.lld = - true` in `config.toml`) + true` in `bootstrap.toml`) - `needs-threads` — ignores if the target does not have threading support +- `needs-subprocess` — ignores if the target does not have subprocess support - `needs-symlink` — ignores if the target does not support symlinks. This can be the case on Windows if the developer did not enable privileged symlink permissions. @@ -188,10 +196,19 @@ settings: specified atomic widths, e.g. the test with `//@ needs-target-has-atomic: 8, 16, ptr` will only run if it supports the comma-separated list of atomic widths. +- `needs-dynamic-linking` — ignores if target does not support dynamic linking + (which is orthogonal to it being unable to create `dylib` and `cdylib` crate types) +- `needs-crate-type` — ignores if target platform does not support one or more + of the comma-delimited list of specified crate types. For example, + `//@ needs-crate-type: cdylib, proc-macro` will cause the test to be ignored + on `wasm32-unknown-unknown` target because the target does not support the + `proc-macro` crate type. +- `needs-target-std` — ignores if target platform does not have std support. +- `ignore-backends` — ignores the listed backends, separated by whitespace characters. +- `needs-backends` — only runs the test if current codegen backend is listed. The following directives will check LLVM support: -- `no-system-llvm` — ignores if the system llvm is used - `exact-llvm-major-version: 19` — ignores if the llvm major version does not match the specified llvm major version. - `min-llvm-version: 13.0` — ignored if the LLVM version is less than the given @@ -214,8 +231,6 @@ The following directives will check LLVM support: [`aarch64-gnu-debug`]), which only runs a subset of `run-make` tests. Other tests with this directive will not run at all, which is usually not what you want. - - Notably, the [`aarch64-gnu-debug`] CI job *currently* only runs `run-make` - tests which additionally contain `clang` in their test name. See also [Debuginfo tests](compiletest.md#debuginfo-tests) for directives for ignoring debuggers. @@ -227,28 +242,30 @@ ignoring debuggers. ### Affecting how tests are built -| Directive | Explanation | Supported test suites | Possible values | -|---------------------|----------------------------------------------------------------------------------------------|---------------------------|------------------------------------------------------------------------------| -| `compile-flags` | Flags passed to `rustc` when building the test or aux file | All except for `run-make` | Any valid `rustc` flags, e.g. `-Awarnings -Dfoo`. Cannot be `-Cincremental`. | -| `edition` | Alias for `compile-flags: --edition=xxx` | All except for `run-make` | Any valid `--edition` value | -| `rustc-env` | Env var to set when running `rustc` | All except for `run-make` | `=` | -| `unset-rustc-env` | Env var to unset when running `rustc` | All except for `run-make` | Any env var name | -| `incremental` | Proper incremental support for tests outside of incremental test suite | `ui`, `crashes` | N/A | -| `no-prefer-dynamic` | Don't use `-C prefer-dynamic`, don't build as a dylib via a `--crate-type=dylib` preset flag | `ui`, `crashes` | N/A | +| Directive | Explanation | Supported test suites | Possible values | +|---------------------|----------------------------------------------------------------------------------------------|---------------------------|--------------------------------------------------------------------------------------------| +| `compile-flags` | Flags passed to `rustc` when building the test or aux file | All except for `run-make` | Any valid `rustc` flags, e.g. `-Awarnings -Dfoo`. Cannot be `-Cincremental` or `--edition` | +| `edition` | The edition used to build the test | All except for `run-make` | Any valid `--edition` value | +| `rustc-env` | Env var to set when running `rustc` | All except for `run-make` | `=` | +| `unset-rustc-env` | Env var to unset when running `rustc` | All except for `run-make` | Any env var name | +| `incremental` | Proper incremental support for tests outside of incremental test suite | `ui`, `crashes` | N/A | +| `no-prefer-dynamic` | Don't use `-C prefer-dynamic`, don't build as a dylib via a `--crate-type=dylib` preset flag | `ui`, `crashes` | N/A |
+ Tests (outside of `run-make`) that want to use incremental tests not in the incremental test-suite must not pass `-C incremental` via `compile-flags`, and must instead use the `//@ incremental` directive. Consider writing the test as a proper incremental test instead. +
### Rustdoc | Directive | Explanation | Supported test suites | Possible values | |-------------|--------------------------------------------------------------|------------------------------------------|---------------------------| -| `doc-flags` | Flags passed to `rustdoc` when building the test or aux file | `rustdoc`, `js-doc-test`, `rustdoc-json` | Any valid `rustdoc` flags | +| `doc-flags` | Flags passed to `rustdoc` when building the test or aux file | `rustdoc`, `rustdoc-js`, `rustdoc-json` | Any valid `rustdoc` flags | +#### Test-suite-specific directives + +The test suites [`rustdoc`][rustdoc-html-tests], [`rustdoc-js`/`rustdoc-js-std`][rustdoc-js-tests] +and [`rustdoc-json`][rustdoc-json-tests] each feature an additional set of directives whose basic +syntax resembles the one of compiletest directives but which are ultimately read and checked by +separate tools. For more information, please read their respective chapters as linked above. + +[rustdoc-html-tests]: ../rustdoc-internals/rustdoc-test-suite.md +[rustdoc-js-tests]: ../rustdoc-internals/search.html#testing-the-search-engine +[rustdoc-json-tests]: ../rustdoc-internals/rustdoc-json-test-suite.md + ### Pretty printing See [Pretty-printer](compiletest.md#pretty-printer-tests). @@ -264,13 +292,12 @@ See [Pretty-printer](compiletest.md#pretty-printer-tests). - `no-auto-check-cfg` — disable auto check-cfg (only for `--check-cfg` tests) - [`revisions`](compiletest.md#revisions) — compile multiple times -- [`unused-revision-names`](compiletest.md#ignoring-unused-revision-names) - - suppress tidy checks for mentioning unknown revision names -[`forbid-output`](compiletest.md#incremental-tests) — incremental cfail rejects output pattern - [`should-ice`](compiletest.md#incremental-tests) — incremental cfail should ICE - [`reference`] — an annotation linking to a rule in the reference +- `disable-gdb-pretty-printers` — disable gdb pretty printers for debuginfo tests [`reference`]: https://github.com/rust-lang/reference/blob/master/docs/authoring.md#test-rule-annotations @@ -286,6 +313,17 @@ test suites that use those tools: - `llvm-cov-flags` adds extra flags when running LLVM's `llvm-cov` tool. - Used by [coverage tests](compiletest.md#coverage-tests) in `coverage-run` mode. +### Tidy specific directives + +The following directives control how the [tidy script](../conventions.md#formatting) +verifies tests. + +- `ignore-tidy-target-specific-tests` disables checking that the appropriate + LLVM component is required (via a `needs-llvm-components` directive) when a + test is compiled for a specific target (via the `--target` flag in a + `compile-flag` directive). +- [`unused-revision-names`](compiletest.md#ignoring-unused-revision-names) - + suppress tidy checks for mentioning unknown revision names. ## Substitutions @@ -320,7 +358,7 @@ described below: - Example: `x86_64-unknown-linux-gnu` See -[`tests/ui/commandline-argfile.rs`](https://github.com/rust-lang/rust/blob/master/tests/ui/argfile/commandline-argfile.rs) +[`tests/ui/argfile/commandline-argfile.rs`](https://github.com/rust-lang/rust/blob/master/tests/ui/argfile/commandline-argfile.rs) for an example of a test that uses this substitution. [output normalization]: ui.md#normalization diff --git a/src/tests/docker.md b/src/tests/docker.md index 31e3825f5..ae0939842 100644 --- a/src/tests/docker.md +++ b/src/tests/docker.md @@ -1,36 +1,44 @@ # Testing with Docker -The Rust tree includes [Docker] image definitions for the platforms used on -GitHub Actions in [`src/ci/docker`]. -The script [`src/ci/docker/run.sh`] is used to build the Docker image, run it, -build Rust within the image, and run the tests. - -You can run these images on your local development machine. This can be -helpful to test environments different from your local system. First you will +The [`src/ci/docker`] directory includes [Docker] image definitions for Linux-based jobs executed on GitHub Actions (non-Linux jobs run outside Docker). You can run these jobs on your local development machine, which can be +helpful to test environments different from your local system. You will need to install Docker on a Linux, Windows, or macOS system (typically Linux will be much faster than Windows or macOS because the latter use virtual -machines to emulate a Linux environment). To enter interactive mode which will -start a bash shell in the container, run `src/ci/docker/run.sh --dev ` -where `` is one of the directory names in `src/ci/docker` (for example -`x86_64-gnu` is a fairly standard Ubuntu environment). - -The docker script will mount your local Rust source tree in read-only mode, -and an `obj` directory in read-write mode. All of the compiler artifacts will -be stored in the `obj` directory. The shell will start out in the `obj` -directory. From there, you can run `../src/ci/run.sh` which will run the build -as defined by the image. - -Alternatively, you can run individual commands to do specific tasks. For -example, you can run `../x test tests/ui` to just run UI tests. -Note that there is some configuration in the [`src/ci/run.sh`] script that you -may need to recreate. Particularly, set `submodules = false` in your -`config.toml` so that it doesn't attempt to modify the read-only directory. +machines to emulate a Linux environment). + +Jobs running in CI are configured through a set of bash scripts, and it is not always trivial to reproduce their behavior locally. If you want to run a CI job locally in the simplest way possible, you can use a provided helper `citool` that tries to replicate what happens on CI as closely as possible: -Some additional notes about using the Docker images: +```bash +cargo run --manifest-path src/ci/citool/Cargo.toml run-local +# For example: +cargo run --manifest-path src/ci/citool/Cargo.toml run-local dist-x86_64-linux-alt +``` +If the above script does not work for you, you would like to have more control of the Docker image execution, or you want to understand what exactly happens during Docker job execution, then continue reading below. + +## The `run.sh` script +The [`src/ci/docker/run.sh`] script is used to build a specific Docker image, run it, +build Rust within the image, and either run tests or prepare a set of archives designed for distribution. The script will mount your local Rust source tree in read-only mode, and an `obj` directory in read-write mode. All the compiler artifacts will be stored in the `obj` directory. The shell will start out in the `obj`directory. From there, it will execute `../src/ci/run.sh` which starts the build as defined by the Docker image. + +You can run `src/ci/docker/run.sh ` directly. A few important notes regarding the `run.sh` script: +- When executed on CI, the script expects that all submodules are checked out. If some submodule that is accessed by the job is not available, the build will result in an error. You should thus make sure that you have all required submodules checked out locally. You can either do that manually through git, or set `submodules = true` in your `bootstrap.toml` and run a command such as `x build` to let bootstrap download the most important submodules (this might not be enough for the given CI job that you are trying to execute though). +- `` corresponds to a single directory located in one of the `src/ci/docker/host-*` directories. Note that image name does not necessarily correspond to a job name, as some jobs execute the same image, but with different environment variables or Docker build arguments (this is a part of the complexity that makes it difficult to run CI jobs locally). +- If you are executing a "dist" job (job beginning with `dist-`), you should set the `DEPLOY=1` environment variable. +- If you are executing an "alternative dist" job (job beginning with `dist-` and ending with `-alt`), you should set the `DEPLOY_ALT=1` environment variable. - Some of the std tests require IPv6 support. Docker on Linux seems to have it disabled by default. Run the commands in [`enable-docker-ipv6.sh`] to enable IPv6 before creating the container. This only needs to be done once. + +### Interactive mode + +Sometimes, it can be useful to build a specific Docker image, and then run custom commands inside it, so that you can experiment with how the given system behaves. You can do that using an interactive mode, which will +start a bash shell in the container, using `src/ci/docker/run.sh --dev `. + +When inside the Docker container, you can run individual commands to do specific tasks. For +example, you can run `../x test tests/ui` to just run UI tests. + +Some additional notes about using the interactive mode: + - The container will be deleted automatically when you exit the shell, however the build artifacts persist in the `obj` directory. If you are switching between different Docker images, the artifacts from previous environments diff --git a/src/tests/fuchsia.md b/src/tests/ecosystem-test-jobs/fuchsia.md similarity index 96% rename from src/tests/fuchsia.md rename to src/tests/ecosystem-test-jobs/fuchsia.md index e96290b92..b19d94d6f 100644 --- a/src/tests/fuchsia.md +++ b/src/tests/ecosystem-test-jobs/fuchsia.md @@ -4,6 +4,14 @@ million lines of Rust code.[^loc] It has caught a large number of [regressions] in the past and was subsequently included in CI. +## What to do if the Fuchsia job breaks? + +Please contact the [fuchsia][fuchsia-ping] ping group and ask them for help. + +```text +@rustbot ping fuchsia +``` + ## Building Fuchsia in CI Fuchsia builds as part of the suite of bors tests that run before a pull request @@ -32,7 +40,7 @@ using your local Rust toolchain. src/ci/docker/run.sh x86_64-fuchsia ``` -See the [Testing with Docker](docker.md) chapter for more details on how to run +See the [Testing with Docker](../docker.md) chapter for more details on how to run and debug jobs with Docker. Note that a Fuchsia checkout is *large* – as of this writing, a checkout and @@ -162,6 +170,7 @@ rustc book][platform-support]. [`public_configs`]: https://gn.googlesource.com/gn/+/main/docs/reference.md#var_public_configs [`//build/config:compiler`]: https://cs.opensource.google/fuchsia/fuchsia/+/main:build/config/BUILD.gn;l=121;drc=c26c473bef93b33117ae417893118907a026fec7 [build system]: https://fuchsia.dev/fuchsia-src/development/build/build_system +[fuchsia-ping]: ../../notification-groups/fuchsia.md [^loc]: As of June 2024, Fuchsia had about 2 million lines of first-party Rust code and a roughly equal amount of third-party code, as counted by tokei diff --git a/src/tests/rust-for-linux.md b/src/tests/ecosystem-test-jobs/rust-for-linux.md similarity index 95% rename from src/tests/rust-for-linux.md rename to src/tests/ecosystem-test-jobs/rust-for-linux.md index c674d1575..d549ec6fc 100644 --- a/src/tests/rust-for-linux.md +++ b/src/tests/ecosystem-test-jobs/rust-for-linux.md @@ -3,26 +3,7 @@ [Rust for Linux](https://rust-for-linux.com/) (RfL) is an effort for adding support for the Rust programming language into the Linux kernel. -## Building Rust for Linux in CI - -Rust for Linux builds as part of the suite of bors tests that run before a pull -request is merged. - -The workflow builds a stage1 sysroot of the Rust compiler, downloads the Linux -kernel, and tries to compile several Rust for Linux drivers and examples using -this sysroot. RfL uses several unstable compiler/language features, therefore -this workflow notifies us if a given compiler change would break it. - -If you are worried that a pull request might break the Rust for Linux builder -and want to test it out before submitting it to the bors queue, simply add this -line to your PR description: - -> try-job: x86_64-rust-for-linux - -Then when you `@bors try` it will pick the job that builds the Rust for Linux -integration. - -## What to do in case of failure +## What to do if the Rust for Linux job breaks? If a PR breaks the Rust for Linux CI job, then: @@ -48,4 +29,23 @@ ping group to ask for help: @rustbot ping rfl ``` -[rfl-ping]: ../notification-groups/rust-for-linux.md +## Building Rust for Linux in CI + +Rust for Linux builds as part of the suite of bors tests that run before a pull +request is merged. + +The workflow builds a stage1 sysroot of the Rust compiler, downloads the Linux +kernel, and tries to compile several Rust for Linux drivers and examples using +this sysroot. RfL uses several unstable compiler/language features, therefore +this workflow notifies us if a given compiler change would break it. + +If you are worried that a pull request might break the Rust for Linux builder +and want to test it out before submitting it to the bors queue, simply add this +line to your PR description: + +> try-job: x86_64-rust-for-linux + +Then when you `@bors try` it will pick the job that builds the Rust for Linux +integration. + +[rfl-ping]: ../../notification-groups/rust-for-linux.md diff --git a/src/tests/ecosystem.md b/src/tests/ecosystem.md index 083601404..eee07dd07 100644 --- a/src/tests/ecosystem.md +++ b/src/tests/ecosystem.md @@ -15,14 +15,16 @@ CI. See the [Crater chapter](crater.md) for more details. `cargotest` is a small tool which runs `cargo test` on a few sample projects (such as `servo`, `ripgrep`, `tokei`, etc.). This runs as part of CI and ensures -there aren't any significant regressions. +there aren't any significant regressions: -> Example: `./x test src/tools/cargotest` +```console +./x test src/tools/cargotest +``` ### Large OSS Project builders We have CI jobs that build large open-source Rust projects that are used as regression tests in CI. Our integration jobs build the following projects: -- [Fuchsia](fuchsia.md) -- [Rust for Linux](rust-for-linux.md) +- [Fuchsia](./ecosystem-test-jobs/fuchsia.md) +- [Rust for Linux](./ecosystem-test-jobs/rust-for-linux.md) diff --git a/src/tests/intro.md b/src/tests/intro.md index ba44a969b..b90c16d60 100644 --- a/src/tests/intro.md +++ b/src/tests/intro.md @@ -1,7 +1,5 @@ # Testing the compiler - - The Rust project runs a wide variety of different tests, orchestrated by the build system (`./x test`). This section gives a brief overview of the different testing tools. Subsequent chapters dive into [running tests](running.md) and @@ -38,7 +36,7 @@ directory, and `x` will essentially run `cargo test` on that package. Examples: | Command | Description | -| ----------------------------------------- | ------------------------------------- | +|-------------------------------------------|---------------------------------------| | `./x test library/std` | Runs tests on `std` only | | `./x test library/core` | Runs tests on `core` only | | `./x test compiler/rustc_data_structures` | Runs tests on `rustc_data_structures` | @@ -86,7 +84,7 @@ above. Examples: | Command | Description | -| ----------------------- | ------------------------------------------------------------------ | +|-------------------------|--------------------------------------------------------------------| | `./x fmt --check` | Checks formatting and exits with an error if formatting is needed. | | `./x fmt` | Runs rustfmt across the entire codebase. | | `./x test tidy --bless` | First runs rustfmt to format the codebase, then runs tidy checks. | @@ -102,20 +100,23 @@ by passing a path to a book to `./x test`. ### Documentation link checker -Links across all documentation is validated with a link checker tool. - -> Example: `./x test src/tools/linkchecker` +Links across all documentation is validated with a link checker tool, +and it can be invoked so: -> Example: `./x test linkchecker` +```console +./x test linkchecker +``` This requires building all of the documentation, which might take a while. -### Dist check +### `distcheck` `distcheck` verifies that the source distribution tarball created by the build system will unpack, build, and run all tests. -> Example: `./x test distcheck` +```console +./x test distcheck +``` ### Tool tests @@ -155,6 +156,10 @@ chapter](ecosystem.md) for more details. A separate infrastructure is used for testing and tracking performance of the compiler. See the [Performance testing chapter](perf.md) for more details. +### Codegen backend testing + +See [Codegen backend testing](./codegen-backend-tests/intro.md). + ## Miscellaneous information There are some other useful testing-related info at [Misc info](misc.md). diff --git a/src/tests/minicore.md b/src/tests/minicore.md index e4853b6d4..23b772790 100644 --- a/src/tests/minicore.md +++ b/src/tests/minicore.md @@ -6,24 +6,44 @@ ui/codegen/assembly test suites. It provides `core` stubs for tests that need to build for cross-compiled targets but do not need/want to run. +
+ +Please note that [`minicore`] is only intended for `core` items, and explicitly +**not** `std` or `alloc` items because `core` items are applicable to a wider +range of tests. + +
+ A test can use [`minicore`] by specifying the `//@ add-core-stubs` directive. Then, mark the test with `#![feature(no_core)]` + `#![no_std]` + `#![no_core]`. Due to Edition 2015 extern prelude rules, you will probably need to declare `minicore` as an extern crate. +## Implied compiler flags + Due to the `no_std` + `no_core` nature of these tests, `//@ add-core-stubs` implies and requires that the test will be built with `-C panic=abort`. -Unwinding panics are not supported. +**Unwinding panics are not supported.** + +Tests will also be built with `-C force-unwind-tables=yes` to preserve CFI +directives in assembly tests. + +TL;DR: `//@ add-core-stubs` implies two compiler flags: + +1. `-C panic=abort` +2. `-C force-unwind-tables=yes` + +## Adding more `core` stubs If you find a `core` item to be missing from the [`minicore`] stub, consider adding it to the test auxiliary if it's likely to be used or is already needed by more than one test. -
-Please note that [`minicore`] is only intended for `core` items, and explicitly -**not** `std` or `alloc` items because `core` items are applicable to a wider -range of tests. -
+## Staying in sync with `core` + +The `minicore` items must be kept up to date with `core`. For consistent +diagnostic output between using `core` and `minicore`, any `diagnostic` +attributes (e.g. `on_unimplemented`) should be replicated exactly in `minicore`. ## Example codegen test that uses `minicore` diff --git a/src/tests/misc.md b/src/tests/misc.md index c0288b3dd..39f881748 100644 --- a/src/tests/misc.md +++ b/src/tests/misc.md @@ -9,7 +9,7 @@ for testing: - `RUSTC_BOOTSTRAP=1` will "cheat" and bypass usual stability checking, allowing you to use unstable features and cli flags on a stable `rustc`. -- `RUSTC_BOOTSTRAP=-1` will force a given `rustc` to pretend that is a stable +- `RUSTC_BOOTSTRAP=-1` will force a given `rustc` to pretend it is a stable compiler, even if it's actually a nightly `rustc`. This is useful because some behaviors of the compiler (e.g. diagnostics) can differ depending on whether the compiler is nightly or not. diff --git a/src/tests/running.md b/src/tests/running.md index 6ce650923..317b65f98 100644 --- a/src/tests/running.md +++ b/src/tests/running.md @@ -1,13 +1,12 @@ # Running tests - - You can run the entire test collection using `x`. But note that running the *entire* test collection is almost never what you want to do during local development because it takes a really long time. For local development, see the subsection after on how to run a subset of tests.
+ Running plain `./x test` will build the stage 1 compiler and then run the whole test suite. This not only include `tests/`, but also `library/`, `compiler/`, `src/tools/` package tests and more. @@ -16,16 +15,17 @@ You usually only want to run a subset of the test suites (or even a smaller set of tests than that) which you expect will exercise your changes. PR CI exercises a subset of test collections, and merge queue CI will exercise all of the test collection. +
-```bash +```text ./x test ``` The test results are cached and previously successful tests are `ignored` during testing. The stdout/stderr contents as well as a timestamp file for every test -can be found under `build//test/` for the given -``. To force-rerun a test (e.g. in case the test runner fails to +can be found under `build//test/` for the given +``. To force-rerun a test (e.g. in case the test runner fails to notice a change) you can use the `--force-rerun` CLI option. > **Note on requirements of external dependencies** @@ -45,22 +45,22 @@ tests. For example, a good "smoke test" that can be used after modifying rustc to see if things are generally working correctly would be to exercise the `ui` test suite ([`tests/ui`]): -```bash +```text ./x test tests/ui ``` -This will run the `ui` test suite. Of course, the choice of test suites is +Of course, the choice of test suites is somewhat arbitrary, and may not suit the task you are doing. For example, if you are hacking on debuginfo, you may be better off with the debuginfo test suite: -```bash +```text ./x test tests/debuginfo ``` If you only need to test a specific subdirectory of tests for any given test suite, you can pass that directory as a filter to `./x test`: -```bash +```text ./x test tests/ui/const-generics ``` @@ -73,7 +73,7 @@ suite, you can pass that directory as a filter to `./x test`: Likewise, you can test a single file by passing its path: -```bash +```text ./x test tests/ui/const-generics/const-test.rs ``` @@ -81,19 +81,19 @@ Likewise, you can test a single file by passing its path: have to use the `--test-args` argument as described [below](#running-an-individual-test). -```bash +```text ./x test src/tools/miri --test-args tests/fail/uninit/padding-enum.rs ``` ### Run only the tidy script -```bash +```text ./x test tidy ``` ### Run tests on the standard library -```bash +```text ./x test --stage 0 library/std ``` @@ -102,27 +102,29 @@ crates, you have to specify those explicitly. ### Run the tidy script and tests on the standard library -```bash +```text ./x test --stage 0 tidy library/std ``` ### Run tests on the standard library using a stage 1 compiler -```bash +```text ./x test --stage 1 library/std ``` -By listing which test suites you want to run you avoid having to run tests for -components you did not change at all. +By listing which test suites you want to run, +you avoid having to run tests for components you did not change at all.
+ Note that bors only runs the tests with the full stage 2 build; therefore, while the tests **usually** work fine with stage 1, there are some limitations. +
### Run all tests using a stage 2 compiler -```bash +```text ./x test --stage 2 ``` @@ -134,13 +136,13 @@ You almost never need to do this; CI will run these tests for you. You may want to run unit tests on a specific file with following: -```bash +```text ./x test compiler/rustc_data_structures/src/thin_vec/tests.rs ``` But unfortunately, it's impossible. You should invoke the following instead: -```bash +```text ./x test compiler/rustc_data_structures/ --test-args thin_vec ``` @@ -151,7 +153,7 @@ often the test they are trying to fix. As mentioned earlier, you may pass the full file path to achieve this, or alternatively one may invoke `x` with the `--test-args` option: -```bash +```text ./x test tests/ui --test-args issue-1234 ``` @@ -172,25 +174,27 @@ additional arguments to the compiler when building the tests. ## Editing and updating the reference files If you have changed the compiler's output intentionally, or you are making a new -test, you can pass `--bless` to the test subcommand. E.g. if some tests in -`tests/ui` are failing, you can run +test, you can pass `--bless` to the test subcommand. + +As an example, +if some tests in `tests/ui` are failing, you can run this command: ```text ./x test tests/ui --bless ``` -to automatically adjust the `.stderr`, `.stdout` or `.fixed` files of -all tests. Of course you can also target just specific tests with the -`--test-args your_test_name` flag, just like when running the tests. +It automatically adjusts the `.stderr`, `.stdout`, or `.fixed` files of all `test/ui` tests. +Of course you can also target just specific tests with the `--test-args your_test_name` flag, +just like when running the tests without the `--bless` flag. ## Configuring test running There are a few options for running tests: -* `config.toml` has the `rust.verbose-tests` option. If `false`, each test will +* `bootstrap.toml` has the `rust.verbose-tests` option. If `false`, each test will print a single dot (the default). If `true`, the name of every test will be printed. This is equivalent to the `--quiet` option in the [Rust test - harness](https://doc.rust-lang.org/rustc/tests/) + harness](https://doc.rust-lang.org/rustc/tests/). * The environment variable `RUST_TEST_THREADS` can be set to the number of concurrent threads to use for testing. @@ -201,7 +205,7 @@ When `--pass $mode` is passed, these tests will be forced to run under the given `$mode` unless the directive `//@ ignore-pass` exists in the test file. For example, you can run all the tests in `tests/ui` as `check-pass`: -```bash +```text ./x test tests/ui --pass check ``` @@ -217,7 +221,7 @@ first look for expected output in `foo.polonius.stderr`, falling back to the usual `foo.stderr` if not found. The following will run the UI test suite in Polonius mode: -```bash +```text ./x test tests/ui --compare-mode=polonius ``` @@ -230,7 +234,7 @@ just `.rs` files, so after [creating a rustup toolchain](../building/how-to-build-and-run.md#creating-a-rustup-toolchain), you can do something like: -```bash +```text rustc +stage1 tests/ui/issue-1234.rs ``` @@ -238,30 +242,6 @@ This is much faster, but doesn't always work. For example, some tests include directives that specify specific compiler flags, or which rely on other crates, and they may not run the same without those options. -## Running `run-make` tests - -### Windows - -Running the `run-make` test suite on Windows is a currently bit more involved. -There are numerous prerequisites and environmental requirements: - -- Install msys2: -- Specify `MSYS2_PATH_TYPE=inherit` in `msys2.ini` in the msys2 installation directory, run the - following with `MSYS2 MSYS`: - - `pacman -Syuu` - - `pacman -S make` - - `pacman -S diffutils` - - `pacman -S binutils` - - `./x test run-make` (`./x test tests/run-make` doesn't work) - -There is [on-going work][port-run-make] to not rely on `Makefile`s in the -run-make test suite. Once this work is completed, you can run the entire -`run-make` test suite on native Windows inside `cmd` or `PowerShell` without -needing to install and use MSYS2. As of Oct 2024, it is -already possible to run the vast majority of the `run-make` test suite outside -of MSYS2, but there will be failures for the tests that still use `Makefile`s -due to not finding `make`. - ## Running tests on a remote machine Tests may be run on a remote machine (e.g. to test builds for a different @@ -274,7 +254,7 @@ execution* so be careful where it is used. To do this, first build `remote-test-server` for the remote machine, e.g. for RISC-V -```sh +```text ./x build src/tools/remote-test-server --target riscv64gc-unknown-linux-gnu ``` @@ -286,7 +266,7 @@ On the remote machine, run the `remote-test-server` with the `--bind 0.0.0.0:12345` flag (and optionally `-v` for verbose output). Output should look like this: -```sh +```text $ ./remote-test-server -v --bind 0.0.0.0:12345 starting test server listening on 0.0.0.0:12345! @@ -300,7 +280,7 @@ restrictive IP address when binding. You can test if the `remote-test-server` is working by connecting to it and sending `ping\n`. It should reply `pong`: -```sh +```text $ nc $REMOTE_IP 12345 ping pong @@ -310,7 +290,7 @@ To run tests using the remote runner, set the `TEST_DEVICE_ADDR` environment variable then use `x` as usual. For example, to run `ui` tests for a RISC-V machine with the IP address `1.2.3.4` use -```sh +```text export TEST_DEVICE_ADDR="1.2.3.4:12345" ./x test tests/ui --target riscv64gc-unknown-linux-gnu ``` @@ -318,7 +298,7 @@ export TEST_DEVICE_ADDR="1.2.3.4:12345" If `remote-test-server` was run with the verbose flag, output on the test machine may look something like -``` +```text [...] run "/tmp/work/test1007/a" run "/tmp/work/test1008/a" @@ -362,7 +342,6 @@ coordinate running tests (see [src/bootstrap/src/core/build_steps/test.rs]). > **TODO** > > - Is there any support for using an iOS emulator? -> - It's also unclear to me how the wasm or asm.js tests are run. [armhf-gnu]: https://github.com/rust-lang/rust/tree/master/src/ci/docker/host-x86_64/armhf-gnu/Dockerfile [QEMU]: https://www.qemu.org/ @@ -370,12 +349,34 @@ coordinate running tests (see [src/bootstrap/src/core/build_steps/test.rs]). [remote-test-server]: https://github.com/rust-lang/rust/tree/master/src/tools/remote-test-server [src/bootstrap/src/core/build_steps/test.rs]: https://github.com/rust-lang/rust/blob/master/src/bootstrap/src/core/build_steps/test.rs +## Testing tests on wasi (wasm32-wasip1) + +Some tests are specific to wasm targets. +To run theste tests, you have to pass `--target wasm32-wasip1` to `x test`. +Additionally, you need the wasi sdk. +Follow the install instructions from the [wasi sdk repository] to get a sysroot on your computer. +On the [wasm32-wasip1 target support page] a minimum version is specified that your sdk must be able to build. +Some cmake commands that take a while and give a lot of very concerning c++ warnings... +Then, in `bootstrap.toml`, point to the sysroot like so: + +``` +[target.wasm32-wasip1] +wasi-root = "/build/sysroot/install/share/wasi-sysroot" +``` + +In my case I git-cloned it next to my rust folder, so it was `../wasi-sdk/build/....` +Now, tests should just run, you don't have to set up anything else. + +[wasi sdk repository]: https://github.com/WebAssembly/wasi-sdk +[wasm32-wasip1 target support page]: https://github.com/rust-lang/rust/blob/master/src/doc/rustc/src/platform-support/wasm32-wasip1.md#building-the-target. + + ## Running rustc_codegen_gcc tests First thing to know is that it only supports linux x86_64 at the moment. We will extend its support later on. -You need to update `codegen-backends` value in your `config.toml` file in the +You need to update `codegen-backends` value in your `bootstrap.toml` file in the `[rust]` section and add "gcc" in the array: ```toml @@ -384,21 +385,21 @@ codegen-backends = ["llvm", "gcc"] Then you need to install libgccjit 12. For example with `apt`: -```bash -$ apt install libgccjit-12-dev +```text +apt install libgccjit-12-dev ``` Now you can run the following command: -```bash -$ ./x test compiler/rustc_codegen_gcc/ +```text +./x test compiler/rustc_codegen_gcc/ ``` If it cannot find the `.so` library (if you installed it with `apt` for example), you need to pass the library file path with `LIBRARY_PATH`: -```bash -$ LIBRARY_PATH=/usr/lib/gcc/x86_64-linux-gnu/12/ ./x test compiler/rustc_codegen_gcc/ +```text +LIBRARY_PATH=/usr/lib/gcc/x86_64-linux-gnu/12/ ./x test compiler/rustc_codegen_gcc/ ``` If you encounter bugs or problems, don't hesitate to open issues on the @@ -406,4 +407,3 @@ If you encounter bugs or problems, don't hesitate to open issues on the repository](https://github.com/rust-lang/rustc_codegen_gcc/). [`tests/ui`]: https://github.com/rust-lang/rust/tree/master/tests/ui -[port-run-make]: https://github.com/rust-lang/rust/issues/121876 diff --git a/src/tests/suggest-tests.md b/src/tests/suggest-tests.md deleted file mode 100644 index 663e8a5af..000000000 --- a/src/tests/suggest-tests.md +++ /dev/null @@ -1,59 +0,0 @@ -# Suggest tests tool - -This chapter is about the internals of and contribution instructions for the -`suggest-tests` tool. For a high-level overview of the tool, see [this -section](../building/suggested.md#x-suggest). This tool is currently in a beta -state and is tracked by [this](https://github.com/rust-lang/rust/issues/109933) -issue on Github. Currently the number of tests it will suggest are very limited -in scope, we are looking to expand this (contributions welcome!). - -## Internals - -The tool is defined in a separate crate -([`src/tools/suggest-tests`](https://github.com/rust-lang/rust/blob/master/src/tools/suggest-tests)) -which outputs suggestions which are parsed by a shim in bootstrap -([`src/bootstrap/src/core/build_steps/suggest.rs`](https://github.com/rust-lang/rust/blob/master/src/bootstrap/src/core/build_steps/suggest.rs)). -The only notable thing the bootstrap shim does is (when invoked with the `--run` -flag) use bootstrap's internal mechanisms to create a new `Builder` and uses it -to invoke the suggested commands. The `suggest-tests` crate is where the fun -happens, two kinds of suggestions are defined: "static" and "dynamic" -suggestions. - -### Static suggestions - -Defined -[here](https://github.com/rust-lang/rust/blob/master/src/tools/suggest-tests/src/static_suggestions.rs). -Static suggestions are simple: they are just -[globs](https://crates.io/crates/glob) which map to a `x` command. In -`suggest-tests`, this is implemented with a simple `macro_rules` macro. - -### Dynamic suggestions - -Defined -[here](https://github.com/rust-lang/rust/blob/master/src/tools/suggest-tests/src/dynamic_suggestions.rs). -These are more complicated than static suggestions and are implemented as -functions with the following signature: `fn(&Path) -> Vec`. In other -words, each suggestion takes a path to a modified file and (after running -arbitrary Rust code) can return any number of suggestions, or none. Dynamic -suggestions are useful for situations where fine-grained control over -suggestions is needed. For example, modifications to the `compiler/xyz/` path -should trigger the `x test compiler/xyz` suggestion. In the future, dynamic -suggestions might even read file contents to determine if (what) tests should -run. - -## Adding a suggestion - -The following steps should serve as a rough guide to add suggestions to -`suggest-tests` (very welcome!): - -1. Determine the rules for your suggestion. Is it simple and operates only on a - single path or does it match globs? Does it need fine-grained control over - the resulting command or does "one size fit all"? -2. Based on the previous step, decide if your suggestion should be implemented - as either static or dynamic. -3. Implement the suggestion. If it is dynamic then a test is highly recommended, - to verify that your logic is correct and to give an example of the - suggestion. See the - [tests.rs](https://github.com/rust-lang/rust/blob/master/src/tools/suggest-tests/src/tests.rs) - file. -4. Open a PR implementing your suggestion. **(TODO: add example PR)** diff --git a/src/tests/ui.md b/src/tests/ui.md index c8536b004..25dd5814c 100644 --- a/src/tests/ui.md +++ b/src/tests/ui.md @@ -1,7 +1,5 @@ # UI tests - - UI tests are a particular [test suite](compiletest.md#test-suites) of compiletest. @@ -13,6 +11,11 @@ used for many other purposes. For example, tests can also be configured to [run the resulting program](#controlling-passfail-expectations) to verify its behavior. +For a survey of each subdirectory's purpose under `tests/ui`, consult the +[SUMMARY.md](https://github.com/rust-lang/rust/tree/master/tests/ui/SUMMARY.md). +This is useful if you write a new test, and are looking for a category to +place it in. + If you need to work with `#![no_std]` cross-compiling tests, consult the [`minicore` test auxiliary](./minicore.md) chapter. @@ -20,9 +23,9 @@ If you need to work with `#![no_std]` cross-compiling tests, consult the ## General structure of a test -A test consists of a Rust source file located anywhere in the `tests/ui` -directory, but they should be placed in a suitable sub-directory. For example, -[`tests/ui/hello.rs`] is a basic hello-world test. +A test consists of a Rust source file located in the `tests/ui` directory. +**Tests must be placed in the appropriate subdirectory** based on their purpose +and testing category - placing tests directly in `tests/ui` is not permitted. Compiletest will use `rustc` to compile the test, and compare the output against the expected output which is stored in a `.stdout` or `.stderr` file located @@ -41,8 +44,6 @@ pass/fail expectations](#controlling-passfail-expectations). By default, a test is built as an executable binary. If you need a different crate type, you can use the `#![crate_type]` attribute to set it as needed. -[`tests/ui/hello.rs`]: https://github.com/rust-lang/rust/blob/master/tests/ui/hello.rs - ## Output comparison UI tests store the expected output from the compiler in `.stderr` and `.stdout` @@ -54,6 +55,11 @@ The output is normalized to ignore unwanted differences, see the [Normalization](#normalization) section. If the file is missing, then compiletest expects the corresponding output to be empty. +A common reason to use normalization, revisions, and most of the other following tools, +is to account for platform differences. Consider alternatives to these tools, like +e.g. using the `extern "rust-invalid"` ABI that is invalid on every platform +instead of fixing the test to use cross-compilation and testing every possibly-invalid ABI. + There can be multiple stdout/stderr files. The general form is: ```text @@ -113,6 +119,8 @@ Compiletest makes the following replacements on the compiler output: - The base directory where the test's output goes is replaced with `$TEST_BUILD_DIR`. This only comes up in a few rare circumstances. Example: `/path/to/rust/build/x86_64-unknown-linux-gnu/test/ui` +- The real directory to the standard library source is replaced with `$SRC_DIR_REAL`. +- The real directory to the compiler source is replaced with `$COMPILER_DIR_REAL`. - Tabs are replaced with `\t`. - Backslashes (`\`) are converted to forward slashes (`/`) within paths (using a heuristic). This helps normalize differences with Windows-style paths. @@ -192,7 +200,7 @@ They have several forms, but generally are a comment with the diagnostic level to write out the entire message, just make sure to include the important part of the message to make it self-documenting. -The error annotation needs to match with the line of the diagnostic. There are +Most error annotations need to match with the line of the diagnostic. There are several ways to match the message with the line (see the examples below): * `~`: Associates the error level and message with the *current* line @@ -202,6 +210,9 @@ several ways to match the message with the line (see the examples below): * `~|`: Associates the error level and message with the *same* line as the *previous comment*. This is more convenient than using multiple carets when there are multiple messages associated with the same line. +* `~v`: Associates the error level and message with the *next* error + annotation line. Each symbol (`v`) that you add adds a line to this, so `~vvv` + is three lines below the error annotation line. Example: @@ -216,6 +227,14 @@ The space character between `//~` (or other variants) and the subsequent text is negligible (i.e. there is no semantic difference between `//~ ERROR` and `//~ERROR` although the former is more common in the codebase). +`~? ` (example being `~? ERROR`) +is used to match diagnostics _without_ line info at all, +or where the line info is outside the main test file[^main test file]. +These annotations can be placed on any line in the test file. + +[^main test file]: This is a file that has the `~?` annotations, +as distinct from aux files, or sources that we have no control over. + ### Error annotation examples Here are examples of error annotations on different lines of UI test source. @@ -270,10 +289,36 @@ fn main() { //~| ERROR this pattern has 1 field, but the corresponding tuple struct has 3 fields [E0023] ``` +#### Positioned above error line + +Use the `//~v` idiom with number of v's in the string to indicate the number +of lines below. This is typically used in lexer or parser tests matching on errors like unclosed +delimiter or unclosed literal happening at the end of file. + +```rust,ignore +// ignore-tidy-trailing-newlines +//~v ERROR this file contains an unclosed delimiter +fn main((ؼ +``` + +#### Error without line information + +Use `//~?` to match an error without line information. +`//~?` is precise and will not match errors if their line information is available. +It should be preferred over `//@ error-pattern` +for tests wishing to match against compiler diagnostics, +due to `//@ error-pattern` being imprecise and non-exhaustive. + +```rust,ignore +//@ compile-flags: --print yyyy + +//~? ERROR unknown print request: `yyyy` +``` + ### `error-pattern` -The `error-pattern` [directive](directives.md) can be used for messages that don't -have a specific span. +The `error-pattern` [directive](directives.md) can be used for runtime messages which don't +have a specific span, or, in exceptional cases, for compile time messages. Let's think about this test: @@ -286,8 +331,8 @@ fn main() { } ``` -We want to ensure this shows "index out of bounds" but we cannot use the `ERROR` -annotation since the error doesn't have any span. Then it's time to use the +We want to ensure this shows "index out of bounds", but we cannot use the `ERROR` +annotation since the runtime error doesn't have any span. Then it's time to use the `error-pattern` directive: ```rust,ignore @@ -300,22 +345,52 @@ fn main() { } ``` -But for strict testing, try to use the `ERROR` annotation as much as possible. +For strict testing of compile time output, try to use the line annotations `//~` as much as +possible, including `//~?` annotations for diagnostics without spans. + +If the compile time output is target dependent or too verbose, use directive +`//@ dont-require-annotations: ` to make the line annotation checking +non-exhaustive. +Some of the compiler messages can stay uncovered by annotations in this mode. + +For checking runtime output, `//@ check-run-results` may be preferable. -### Error levels +Only use `error-pattern` if none of the above works, such as when finding a +specific string pattern in a runtime panic output. -The error levels that you can have are: +Line annotations `//~` and `error-pattern` are compatible and can be used in the same test. + +### Diagnostic kinds (error levels) + +The diagnostic kinds that you can have are: - `ERROR` -- `WARN` or `WARNING` +- `WARN` (or `WARNING`) - `NOTE` -- `HELP` and `SUGGESTION` - -You are allowed to not include a level, but you should include it at least for -the primary message. +- `HELP` +- `SUGGESTION` +- `RAW` -The `SUGGESTION` level is used for specifying what the expected replacement text +The `SUGGESTION` kind is used for specifying what the expected replacement text should be for a diagnostic suggestion. +The `RAW` kind can be used for matching on lines from non-structured output sometimes emitted +by the compiler instead of or in addition to structured json. + +`ERROR` and `WARN` kinds are required to be exhaustively covered by line annotations +`//~` by default. + +Other kinds only need to be line-annotated if at least one annotation of that kind appears +in the test file. For example, one `//~ NOTE` will also require all other `//~ NOTE`s in the file +to be written out explicitly. + +Use directive `//@ dont-require-annotations` to opt out of exhaustive annotations. +E.g. use `//@ dont-require-annotations: NOTE` to annotate notes selectively. +Avoid using this directive for `ERROR`s and `WARN`ings, unless there's a serious reason, like +target-dependent compiler output. + +Some diagnostics are never required to be line-annotated, regardless of their kind or directives, +for example secondary lines of multiline diagnostics, +or ubiquitous diagnostics like `aborting due to N previous errors`. UI tests use the `-A unused` flag by default to ignore all unused warnings, as unused warnings are usually not the focus of a test. However, simple code @@ -353,7 +428,7 @@ would be a `.mir.stderr` and `.thir.stderr` file with the different outputs of the different revisions. > Note: cfg revisions also work inside the source code with `#[cfg]` attributes. -> +> > By convention, the `FALSE` cfg is used to have an always-false config. ## Controlling pass/fail expectations @@ -370,19 +445,29 @@ even run the resulting program. Just add one of the following - `//@ build-pass` — compilation and linking should succeed but do not run the resulting binary. - `//@ run-pass` — compilation should succeed and running the resulting - binary should also succeed. + binary should make it exit with code 0 which indicates success. - Fail directives: - `//@ check-fail` — compilation should fail (the codegen phase is skipped). This is the default for UI tests. - `//@ build-fail` — compilation should fail during the codegen phase. - This will run `rustc` twice, once to verify that it compiles successfully - without the codegen phase, then a second time the full compile should - fail. + This will run `rustc` twice: + - First time is to ensure that the compile succeeds without the codegen phase + - Second time is to ensure that the full compile fails - `//@ run-fail` — compilation should succeed, but running the resulting - binary should fail. - -For `run-pass` and `run-fail` tests, by default the output of the program itself -is not checked. + binary should make it exit with a code in the range `1..=127` which + indicates regular failure. On targets without unwind support, crashes + are also accepted. + - `//@ run-crash` — compilation should succeed, but running the resulting + binary should fail with a crash. Crashing is defined as "not exiting with + a code in the range `0..=127`". Example on Linux: Termination by `SIGABRT` + or `SIGSEGV`. Example on Windows: Exiting with the code for + `STATUS_ILLEGAL_INSTRUCTION` (`0xC000001D`). + - `//@ run-fail-or-crash` — compilation should succeed, but running the + resulting binary should either `run-fail` or `run-crash`. Useful if a test + crashes on some targets but just fails on others. + +For `run-pass`. `run-fail`, `run-crash` and `run-fail-or-crash` tests, by +default the output of the program itself is not checked. If you want to check the output of running the program, include the `check-run-results` directive. This will check for a `.run.stderr` and @@ -415,6 +500,14 @@ reasons, including: can alert the developer so they know that the associated issue has been fixed and can possibly be closed. +This directive takes comma-separated issue numbers as arguments, or `"unknown"`: + +- `//@ known-bug: #123, #456` (when the issues are on rust-lang/rust) +- `//@ known-bug: rust-lang/chalk#123456` + (allows arbitrary text before the `#`, which is useful when the issue is on another repo) +- `//@ known-bug: unknown` + (when there is no known issue yet; preferably open one if it does not already exist) + Do not include [error annotations](#error-annotations) in a test with `known-bug`. The test should still include other normal directives and stdout/stderr files. @@ -530,4 +623,27 @@ with "user-facing" Rust alone. Indeed, one could say that this slightly abuses the term "UI" (*user* interface) and turns such UI tests from black-box tests into white-box ones. Use them carefully and sparingly. -[compiler debugging]: ../compiler-debugging.md#rustc_test-attributes +[compiler debugging]: ../compiler-debugging.md#rustc_-test-attributes + +## UI test mode preset lint levels + +By default, test suites under UI test mode (`tests/ui`, `tests/ui-fulldeps`, +but not `tests/rustdoc-ui`) will specify + +- `-A unused` +- `-A internal_features` + +If: + +- The ui test's pass mode is below `run` (i.e. check or build). +- No compare modes are specified. + +Since they can be very noisy in ui tests. + +You can override them with `compile-flags` lint level flags or +in-source lint level attributes as required. + +Note that the `rustfix` version will *not* have `-A unused` passed, +meaning that you may have to `#[allow(unused)]` to suppress `unused` +lints on the rustfix'd file (because we might be testing rustfix +on `unused` lints themselves). diff --git a/src/the-parser.md b/src/the-parser.md index 703ef2794..601a81e2e 100644 --- a/src/the-parser.md +++ b/src/the-parser.md @@ -1,4 +1,4 @@ -# Lexing and Parsing +# Lexing and parsing The very first thing the compiler does is take the program (in UTF-8 Unicode text) and turn it into a data format the compiler can work with more conveniently than strings. @@ -52,14 +52,14 @@ the token stream, and then execute the parser to get a [`Crate`] (the root AST node). To minimize the amount of copying that is done, -both [`StringReader`] and [`Parser`] have lifetimes which bind them to the parent [`ParseSess`]. +both [`Lexer`] and [`Parser`] have lifetimes which bind them to the parent [`ParseSess`]. This contains all the information needed while parsing, as well as the [`SourceMap`] itself. Note that while parsing, we may encounter macro definitions or invocations. We set these aside to be expanded (see [Macro Expansion](./macro-expansion.md)). Expansion itself may require parsing the output of a macro, which may reveal more macros to be expanded, and so on. -## More on Lexical Analysis +## More on lexical analysis Code for lexical analysis is split between two crates: @@ -67,7 +67,7 @@ Code for lexical analysis is split between two crates: constituting tokens. Although it is popular to implement lexers as generated finite state machines, the lexer in [`rustc_lexer`] is hand-written. -- [`StringReader`] integrates [`rustc_lexer`] with data structures specific to +- [`Lexer`] integrates [`rustc_lexer`] with data structures specific to `rustc`. Specifically, it adds `Span` information to tokens returned by [`rustc_lexer`] and interns identifiers. @@ -76,7 +76,7 @@ Code for lexical analysis is split between two crates: [`ParseSess`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_session/parse/struct.ParseSess.html [`rustc_lexer`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_lexer/index.html [`SourceMap`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_span/source_map/struct.SourceMap.html -[`StringReader`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_parse/lexer/struct.StringReader.html +[`Lexer`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_parse/lexer/struct.Lexer.html [ast module]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_ast/ast/index.html [ast]: ./ast-validation.md [parser]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_parse/parser/index.html diff --git a/src/thir.md b/src/thir.md index 73d09ad80..3d3dafaef 100644 --- a/src/thir.md +++ b/src/thir.md @@ -1,7 +1,5 @@ # The THIR - - The THIR ("Typed High-Level Intermediate Representation"), previously called HAIR for "High-Level Abstract IR", is another IR used by rustc that is generated after [type checking]. It is (as of January 2024) used for diff --git a/src/tracing.md b/src/tracing.md index af484ab5f..5e5b81fc6 100644 --- a/src/tracing.md +++ b/src/tracing.md @@ -1,7 +1,5 @@ # Using tracing to debug the compiler - - The compiler has a lot of [`debug!`] (or `trace!`) calls, which print out logging information at many points. These are very useful to at least narrow down the location of a bug if not to find it entirely, or just to orient yourself as to why the @@ -185,11 +183,11 @@ rustc. While calls to `error!`, `warn!` and `info!` are included in every build of the compiler, calls to `debug!` and `trace!` are only included in the program if -`debug-logging=true` is turned on in config.toml (it is +`debug-logging=true` is turned on in bootstrap.toml (it is turned off by default), so if you don't see `DEBUG` logs, especially if you run the compiler with `RUSTC_LOG=rustc rustc some.rs` and only see `INFO` logs, make sure that `debug-logging=true` is turned on in your -config.toml. +bootstrap.toml. ## Logging etiquette and conventions diff --git a/src/traits/caching.md b/src/traits/caching.md index a9f20969b..c44722a1d 100644 --- a/src/traits/caching.md +++ b/src/traits/caching.md @@ -61,7 +61,7 @@ to be pretty clearly safe and also still retains a very high hit rate **TODO**: it looks like `pick_candidate_cache` no longer exists. In general, is this section still accurate at all? -[`ParamEnv`]: ../param_env/param_env_summary.html +[`ParamEnv`]: ../typing_parameter_envs.html [`tcx`]: ../ty.html [#18290]: https://github.com/rust-lang/rust/issues/18290 [#22019]: https://github.com/rust-lang/rust/issues/22019 diff --git a/src/traits/goals-and-clauses.md b/src/traits/goals-and-clauses.md index 40fd4581b..2884ca5a0 100644 --- a/src/traits/goals-and-clauses.md +++ b/src/traits/goals-and-clauses.md @@ -1,7 +1,5 @@ # Goals and clauses - - In logic programming terms, a **goal** is something that you must prove and a **clause** is something that you know is true. As described in the [lowering to logic](./lowering-to-logic.html) diff --git a/src/traits/implied-bounds.md b/src/traits/implied-bounds.md index 911553ad3..cdcb90d3e 100644 --- a/src/traits/implied-bounds.md +++ b/src/traits/implied-bounds.md @@ -40,8 +40,8 @@ requirements of impls and functions as explicit predicates. ### using implicit implied bounds as assumptions These bounds are not added to the `ParamEnv` of the affected item itself. For lexical -region resolution they are added using [`fn OutlivesEnvironment::with_bounds`]. -Similarly,during MIR borrowck we add them using +region resolution they are added using [`fn OutlivesEnvironment::from_normalized_bounds`]. +Similarly, during MIR borrowck we add them using [`fn UniversalRegionRelationsBuilder::add_implied_bounds`]. [We add implied bounds for the function signature and impl header in MIR borrowck][mir]. @@ -55,7 +55,7 @@ The assumed outlives constraints for implicit bounds are computed using the MIR borrowck adds the outlives constraints for both the normalized and unnormalized types, lexical region resolution [only uses the unnormalized types][notnorm]. -[`fn OutlivesEnvironment::with_bounds`]: https://github.com/rust-lang/rust/blob/5b8bc568d28b2e922290c9a966b3231d0ce9398b/compiler/rustc_infer/src/infer/outlives/env.rs#L90-L97 +[`fn OutlivesEnvironment::from_normalized_bounds`]: https://github.com/rust-lang/rust/blob/8239a37f9c0951a037cfc51763ea52a20e71e6bd/compiler/rustc_infer/src/infer/outlives/env.rs#L50-L55 [`fn UniversalRegionRelationsBuilder::add_implied_bounds`]: https://github.com/rust-lang/rust/blob/5b8bc568d28b2e922290c9a966b3231d0ce9398b/compiler/rustc_borrowck/src/type_check/free_region_relations.rs#L316 [mir]: https://github.com/rust-lang/rust/blob/91cae1dcdcf1a31bd8a92e4a63793d65cfe289bb/compiler/rustc_borrowck/src/type_check/free_region_relations.rs#L258-L332 [`fn assumed_wf_types`]: https://github.com/rust-lang/rust/blob/5b8bc568d28b2e922290c9a966b3231d0ce9398b/compiler/rustc_ty_utils/src/implied_bounds.rs#L21 @@ -81,4 +81,4 @@ This results in multiple unsoundnesses: [#25860]: https://github.com/rust-lang/rust/issues/25860 [#84591]: https://github.com/rust-lang/rust/issues/84591 -[#100051]: https://github.com/rust-lang/rust/issues/100051 \ No newline at end of file +[#100051]: https://github.com/rust-lang/rust/issues/100051 diff --git a/src/traits/lowering-to-logic.md b/src/traits/lowering-to-logic.md index 1248d4346..cc8b3bf80 100644 --- a/src/traits/lowering-to-logic.md +++ b/src/traits/lowering-to-logic.md @@ -1,7 +1,5 @@ # Lowering to logic - - The key observation here is that the Rust trait system is basically a kind of logic, and it can be mapped onto standard logical inference rules. We can then look for solutions to those inference rules in a diff --git a/src/traits/resolution.md b/src/traits/resolution.md index 26eb72458..ccb2b0426 100644 --- a/src/traits/resolution.md +++ b/src/traits/resolution.md @@ -1,7 +1,5 @@ # Trait resolution (old-style) - - This chapter describes the general process of _trait resolution_ and points out some non-obvious things. @@ -183,7 +181,7 @@ in that list. If so, it is considered satisfied. More precisely, we want to check whether there is a where-clause obligation that is for the same trait (or some subtrait) and which can match against the obligation. -[parameter environment]: ../param_env/param_env_summary.html +[parameter environment]: ../typing_parameter_envs.html Consider this simple example: diff --git a/src/traits/unsize.md b/src/traits/unsize.md index dd57a1b07..98a445257 100644 --- a/src/traits/unsize.md +++ b/src/traits/unsize.md @@ -32,21 +32,21 @@ Built-in implementations are provided for: ## Structural implementations -There are two implementations of `Unsize` which can be thought of as +There is one implementation of `Unsize` which can be thought of as structural: -* `(A1, A2, .., An): Unsize<(A1, A2, .., U)>` given `An: Unsize`, which - allows the tail field of a tuple to be unsized. This is gated behind the - [`unsized_tuple_coercion`] feature. * `Struct<.., Pi, .., Pj, ..>: Unsize>` given `TailField: Unsize`, which allows the tail field of a struct to be unsized if it is the only field that mentions generic parameters `Pi`, .., `Pj` (which don't need to be contiguous). -The rules for the latter implementation are slightly complicated, since they +The rules for struct unsizing are slightly complicated, since they may allow more than one parameter to be changed (not necessarily unsized) and are best stated in terms of the tail field of the struct. -[`unsized_tuple_coercion`]: https://doc.rust-lang.org/beta/unstable-book/language-features/unsized-tuple-coercion.html +(Tuple unsizing was previously implemented behind the feature gate +`unsized_tuple_coercion`, but the implementation was removed by [#137728].) + +[#137728]: https://github.com/rust-lang/rust/pull/137728 ## Upcasting implementations diff --git a/src/ty-fold.md b/src/ty-fold.md index d4d0952fc..23253022f 100644 --- a/src/ty-fold.md +++ b/src/ty-fold.md @@ -1,26 +1,28 @@ + # `TypeFoldable` and `TypeFolder` -In the previous chapter we discussed instantiating binders. This must involves looking at everything inside of a `Early/Binder` -to find any usages of the bound vars in order to replace them. Binders can wrap an arbitrary rust type `T` not just a `Ty` so -how do we implement the `instantiate` methods on the `Early/Binder` types. +In [a previous chapter], we discussed instantiating binders. +This involves looking at everything inside of a `Early(Binder)` +to find any usages of the bound vars in order to replace them. +Binders can wrap an arbitrary Rust type `T`, not just a `Ty`. +So, how do we implement the `instantiate` methods on the `Early/Binder` types? The answer is a couple of traits: -[`TypeFoldable`](https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/fold/trait.TypeFoldable.html) +[`TypeFoldable`] and -[`TypeFolder`](https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/fold/trait.TypeFolder.html). +[`TypeFolder`]. - `TypeFoldable` is implemented by types that embed type information. It allows you to recursively process the contents of the `TypeFoldable` and do stuff to them. - `TypeFolder` defines what you want to do with the types you encounter while processing the `TypeFoldable`. -For example, the `TypeFolder` trait has a method -[`fold_ty`](https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/fold/trait.TypeFolder.html#method.fold_ty) -that takes a type as input and returns a new type as a result. `TypeFoldable` invokes the -`TypeFolder` `fold_foo` methods on itself, giving the `TypeFolder` access to its contents (the -types, regions, etc that are contained within). +For example, the `TypeFolder` trait has a method [`fold_ty`] +that takes a type as input and returns a new type as a result. +`TypeFoldable` invokes the `TypeFolder` `fold_foo` methods on itself, +giving the `TypeFolder` access to its contents (the types, regions, etc that are contained within). -You can think of it with this analogy to the iterator combinators we have come to love in rust: +You can think of it with this analogy to the iterator combinators we have come to love in Rust: ```rust,ignore vec.iter().map(|e1| foo(e2)).collect() @@ -33,8 +35,7 @@ So to reiterate: - `TypeFolder` is a trait that defines a “map” operation. - `TypeFoldable` is a trait that is implemented by things that embed types. -In the case of `subst`, we can see that it is implemented as a `TypeFolder`: -[`ArgFolder`](https://doc.rust-lang.org/nightly/nightly-rustc/rustc_type_ir/binder/struct.ArgFolder.html). +In the case of `subst`, we can see that it is implemented as a `TypeFolder`: [`ArgFolder`]. Looking at its implementation, we see where the actual substitutions are happening. However, you might also notice that the implementation calls this `super_fold_with` method. What is @@ -88,17 +89,22 @@ things. We only want to do something when we reach a type. That means there may `TypeFoldable` types whose implementations basically just forward to their fields’ `TypeFoldable` implementations. Such implementations of `TypeFoldable` tend to be pretty tedious to write by hand. For this reason, there is a `derive` macro that allows you to `#![derive(TypeFoldable)]`. It is -defined -[here](https://github.com/rust-lang/rust/blob/master/compiler/rustc_macros/src/type_foldable.rs). - -**`subst`** In the case of substitutions the [actual -folder](https://github.com/rust-lang/rust/blob/75ff3110ac6d8a0259023b83fd20d7ab295f8dd6/src/librustc_middle/ty/subst.rs#L440-L451) -is going to be doing the indexing we’ve already mentioned. There we define a `Folder` and call -`fold_with` on the `TypeFoldable` to process yourself. Then -[fold_ty](https://github.com/rust-lang/rust/blob/75ff3110ac6d8a0259023b83fd20d7ab295f8dd6/src/librustc_middle/ty/subst.rs#L512-L536) -the method that process each type it looks for a `ty::Param` and for those it replaces it for -something from the list of substitutions, otherwise recursively process the type. To replace it, -calls -[ty_for_param](https://github.com/rust-lang/rust/blob/75ff3110ac6d8a0259023b83fd20d7ab295f8dd6/src/librustc_middle/ty/subst.rs#L552-L587) +defined [here]. + +**`subst`** In the case of substitutions the [actual folder] +is going to be doing the indexing we’ve already mentioned. +There we define a `Folder` and call `fold_with` on the `TypeFoldable` to process yourself. +Then [fold_ty] the method that process each type it looks for a `ty::Param` and for those +it replaces it for something from the list of substitutions, otherwise recursively process the type. +To replace it, calls [ty_for_param] and all that does is index into the list of substitutions with the index of the `Param`. +[a previous chapter]: ty_module/instantiating_binders.md +[`TypeFoldable`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/trait.TypeFoldable.html +[`TypeFolder`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/trait.TypeFolder.html +[`fold_ty`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/trait.TypeFolder.html#method.fold_ty +[`ArgFolder`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_type_ir/binder/struct.ArgFolder.html +[here]: https://github.com/rust-lang/rust/blob/master/compiler/rustc_macros/src/type_foldable.rs +[actual folder]: https://github.com/rust-lang/rust/blob/75ff3110ac6d8a0259023b83fd20d7ab295f8dd6/src/librustc_middle/ty/subst.rs#L440-L451 +[fold_ty]: https://github.com/rust-lang/rust/blob/75ff3110ac6d8a0259023b83fd20d7ab295f8dd6/src/librustc_middle/ty/subst.rs#L512-L536 +[ty_for_param]: https://github.com/rust-lang/rust/blob/75ff3110ac6d8a0259023b83fd20d7ab295f8dd6/src/librustc_middle/ty/subst.rs#L552-L587 diff --git a/src/ty.md b/src/ty.md index b33d54035..4055f475e 100644 --- a/src/ty.md +++ b/src/ty.md @@ -1,7 +1,5 @@ # The `ty` module: representing types - - The `ty` module defines how the Rust compiler represents types internally. It also defines the *typing context* (`tcx` or `TyCtxt`), which is the central data structure in the compiler. @@ -61,11 +59,11 @@ Here is a summary: | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | Describe the *syntax* of a type: what the user wrote (with some desugaring). | Describe the *semantics* of a type: the meaning of what the user wrote. | | Each `rustc_hir::Ty` has its own spans corresponding to the appropriate place in the program. | Doesn’t correspond to a single place in the user’s program. | -| `rustc_hir::Ty` has generics and lifetimes; however, some of those lifetimes are special markers like [`LifetimeName::Implicit`][implicit]. | `ty::Ty` has the full type, including generics and lifetimes, even if the user left them out | -| `fn foo(x: u32) → u32 { }` - Two `rustc_hir::Ty` representing each usage of `u32`, each has its own `Span`s, and `rustc_hir::Ty` doesn’t tell us that both are the same type | `fn foo(x: u32) → u32 { }` - One `ty::Ty` for all instances of `u32` throughout the program, and `ty::Ty` tells us that both usages of `u32` mean the same type. | -| `fn foo(x: &u32) -> &u32)` - Two `rustc_hir::Ty` again. Lifetimes for the references show up in the `rustc_hir::Ty`s using a special marker, [`LifetimeName::Implicit`][implicit]. | `fn foo(x: &u32) -> &u32)`- A single `ty::Ty`. The `ty::Ty` has the hidden lifetime param. | +| `rustc_hir::Ty` has generics and lifetimes; however, some of those lifetimes are special markers like [`LifetimeKind::Implicit`][implicit]. | `ty::Ty` has the full type, including generics and lifetimes, even if the user left them out | +| `fn foo(x: u32) -> u32 { }` - Two `rustc_hir::Ty` representing each usage of `u32`, each has its own `Span`s, and `rustc_hir::Ty` doesn’t tell us that both are the same type | `fn foo(x: u32) -> u32 { }` - One `ty::Ty` for all instances of `u32` throughout the program, and `ty::Ty` tells us that both usages of `u32` mean the same type. | +| `fn foo(x: &u32) -> &u32 { }` - Two `rustc_hir::Ty` again. Lifetimes for the references show up in the `rustc_hir::Ty`s using a special marker, [`LifetimeKind::Implicit`][implicit]. | `fn foo(x: &u32) -> &u32 { }`- A single `ty::Ty`. The `ty::Ty` has the hidden lifetime param. | -[implicit]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir/hir/enum.LifetimeName.html#variant.Implicit +[implicit]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir/hir/enum.LifetimeKind.html#variant.Implicit **Order** @@ -323,4 +321,4 @@ When looking at the debug output of `Ty` or simply talking about different types - Generic parameters: `{name}/#{index}` e.g. `T/#0`, where `index` corresponds to its position in the list of generic parameters - Inference variables: `?{id}` e.g. `?x`/`?0`, where `id` identifies the inference variable - Variables from binders: `^{binder}_{index}` e.g. `^0_x`/`^0_2`, where `binder` and `index` identify which variable from which binder is being referred to -- Placeholders: `!{id}` or `!{id}_{universe}` e.g. `!x`/`!0`/`!x_2`/`!0_2`, representing some unique type in the specified universe. The universe is often elided when it is `0` \ No newline at end of file +- Placeholders: `!{id}` or `!{id}_{universe}` e.g. `!x`/`!0`/`!x_2`/`!0_2`, representing some unique type in the specified universe. The universe is often elided when it is `0` diff --git a/src/ty_module/binders.md b/src/ty_module/binders.md index 6f31103d7..7fd9eeed5 100644 --- a/src/ty_module/binders.md +++ b/src/ty_module/binders.md @@ -1,6 +1,6 @@ # `Binder` and Higher ranked regions -Sometimes we define generic parameters not on an item but as part of a type or a where clauses. As an example the type `for<'a> fn(&'a u32)` or the where clause `for<'a> T: Trait<'a>` both introduce a generic lifetime named `'a`. Currently there is no stable syntax for `for` or `for` but on nightly `feature(non_lifetime_binders)` feature can be used to write where clauses (but not types) using `for`/`for`. +Sometimes we define generic parameters not on an item but as part of a type or a where clause. As an example the type `for<'a> fn(&'a u32)` or the where clause `for<'a> T: Trait<'a>` both introduce a generic lifetime named `'a`. Currently there is no stable syntax for `for` or `for` but on nightly `feature(non_lifetime_binders)` can be used to write where clauses (but not types) using `for`/`for`. The `for` is referred to as a "binder" because it brings new names into scope. In rustc we use the `Binder` type to track where these parameters are introduced and what the parameters are (i.e. how many and whether the parameter is a type/const/region). A type such as `for<'a> fn(&'a u32)` would be represented in rustc as: @@ -13,10 +13,11 @@ Binder( Usages of these parameters is represented by the `RegionKind::Bound` (or `TyKind::Bound`/`ConstKind::Bound` variants). These bound regions/types/consts are composed of two main pieces of data: - A [DebruijnIndex](../appendix/background.md#what-is-a-de-bruijn-index) to specify which binder we are referring to. -- A [`BoundVar`] which specifies which of the parameters the `Binder` introduces we are referring to. -- We also sometimes store some extra information for diagnostics reasons via the [`BoundTyKind`]/[`BoundRegionKind`] but this is not important for type equality or more generally the semantics of `Ty`. (omitted from the above example) +- A [`BoundVar`] which specifies which of the parameters that the `Binder` introduces we are referring to. -In debug output (and also informally when talking to eachother) we tend to write these bound variables in the format of `^DebruijnIndex_BoundVar`. The above example would instead be written as `Binder(fn(&'^0_0), &[BoundVariableKind::Region])`. Sometimes when the `DebruijnIndex` is `0` we just omit it and would write `^0`. +We also sometimes store some extra information for diagnostics reasons via the [`BoundTyKind`]/[`BoundRegionKind`] but this is not important for type equality or more generally the semantics of `Ty`. (omitted from the above example) + +In debug output (and also informally when talking to each other) we tend to write these bound variables in the format of `^DebruijnIndex_BoundVar`. The above example would instead be written as `Binder(fn(&'^0_0), &[BoundVariableKind::Region])`. Sometimes when the `DebruijnIndex` is `0` we just omit it and would write `^0`. Another concrete example, this time a mixture of `for<'a>` in a where clause and a type: ``` @@ -40,10 +41,10 @@ We did not always explicitly track the set of bound vars introduced by each `Bin ``` Binder( fn(&'^1_0 &'^1 T/#0), - &[BoundVariarbleKind::Region(...)], + &[BoundVariableKind::Region(...)], ) ``` -This would cause all kinds of issues as the region `'^1_0` refers to a binder at a higher level than the outermost binder i.e. it is an escaping bound var. The `'^1` region (also writeable as `'^0_1`) is also ill formed as the binder it refers to does not introduce a second parameter. Modern day rustc will ICE when constructing this binder due to both of those regions, in the past we would have simply allowed this to work and then ran into issues in other parts of the codebase. +This would cause all kinds of issues as the region `'^1_0` refers to a binder at a higher level than the outermost binder i.e. it is an escaping bound var. The `'^1` region (also writeable as `'^0_1`) is also ill formed as the binder it refers to does not introduce a second parameter. Modern day rustc will ICE when constructing this binder due to both of those reasons, in the past we would have simply allowed this to work and then ran into issues in other parts of the codebase. [`Binder`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/struct.Binder.html [`BoundVar`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/struct.BoundVar.html diff --git a/src/ty_module/early_binder.md b/src/ty_module/early_binder.md index e8ff3a530..d83d73c95 100644 --- a/src/ty_module/early_binder.md +++ b/src/ty_module/early_binder.md @@ -48,7 +48,7 @@ fn bar(foo: Foo) { In the compiler the `instantiate` call for this is done in [`FieldDef::ty`] ([src][field_def_ty_src]), at some point during type checking `bar` we will wind up calling `FieldDef::ty(x, &[u32, f32])` in order to obtain the type of `foo.x`. **Note on indices:** It is a bug if the index of a `Param` does not match what the `EarlyBinder` binds. For -example, if the index is out of bounds or the index index of a lifetime corresponds to a type parameter. +example, if the index is out of bounds or the index of a lifetime corresponds to a type parameter. These sorts of errors are caught earlier in the compiler during name resolution where we disallow references to generics parameters introduced by items that should not be nameable by the inner item. diff --git a/src/ty_module/instantiating_binders.md b/src/ty_module/instantiating_binders.md index 04d56ccbc..0d1108c72 100644 --- a/src/ty_module/instantiating_binders.md +++ b/src/ty_module/instantiating_binders.md @@ -77,7 +77,7 @@ This end result is incorrect as we had two separate binders introducing their ow While in theory we could make this work it would be quite involved and more complex than the current setup, we would have to: - "rewrite" bound variables to have a higher `DebruijnIndex` whenever instantiating a `Binder`/`EarlyBinder` with a `Bound` ty/const/region -- When inferring an inference variable to a bound var, if that bound var is from a binder enterred after creating the infer var, we would have to lower the `DebruijnIndex` of the var. +- When inferring an inference variable to a bound var, if that bound var is from a binder entered after creating the infer var, we would have to lower the `DebruijnIndex` of the var. - Separately track what binder an inference variable was created inside of, also what the innermost binder it can name parameters from (currently we only have to track the latter) - When resolving inference variables rewrite any bound variables according to the current binder depth of the infcx - Maybe more (while writing this list items kept getting added so it seems naive to think this is exhaustive) @@ -105,7 +105,8 @@ the `RePlaceholder` for the `'b` parameter is in a higher universe to track the ## Instantiating with `ReLateParam` -As discussed in a previous chapter, `RegionKind` has two variants for representing generic parameters, `ReLateParam` and `ReEarlyParam`. `ReLateParam` is conceptually a `Placeholder` that is always in the root universe (`U0`). It is used when instantiating late bound parameters of functions/closures while inside of them. Its actual representation is relatively different from both `ReEarlyParam` and `RePlaceholder`: +As discussed in [the chapter about representing types][representing-types], `RegionKind` has two variants for representing generic parameters, `ReLateParam` and `ReEarlyParam`. +`ReLateParam` is conceptually a `Placeholder` that is always in the root universe (`U0`). It is used when instantiating late bound parameters of functions/closures while inside of them. Its actual representation is relatively different from both `ReEarlyParam` and `RePlaceholder`: - A `DefId` for the item that introduced the late bound generic parameter - A [`BoundRegionKind`] which either specifies the `DefId` of the generic parameter and its name (via a `Symbol`), or that this placeholder is representing the anonymous lifetime of a `Fn`/`FnMut` closure's self borrow. There is also a variant for `BrAnon` but this is not used for `ReLateParam`. @@ -133,6 +134,7 @@ Generally whenever we have a `Binder` for late bound parameters on a function/cl As a concrete example, accessing the signature of a function we are type checking will be represented as `EarlyBinder>`. As we are already "inside" of these binders, we would call `instantiate_identity` followed by `liberate_late_bound_regions`. [`liberate_late_bound_regions`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/context/struct.TyCtxt.html#method.liberate_late_bound_regions +[representing-types]: param_ty_const_regions.md [`BoundRegionKind`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/enum.BoundRegionKind.html [`enter_forall`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_trait_selection/infer/struct.InferCtxt.html#method.enter_forall [ch_placeholders_universes]: ../borrow_check/region_inference/placeholders_and_universes.md diff --git a/src/ty_module/param_ty_const_regions.md b/src/ty_module/param_ty_const_regions.md index 6b467724f..493693c9a 100644 --- a/src/ty_module/param_ty_const_regions.md +++ b/src/ty_module/param_ty_const_regions.md @@ -11,15 +11,15 @@ TyKind::Ref( There are three separate ways we represent usages of generic parameters: - [`TyKind::Param`]/[`ConstKind::Param`]/[`RegionKind::EarlyParam`] for early bound generic parameters (note: all type and const parameters are considered early bound, see the [chapter on early vs late bound parameters][ch_early_late_bound] for more information) -- [`TyKind::Bound`]/[`ConstKind::Bound`]/[`RegionKind::Bound`] for references to parameters introduced via higher ranked bounds or higher ranked types i.e. `for<'a> fn(&'a u32)` or `for<'a> T: Trait<'a>`. This will be discussed in the [chapter on `Binder`s][ch_binders]. -- [`RegionKind::LateParam`] for late bound lifetime parameters, `LateParam` will be discussed in the [chapter on instantiating `Binder`s][ch_instantiating_binders]. +- [`TyKind::Bound`]/[`ConstKind::Bound`]/[`RegionKind::Bound`] for references to parameters introduced via higher ranked bounds or higher ranked types i.e. `for<'a> fn(&'a u32)` or `for<'a> T: Trait<'a>`. This is discussed in the [chapter on `Binder`s][ch_binders]. +- [`RegionKind::LateParam`] for late bound lifetime parameters, `LateParam` is discussed in the [chapter on instantiating `Binder`s][ch_instantiating_binders]. -This chapter will only cover `TyKind::Param` `ConstKind::Param` and `RegionKind::EarlyParam`. +This chapter only covers `TyKind::Param` `ConstKind::Param` and `RegionKind::EarlyParam`. ## Ty/Const Parameters -As `TyKind::Param` and `ConstKind::Param` are implemented identically this section will only refer to `TyKind::Param` for simplicity. However -you should keep in mind that everything here also is true of `ConstKind::Param` +As `TyKind::Param` and `ConstKind::Param` are implemented identically this section only refers to `TyKind::Param` for simplicity. +However you should keep in mind that everything here also is true of `ConstKind::Param` Each `TyKind::Param` contains two things: the name of the parameter and an index. @@ -53,7 +53,7 @@ Concretely given the `ty::Generics` for the item the parameter is defined on, if The index fully defines the `Ty` and is the only part of `TyKind::Param` that matters for reasoning about the code we are compiling. -Generally we do not care what the name is and only use the index is included for diagnostics and debug logs as otherwise it would be +Generally we do not care what the name is and only use the index. The name is included for diagnostics and debug logs as otherwise it would be incredibly difficult to understand the output, i.e. `Vec: Sized` vs `Vec: Sized`. In debug output, parameter types are often printed out as `{name}/#{index}`, for example in the function `foo` if we were to debug print `Vec` it would be written as `Vec`. @@ -65,7 +65,7 @@ The rules against shadowing make this difficult but those language rules could c ### Lifetime parameters -In contrast to `Ty`/`Const`'s `Param` singular `Param` variant, lifetimes have two variants for representing region parameters: [`RegionKind::EarlyParam`] and [`RegionKind::LateParam`]. The reason for this is due to function's distinguishing between [early and late bound parameters](../early-late-bound-params/early-late-bound-summary.md) which is discussed in an earlier chapter (see link). +In contrast to `Ty`/`Const`'s `Param` singular `Param` variant, lifetimes have two variants for representing region parameters: [`RegionKind::EarlyParam`] and [`RegionKind::LateParam`]. The reason for this is due to function's distinguishing between [early and late bound parameters][ch_early_late_bound] which is discussed in an earlier chapter (see link). `RegionKind::EarlyParam` is structured identically to `Ty/Const`'s `Param` variant, it is simply a `u32` index and a `Symbol`. For lifetime parameters defined on non-function items we always use `ReEarlyParam`. For functions we use `ReEarlyParam` for any early bound parameters and `ReLateParam` for any late bound parameters. Note that just like `Ty` and `Const` params we often debug format them as `'SYMBOL/#INDEX`, see for example: @@ -83,9 +83,9 @@ fn foo<'a, 'b, T: 'a>(one: T, two: &'a &'b u32) -> &'b u32 { } ``` -`RegionKind::LateParam` will be discussed more in the chapter on [instantiating binders][ch_instantiating_binders]. +`RegionKind::LateParam` is discussed more in the chapter on [instantiating binders][ch_instantiating_binders]. -[ch_early_late_bound]: ../early-late-bound-params/early-late-bound-summary.md +[ch_early_late_bound]: ../early_late_parameters.md [ch_binders]: ./binders.md [ch_instantiating_binders]: ./instantiating_binders.md [`BoundRegionKind`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/enum.BoundRegionKind.html diff --git a/src/type-checking.md b/src/type-checking.md index b60694201..4e8b30b19 100644 --- a/src/type-checking.md +++ b/src/type-checking.md @@ -17,7 +17,7 @@ Type "collection" is the process of converting the types found in the HIR **internal representation** used by the compiler (`Ty<'tcx>`) – we also do similar conversions for where-clauses and other bits of the function signature. -To try and get a sense for the difference, consider this function: +To try and get a sense of the difference, consider this function: ```rust,ignore struct Foo { } diff --git a/src/type-inference.md b/src/type-inference.md index c03fa36d7..2243205f1 100644 --- a/src/type-inference.md +++ b/src/type-inference.md @@ -1,7 +1,5 @@ # Type inference - - Type inference is the process of automatic detection of the type of an expression. @@ -19,7 +17,7 @@ Here, the type of `things` is *inferred* to be `Vec<&str>` because of the value we push into `things`. The type inference is based on the standard Hindley-Milner (HM) type inference -algorithm, but extended in various way to accommodate subtyping, region +algorithm, but extended in various ways to accommodate subtyping, region inference, and higher-ranked types. ## A note on terminology diff --git a/src/typing_parameter_envs.md b/src/typing_parameter_envs.md new file mode 100644 index 000000000..db15467a4 --- /dev/null +++ b/src/typing_parameter_envs.md @@ -0,0 +1,205 @@ +# Typing/Parameter Environments + +## Typing Environments + +When interacting with the type system there are a few variables to consider that can affect the results of trait solving. The set of in-scope where clauses, and what phase of the compiler type system operations are being performed in (the [`ParamEnv`][penv] and [`TypingMode`][tmode] structs respectively). + +When an environment to perform type system operations in has not yet been created, the [`TypingEnv`][tenv] can be used to bundle all of the external context required into a single type. + +Once a context to perform type system operations in has been created (e.g. an [`ObligationCtxt`][ocx] or [`FnCtxt`][fnctxt]) a `TypingEnv` is typically not stored anywhere as only the `TypingMode` is a property of the whole environment, whereas different `ParamEnv`s can be used on a per-goal basis. + +[ocx]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_trait_selection/traits/struct.ObligationCtxt.html +[fnctxt]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir_typeck/fn_ctxt/struct.FnCtxt.html + +## Parameter Environments + +### What is a `ParamEnv` + +The [`ParamEnv`][penv] is a list of in-scope where-clauses, it typically corresponds to a specific item's where clauses. Some clauses are not explicitly written but are instead implicitly added in the [`predicates_of`][predicates_of] query, such as `ConstArgHasType` or (some) implied bounds. + +In most cases `ParamEnv`s are initially created via the [`param_env` query][query] which returns a `ParamEnv` derived from the provided item's where clauses. A `ParamEnv` can also be created with arbitrary sets of clauses that are not derived from a specific item, such as in [`compare_method_predicate_entailment`][method_pred_entailment] where we create a hybrid `ParamEnv` consisting of the impl's where clauses and the trait definition's function's where clauses. + +--- + +If we have a function such as: +```rust +// `foo` would have a `ParamEnv` of: +// `[T: Sized, T: Trait, ::Assoc: Clone]` +fn foo() +where + ::Assoc: Clone, +{} +``` +If we were conceptually inside of `foo` (for example, type-checking or linting it) we would use this `ParamEnv` everywhere that we interact with the type system. This would allow things such as [normalization], evaluating generic constants, and proving where clauses/goals, to rely on `T` being sized, implementing `Trait`, etc. + +A more concrete example: +```rust +// `foo` would have a `ParamEnv` of: +// `[T: Sized, T: Clone]` +fn foo(a: T) { + // when typechecking `foo` we require all the where clauses on `requires_clone` + // to hold in order for it to be legal to call. This means we have to + // prove `T: Clone`. As we are type checking `foo` we use `foo`'s + // environment when trying to check that `T: Clone` holds. + // + // Trying to prove `T: Clone` with a `ParamEnv` of `[T: Sized, T: Clone]` + // will trivially succeed as bound we want to prove is in our environment. + requires_clone(a); +} +``` + +Or alternatively an example that would not compile: +```rust +// `foo2` would have a `ParamEnv` of: +// `[T: Sized]` +fn foo2(a: T) { + // When typechecking `foo2` we attempt to prove `T: Clone`. + // As we are type checking `foo2` we use `foo2`'s environment + // when trying to prove `T: Clone`. + // + // Trying to prove `T: Clone` with a `ParamEnv` of `[T: Sized]` will + // fail as there is nothing in the environment telling the trait solver + // that `T` implements `Clone` and there exists no user written impl + // that could apply. + requires_clone(a); +} +``` + +[predicates_of]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir_analysis/collect/predicates_of/fn.predicates_of.html +[method_pred_entailment]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir_analysis/check/compare_impl_item/fn.compare_method_predicate_entailment.html +[query]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/context/struct.TyCtxt.html#method.param_env +[normalization]: normalization.md + +### Acquiring a `ParamEnv` + +Using the wrong [`ParamEnv`][penv] when interacting with the type system can lead to ICEs, illformed programs compiling, or erroring when we shouldn't. See [#82159](https://github.com/rust-lang/rust/pull/82159) and [#82067](https://github.com/rust-lang/rust/pull/82067) as examples of PRs that modified the compiler to use the correct param env and in the process fixed ICEs. + +In the large majority of cases, when a `ParamEnv` is required it either already exists somewhere in scope, or above in the call stack and should be passed down. A non exhaustive list of places where you might find an existing `ParamEnv`: +- During typeck `FnCtxt` has a [`param_env` field][fnctxt_param_env] +- When writing late lints the `LateContext` has a [`param_env` field][latectxt_param_env] +- During well formedness checking the `WfCheckingCtxt` has a [`param_env` field][wfckctxt_param_env] +- The `TypeChecker` used for MIR Typeck has a [`param_env` field][mirtypeck_param_env] +- In the next-gen trait solver all `Goal`s have a [`param_env` field][goal_param_env] specifying what environment to prove the goal in +- When editing an existing [`TypeRelation`][typerelation] if it implements [`PredicateEmittingRelation`][predicate_emitting_relation] then a [`param_env` method][typerelation_param_env] will be available. + +If you aren't sure if there's a `ParamEnv` in scope somewhere that can be used it can be worth opening a thread in the [`#t-compiler/help`][compiler_help] zulip stream where someone may be able to point out where a `ParamEnv` can be acquired from. + +Manually constructing a `ParamEnv` is typically only needed at the start of some kind of top level analysis (e.g. hir typeck or borrow checking). In such cases there are three ways it can be done: +- Calling the [`tcx.param_env(def_id)` query][param_env_query] which returns the environment associated with a given definition. +- Creating an empty environment with [`ParamEnv::empty`][env_empty]. +- Using [`ParamEnv::new`][param_env_new] to construct an env with an arbitrary set of where clauses. Then calling [`traits::normalize_param_env_or_error`][normalize_env_or_error] to handle normalizing and elaborating all the where clauses in the env. + +Using the `param_env` query is by far the most common way to construct a `ParamEnv` as most of the time the compiler is performing an analysis as part of some specific definition. + +Creating an empty environment with `ParamEnv::empty` is typically only done either in codegen (indirectly via [`TypingEnv::fully_monomorphized`][tenv_mono]), or as part of some analysis that do not expect to ever encounter generic parameters (e.g. various parts of coherence/orphan check). + +Creating an env from an arbitrary set of where clauses is usually unnecessary and should only be done if the environment you need does not correspond to an actual item in the source code (e.g. [`compare_method_predicate_entailment`][method_pred_entailment]). + +[param_env_new]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/struct.ParamEnv.html#method.new +[normalize_env_or_error]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_trait_selection/traits/fn.normalize_param_env_or_error.html +[fnctxt_param_env]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir_typeck/fn_ctxt/struct.FnCtxt.html#structfield.param_env +[latectxt_param_env]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_lint/context/struct.LateContext.html#structfield.param_env +[wfckctxt_param_env]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir_analysis/check/wfcheck/struct.WfCheckingCtxt.html#structfield.param_env +[goal_param_env]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_infer/infer/canonical/ir/solve/struct.Goal.html#structfield.param_env +[typerelation_param_env]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_infer/infer/trait.PredicateEmittingRelation.html#tymethod.param_env +[typerelation]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/relate/trait.TypeRelation.html +[mirtypeck_param_env]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_borrowck/type_check/struct.TypeChecker.html#structfield.param_env +[env_empty]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/struct.ParamEnv.html#method.empty +[param_env_query]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir_typeck/fn_ctxt/struct.FnCtxt.html#structfield.param_env +[method_pred_entailment]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir_analysis/check/compare_impl_item/fn.compare_method_predicate_entailment.html +[predicate_emitting_relation]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/relate/combine/trait.PredicateEmittingRelation.html +[tenv_mono]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/struct.TypingEnv.html#method.fully_monomorphized +[compiler_help]: https://rust-lang.zulipchat.com/#narrow/channel/182449-t-compiler.2Fhelp + +### How are `ParamEnv`s constructed + +Creating a [`ParamEnv`][pe] is more complicated than simply using the list of where clauses defined on an item as written by the user. We need to both elaborate supertraits into the env and fully normalize all aliases. This logic is handled by [`traits::normalize_param_env_or_error`][normalize_env_or_error] (even though it does not mention anything about elaboration). + +#### Elaborating supertraits + +When we have a function such as `fn foo()` we would like to be able to prove `T: Clone` inside of the function as the `Copy` trait has a `Clone` supertrait. Constructing a `ParamEnv` looks at all of the trait bounds in the env and explicitly adds new where clauses to the `ParamEnv` for any supertraits found on the traits. + +A concrete example would be the following function: +```rust +trait Trait: SuperTrait {} +trait SuperTrait: SuperSuperTrait {} + +// `bar`'s unelaborated `ParamEnv` would be: +// `[T: Sized, T: Copy, T: Trait]` +fn bar(a: T) { + requires_impl(a); +} + +fn requires_impl(a: T) {} +``` + +If we did not elaborate the env then the `requires_impl` call would fail to typecheck as we would not be able to prove `T: Clone` or `T: SuperSuperTrait`. In practice we elaborate the env which means that `bar`'s `ParamEnv` is actually: +`[T: Sized, T: Copy, T: Clone, T: Trait, T: SuperTrait, T: SuperSuperTrait]` +This allows us to prove `T: Clone` and `T: SuperSuperTrait` when type checking `bar`. + +The `Clone` trait has a `Sized` supertrait however we do not end up with two `T: Sized` bounds in the env (one for the supertrait and one for the implicitly added `T: Sized` bound) as the elaboration process (implemented via [`util::elaborate`][elaborate]) deduplicates where clauses. + +A side effect of this is that even if no actual elaboration of supertraits takes place, the existing where clauses in the env are _also_ deduplicated. See the following example: +```rust +trait Trait {} +// The unelaborated `ParamEnv` would be: +// `[T: Sized, T: Trait, T: Trait]` +// but after elaboration it would be: +// `[T: Sized, T: Trait]` +fn foo() {} +``` + +The [next-gen trait solver][next-gen-solver] also requires this elaboration to take place. + +[elaborate]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_infer/traits/util/fn.elaborate.html +[next-gen-solver]: ./solve/trait-solving.md + +#### Normalizing all bounds + +In the old trait solver the where clauses stored in `ParamEnv` are required to be fully normalized as otherwise the trait solver will not function correctly. A concrete example of needing to normalize the `ParamEnv` is the following: +```rust +trait Trait { + type Assoc; +} + +trait Other { + type Bar; +} + +impl Other for T { + type Bar = u32; +} + +// `foo`'s unnormalized `ParamEnv` would be: +// `[T: Sized, U: Sized, U: Trait]` +fn foo(a: U) +where + U: Trait<::Bar>, +{ + requires_impl(a); +} + +fn requires_impl>(_: U) {} +``` + +As humans we can tell that `::Bar` is equal to `u32` so the trait bound on `U` is equivalent to `U: Trait`. In practice trying to prove `U: Trait` in the old solver in this environment would fail as it is unable to determine that `::Bar` is equal to `u32`. + +To work around this we normalize `ParamEnv`'s after constructing them so that `foo`'s `ParamEnv` is actually: `[T: Sized, U: Sized, U: Trait]` which means the trait solver is now able to use the `U: Trait` in the `ParamEnv` to determine that the trait bound `U: Trait` holds. + +This workaround does not work in all cases as normalizing associated types requires a `ParamEnv` which introduces a bootstrapping problem. We need a normalized `ParamEnv` in order for normalization to give correct results, but we need to normalize to get that `ParamEnv`. Currently we normalize the `ParamEnv` once using the unnormalized param env and it tends to give okay results in practice even though there are some examples where this breaks ([example]). + +In the next-gen trait solver the requirement for all where clauses in the `ParamEnv` to be fully normalized is not present and so we do not normalize when constructing `ParamEnv`s. + +[example]: https://play.rust-lang.org/?version=stable&mode=debug&edition=2021&gist=e6933265ea3e84eaa47019465739992c +[pe]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/struct.ParamEnv.html +[normalize_env_or_error]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_trait_selection/traits/fn.normalize_param_env_or_error.html + +## Typing Modes + +Depending on what context we are performing type system operations in, different behaviour may be required. For example during coherence there are stronger requirements about when we can consider goals to not hold or when we can consider types to be unequal. + +Tracking which "phase" of the compiler type system operations are being performed in is done by the [`TypingMode`][tmode] enum. The documentation on the `TypingMode` enum is quite good so instead of repeating it here verbatim we would recommend reading the API documentation directly. + +[penv]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/struct.ParamEnv.html +[tenv]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/struct.TypingEnv.html +[tmode]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/type.TypingMode.html diff --git a/src/unsafety-checking.md b/src/unsafety-checking.md index 113087894..fbc19d896 100644 --- a/src/unsafety-checking.md +++ b/src/unsafety-checking.md @@ -1,4 +1,4 @@ -# Unsafety Checking +# Unsafety checking Certain expressions in Rust can violate memory safety and as such need to be inside an `unsafe` block or function. The compiler will also warn if an unsafe diff --git a/src/variance.md b/src/variance.md index ad4fa4adf..7aa014071 100644 --- a/src/variance.md +++ b/src/variance.md @@ -1,7 +1,5 @@ # Variance of type and lifetime parameters - - For a more general background on variance, see the [background] appendix. [background]: ./appendix/background.html diff --git a/src/walkthrough.md b/src/walkthrough.md index 6e07ceb7d..b4c337934 100644 --- a/src/walkthrough.md +++ b/src/walkthrough.md @@ -1,7 +1,5 @@ # Walkthrough: a typical contribution - - There are _a lot_ of ways to contribute to the Rust compiler, including fixing bugs, improving performance, helping design features, providing feedback on existing features, etc. This chapter does not claim to scratch the surface. @@ -221,7 +219,7 @@ There are a couple of things that may happen for some PRs during the review proc some merge conflicts with other PRs that happen to get merged first. You should fix these merge conflicts using the normal git procedures. -[crater]: ./tests/intro.html#crater +[crater]: ./tests/crater.html If you are not doing a new feature or something like that (e.g. if you are fixing a bug), then that's it! Thanks for your contribution :) diff --git a/triagebot.toml b/triagebot.toml index ccb0de862..3ac5d57a5 100644 --- a/triagebot.toml +++ b/triagebot.toml @@ -1,8 +1,96 @@ -[assign] +# This file's format is documented at +# https://forge.rust-lang.org/triagebot/pr-assignment.html#configuration + +[autolabel."needs-triage"] +new_issue = true +exclude_labels = [ + "A-diagnostics", + "C-tracking-issue", +] + +[review-submitted] +# This label is added when a "request changes" review is submitted. +reviewed_label = "S-waiting-on-author" +# These labels are removed when a "request changes" review is submitted. +review_labels = ["S-waiting-on-review"] + +[review-requested] +# Those labels are removed when PR author requests a review from an assignee +remove_labels = ["S-waiting-on-author"] +# Those labels are added when PR author requests a review from an assignee +add_labels = ["S-waiting-on-review"] + +# Enable shortcuts like `@rustbot ready` +# Documentation at: https://forge.rust-lang.org/triagebot/shortcuts.html +[shortcut] + +[autolabel."S-waiting-on-review"] +new_pr = true + +# Enable issue transfers within the org +# Documentation at: https://forge.rust-lang.org/triagebot/transfer.html +[transfer] [relabel] allow-unauthenticated = [ - "waiting-on-review", - "waiting-on-author", - "blocked", + "-Z*", + "A-*", + "C-*", + "D-*", + "E-*", + "F-*", + "I-*", + "L-*", + "O-*", + "PG-*", + "S-*", + "T-*", + "WG-*", + "needs-triage", +] + +# Enable `@rustbot note` functionality +# Documentation at: https://forge.rust-lang.org/triagebot/note.html +[note] + +# Prevents mentions in commits to avoid users being spammed +# Documentation at: https://forge.rust-lang.org/triagebot/no-mentions.html +[no-mentions] + +# Canonicalize issue numbers to avoid closing the wrong issue +# when commits are included in subtrees, as well as warning links in commits. +# Documentation at: https://forge.rust-lang.org/triagebot/issue-links.html +[issue-links] + +[behind-upstream] +days-threshold = 7 + +# Enable triagebot (PR) assignment. +# Documentation at: https://forge.rust-lang.org/triagebot/pr-assignment.html +[assign] + +# NOTE: do not add `[assign.owners]` if we still wish to keep the opt-in +# reviewer model, as `[assign.owners]` will cause triagebot auto-reviewer +# assignment to kick in. + +# Custom PR welcome message for when no auto reviewer assignment is performed +# and no explicit manual reviewer selection is made. +# Documentation at: https://forge.rust-lang.org/triagebot/pr-assignment.html#custom-welcome-messages +[assign.custom_welcome_messages] +welcome-message = "" +welcome-message-no-reviewer = """\ +Thanks for the PR. If you have write access, feel free to merge this PR if it \ +does not need reviews. You can request a review using `r? rustc-dev-guide` or \ +`r? `. +""" + +# Groups for `r? `. +# Documentation at: https://forge.rust-lang.org/triagebot/pr-assignment.html#usage +# Keep members alphanumerically sorted. +[assign.adhoc_groups] +rustc-dev-guide = [ + "@BoxyUwU", + "@jieyouxu", + "@jyn514", + "@tshepang", ]