diff --git a/.backportrc.json b/.backportrc.json index 12ba0eafe..e636dae1b 100644 --- a/.backportrc.json +++ b/.backportrc.json @@ -1,5 +1,5 @@ { "upstream": "elastic/stack-docs", - "branches": [{ "name": "8.10", "checked": true }, "8.9", "8.8", "8.7", "8.6", "8.5", "8.4", "8.3", "8.2", "8.1", "8.0", "7.17", "7.16", "7.15", "7.14", "7.13", "7.12", "7.11", "7.10", "7.9", "7.8", "7.7", "7.6", "7.5", "7.4", "7.3", "7.2", "7.1", "7.0", "6.8", "6.7", "6.6", "6.5", "6.4", "6.3", "6.2", "6.1", "6.0", "5.6"], + "branches": [{ "name": "8.x", "checked": true }, "8.19", "8.18", "8.17", "8.16", "8.15", "8.14", "8.13", "8.12", "8.11", "8.10", "8.9", "8.8", "8.7", "8.6", "8.5", "8.4", "8.3", "8.2", "8.1", "8.0", "7.17", "7.16", "7.15", "7.14", "7.13", "7.12", "7.11", "7.10", "7.9", "7.8", "7.7", "7.6", "7.5", "7.4", "7.3", "7.2", "7.1", "7.0", "6.8", "6.7", "6.6", "6.5", "6.4", "6.3", "6.2", "6.1", "6.0", "5.6"], "labels": ["backport"] } diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 000000000..e51b87fd7 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,3 @@ +* @elastic/platform-docs +/docs/en/stack/ml/ @elastic/mlr-docs +/docs/en/glossary/terms/ @demisperazza \ No newline at end of file diff --git a/.github/workflows/comment-on-asciidoc-changes.yml b/.github/workflows/comment-on-asciidoc-changes.yml new file mode 100644 index 000000000..8e5f836b1 --- /dev/null +++ b/.github/workflows/comment-on-asciidoc-changes.yml @@ -0,0 +1,21 @@ +--- +name: Comment on PR for .asciidoc changes + +on: + # We need to use pull_request_target to be able to comment on PRs from forks + pull_request_target: + types: + - synchronize + - opened + - reopened + branches: + - main + - master + - "9.0" + +jobs: + comment-on-asciidoc-change: + permissions: + contents: read + pull-requests: write + uses: elastic/docs-builder/.github/workflows/comment-on-asciidoc-changes.yml@main diff --git a/.github/workflows/docs-preview-links.yml b/.github/workflows/docs-preview-links.yml index 79ea1de77..b850bc5c9 100644 --- a/.github/workflows/docs-preview-links.yml +++ b/.github/workflows/docs-preview-links.yml @@ -1,26 +1,20 @@ -name: Docs Preview Links +name: docs-preview on: pull_request_target: types: [opened] paths: - '**.asciidoc' +permissions: + pull-requests: write jobs: - doc-preview: + doc-preview-pr: runs-on: ubuntu-latest steps: - - uses: actions/github-script@v6 - name: Add doc preview links + - uses: elastic/docs/.github/actions/docs-preview@master with: - script: | - const pr = context.payload.pull_request; - const comment = `Documentation preview: - - ✨ [Changed pages](https://${context.repo.repo}_${pr.number}.docs-preview.app.elstc.co/diff)`; - - github.rest.issues.createComment({ - issue_number: context.issue.number, - owner: context.repo.owner, - repo: context.repo.repo, - body: comment, - }); \ No newline at end of file + github-token: ${{ secrets.GITHUB_TOKEN }} + repo: ${{ github.event.repository.name }} + preview-path: 'guide/index.html' + pr: ${{ github.event.pull_request.number }} \ No newline at end of file diff --git a/.mergify.yml b/.mergify.yml index 4e635661f..f886533dd 100644 --- a/.mergify.yml +++ b/.mergify.yml @@ -15,6 +15,131 @@ pull_request_rules: git merge /{{base}} git push {{head}} ``` + - name: backport patches to main branch + conditions: + - merged + - label=backport-main + actions: + backport: + assignees: + - "{{ author }}" + labels: + - "backport" + branches: + - "main" + title: "[{{ destination_branch }}](backport #{{ number }}) {{ title }}" + - name: backport patches to 8.x branch + conditions: + - merged + - base=main + - label=backport-8.x + actions: + backport: + assignees: + - "{{ author }}" + branches: + - "8.x" + title: "[{{ destination_branch }}] {{ title }} (backport #{{ number }})" + labels: + - backport + - name: backport patches to 8.17 branch + conditions: + - merged + - base=main + - label=backport-8.17 + actions: + backport: + assignees: + - "{{ author }}" + branches: + - "8.17" + title: "[{{ destination_branch }}] {{ title }} (backport #{{ number }})" + labels: + - backport + - name: backport patches to 8.16 branch + conditions: + - merged + - base=main + - label=backport-8.16 + actions: + backport: + assignees: + - "{{ author }}" + branches: + - "8.16" + title: "[{{ destination_branch }}] {{ title }} (backport #{{ number }})" + labels: + - backport + - name: backport patches to 8.15 branch + conditions: + - merged + - base=main + - label=backport-8.15 + actions: + backport: + assignees: + - "{{ author }}" + branches: + - "8.15" + title: "[{{ destination_branch }}] {{ title }} (backport #{{ number }})" + labels: + - backport + - name: backport patches to 8.14 branch + conditions: + - merged + - base=main + - label=backport-8.14 + actions: + backport: + assignees: + - "{{ author }}" + branches: + - "8.14" + title: "[{{ destination_branch }}] {{ title }} (backport #{{ number }})" + labels: + - backport + - name: backport patches to 8.13 branch + conditions: + - merged + - base=main + - label=backport-8.13 + actions: + backport: + assignees: + - "{{ author }}" + branches: + - "8.13" + title: "[{{ destination_branch }}] {{ title }} (backport #{{ number }})" + labels: + - backport + - name: backport patches to 8.12 branch + conditions: + - merged + - base=main + - label=backport-8.12 + actions: + backport: + assignees: + - "{{ author }}" + branches: + - "8.12" + title: "[{{ destination_branch }}] {{ title }} (backport #{{ number }})" + labels: + - backport + - name: backport patches to 8.11 branch + conditions: + - merged + - base=main + - label=backport-8.11 + actions: + backport: + assignees: + - "{{ author }}" + branches: + - "8.11" + title: "[{{ destination_branch }}] {{ title }} (backport #{{ number }})" + labels: + - backport - name: backport patches to 8.10 branch conditions: - merged diff --git a/LICENSE b/LICENSE new file mode 100644 index 000000000..3811ab9a1 --- /dev/null +++ b/LICENSE @@ -0,0 +1,402 @@ +Attribution-NonCommercial-NoDerivatives 4.0 International + +======================================================================= + +Creative Commons Corporation ("Creative Commons") is not a law firm and +does not provide legal services or legal advice. Distribution of +Creative Commons public licenses does not create a lawyer-client or +other relationship. Creative Commons makes its licenses and related +information available on an "as-is" basis. Creative Commons gives no +warranties regarding its licenses, any material licensed under their +terms and conditions, or any related information. Creative Commons +disclaims all liability for damages resulting from their use to the +fullest extent possible. + +Using Creative Commons Public Licenses + +Creative Commons public licenses provide a standard set of terms and +conditions that creators and other rights holders may use to share +original works of authorship and other material subject to copyright +and certain other rights specified in the public license below. The +following considerations are for informational purposes only, are not +exhaustive, and do not form part of our licenses. + + Considerations for licensors: Our public licenses are + intended for use by those authorized to give the public + permission to use material in ways otherwise restricted by + copyright and certain other rights. Our licenses are + irrevocable. Licensors should read and understand the terms + and conditions of the license they choose before applying it. + Licensors should also secure all rights necessary before + applying our licenses so that the public can reuse the + material as expected. Licensors should clearly mark any + material not subject to the license. This includes other CC- + licensed material, or material used under an exception or + limitation to copyright. More considerations for licensors: + wiki.creativecommons.org/Considerations_for_licensors + + Considerations for the public: By using one of our public + licenses, a licensor grants the public permission to use the + licensed material under specified terms and conditions. If + the licensor's permission is not necessary for any reason--for + example, because of any applicable exception or limitation to + copyright--then that use is not regulated by the license. Our + licenses grant only permissions under copyright and certain + other rights that a licensor has authority to grant. Use of + the licensed material may still be restricted for other + reasons, including because others have copyright or other + rights in the material. A licensor may make special requests, + such as asking that all changes be marked or described. + Although not required by our licenses, you are encouraged to + respect those requests where reasonable. More considerations + for the public: + wiki.creativecommons.org/Considerations_for_licensees + +======================================================================= + +Creative Commons Attribution-NonCommercial-NoDerivatives 4.0 +International Public License + +By exercising the Licensed Rights (defined below), You accept and agree +to be bound by the terms and conditions of this Creative Commons +Attribution-NonCommercial-NoDerivatives 4.0 International Public +License ("Public License"). To the extent this Public License may be +interpreted as a contract, You are granted the Licensed Rights in +consideration of Your acceptance of these terms and conditions, and the +Licensor grants You such rights in consideration of benefits the +Licensor receives from making the Licensed Material available under +these terms and conditions. + + +Section 1 -- Definitions. + + a. Adapted Material means material subject to Copyright and Similar + Rights that is derived from or based upon the Licensed Material + and in which the Licensed Material is translated, altered, + arranged, transformed, or otherwise modified in a manner requiring + permission under the Copyright and Similar Rights held by the + Licensor. For purposes of this Public License, where the Licensed + Material is a musical work, performance, or sound recording, + Adapted Material is always produced where the Licensed Material is + synched in timed relation with a moving image. + + b. Copyright and Similar Rights means copyright and/or similar rights + closely related to copyright including, without limitation, + performance, broadcast, sound recording, and Sui Generis Database + Rights, without regard to how the rights are labeled or + categorized. For purposes of this Public License, the rights + specified in Section 2(b)(1)-(2) are not Copyright and Similar + Rights. + + c. Effective Technological Measures means those measures that, in the + absence of proper authority, may not be circumvented under laws + fulfilling obligations under Article 11 of the WIPO Copyright + Treaty adopted on December 20, 1996, and/or similar international + agreements. + + d. Exceptions and Limitations means fair use, fair dealing, and/or + any other exception or limitation to Copyright and Similar Rights + that applies to Your use of the Licensed Material. + + e. Licensed Material means the artistic or literary work, database, + or other material to which the Licensor applied this Public + License. + + f. Licensed Rights means the rights granted to You subject to the + terms and conditions of this Public License, which are limited to + all Copyright and Similar Rights that apply to Your use of the + Licensed Material and that the Licensor has authority to license. + + g. Licensor means the individual(s) or entity(ies) granting rights + under this Public License. + + h. NonCommercial means not primarily intended for or directed towards + commercial advantage or monetary compensation. For purposes of + this Public License, the exchange of the Licensed Material for + other material subject to Copyright and Similar Rights by digital + file-sharing or similar means is NonCommercial provided there is + no payment of monetary compensation in connection with the + exchange. + + i. Share means to provide material to the public by any means or + process that requires permission under the Licensed Rights, such + as reproduction, public display, public performance, distribution, + dissemination, communication, or importation, and to make material + available to the public including in ways that members of the + public may access the material from a place and at a time + individually chosen by them. + + j. Sui Generis Database Rights means rights other than copyright + resulting from Directive 96/9/EC of the European Parliament and of + the Council of 11 March 1996 on the legal protection of databases, + as amended and/or succeeded, as well as other essentially + equivalent rights anywhere in the world. + + k. You means the individual or entity exercising the Licensed Rights + under this Public License. Your has a corresponding meaning. + + +Section 2 -- Scope. + + a. License grant. + + 1. Subject to the terms and conditions of this Public License, + the Licensor hereby grants You a worldwide, royalty-free, + non-sublicensable, non-exclusive, irrevocable license to + exercise the Licensed Rights in the Licensed Material to: + + a. reproduce and Share the Licensed Material, in whole or + in part, for NonCommercial purposes only; and + + b. produce and reproduce, but not Share, Adapted Material + for NonCommercial purposes only. + + 2. Exceptions and Limitations. For the avoidance of doubt, where + Exceptions and Limitations apply to Your use, this Public + License does not apply, and You do not need to comply with + its terms and conditions. + + 3. Term. The term of this Public License is specified in Section + 6(a). + + 4. Media and formats; technical modifications allowed. The + Licensor authorizes You to exercise the Licensed Rights in + all media and formats whether now known or hereafter created, + and to make technical modifications necessary to do so. The + Licensor waives and/or agrees not to assert any right or + authority to forbid You from making technical modifications + necessary to exercise the Licensed Rights, including + technical modifications necessary to circumvent Effective + Technological Measures. For purposes of this Public License, + simply making modifications authorized by this Section 2(a) + (4) never produces Adapted Material. + + 5. Downstream recipients. + + a. Offer from the Licensor -- Licensed Material. Every + recipient of the Licensed Material automatically + receives an offer from the Licensor to exercise the + Licensed Rights under the terms and conditions of this + Public License. + + b. No downstream restrictions. You may not offer or impose + any additional or different terms or conditions on, or + apply any Effective Technological Measures to, the + Licensed Material if doing so restricts exercise of the + Licensed Rights by any recipient of the Licensed + Material. + + 6. No endorsement. Nothing in this Public License constitutes or + may be construed as permission to assert or imply that You + are, or that Your use of the Licensed Material is, connected + with, or sponsored, endorsed, or granted official status by, + the Licensor or others designated to receive attribution as + provided in Section 3(a)(1)(A)(i). + + b. Other rights. + + 1. Moral rights, such as the right of integrity, are not + licensed under this Public License, nor are publicity, + privacy, and/or other similar personality rights; however, to + the extent possible, the Licensor waives and/or agrees not to + assert any such rights held by the Licensor to the limited + extent necessary to allow You to exercise the Licensed + Rights, but not otherwise. + + 2. Patent and trademark rights are not licensed under this + Public License. + + 3. To the extent possible, the Licensor waives any right to + collect royalties from You for the exercise of the Licensed + Rights, whether directly or through a collecting society + under any voluntary or waivable statutory or compulsory + licensing scheme. In all other cases the Licensor expressly + reserves any right to collect such royalties, including when + the Licensed Material is used other than for NonCommercial + purposes. + + +Section 3 -- License Conditions. + +Your exercise of the Licensed Rights is expressly made subject to the +following conditions. + + a. Attribution. + + 1. If You Share the Licensed Material, You must: + + a. retain the following if it is supplied by the Licensor + with the Licensed Material: + + i. identification of the creator(s) of the Licensed + Material and any others designated to receive + attribution, in any reasonable manner requested by + the Licensor (including by pseudonym if + designated); + + ii. a copyright notice; + + iii. a notice that refers to this Public License; + + iv. a notice that refers to the disclaimer of + warranties; + + v. a URI or hyperlink to the Licensed Material to the + extent reasonably practicable; + + b. indicate if You modified the Licensed Material and + retain an indication of any previous modifications; and + + c. indicate the Licensed Material is licensed under this + Public License, and include the text of, or the URI or + hyperlink to, this Public License. + + For the avoidance of doubt, You do not have permission under + this Public License to Share Adapted Material. + + 2. You may satisfy the conditions in Section 3(a)(1) in any + reasonable manner based on the medium, means, and context in + which You Share the Licensed Material. For example, it may be + reasonable to satisfy the conditions by providing a URI or + hyperlink to a resource that includes the required + information. + + 3. If requested by the Licensor, You must remove any of the + information required by Section 3(a)(1)(A) to the extent + reasonably practicable. + + +Section 4 -- Sui Generis Database Rights. + +Where the Licensed Rights include Sui Generis Database Rights that +apply to Your use of the Licensed Material: + + a. for the avoidance of doubt, Section 2(a)(1) grants You the right + to extract, reuse, reproduce, and Share all or a substantial + portion of the contents of the database for NonCommercial purposes + only and provided You do not Share Adapted Material; + + b. if You include all or a substantial portion of the database + contents in a database in which You have Sui Generis Database + Rights, then the database in which You have Sui Generis Database + Rights (but not its individual contents) is Adapted Material; and + + c. You must comply with the conditions in Section 3(a) if You Share + all or a substantial portion of the contents of the database. + +For the avoidance of doubt, this Section 4 supplements and does not +replace Your obligations under this Public License where the Licensed +Rights include other Copyright and Similar Rights. + + +Section 5 -- Disclaimer of Warranties and Limitation of Liability. + + a. UNLESS OTHERWISE SEPARATELY UNDERTAKEN BY THE LICENSOR, TO THE + EXTENT POSSIBLE, THE LICENSOR OFFERS THE LICENSED MATERIAL AS-IS + AND AS-AVAILABLE, AND MAKES NO REPRESENTATIONS OR WARRANTIES OF + ANY KIND CONCERNING THE LICENSED MATERIAL, WHETHER EXPRESS, + IMPLIED, STATUTORY, OR OTHER. THIS INCLUDES, WITHOUT LIMITATION, + WARRANTIES OF TITLE, MERCHANTABILITY, FITNESS FOR A PARTICULAR + PURPOSE, NON-INFRINGEMENT, ABSENCE OF LATENT OR OTHER DEFECTS, + ACCURACY, OR THE PRESENCE OR ABSENCE OF ERRORS, WHETHER OR NOT + KNOWN OR DISCOVERABLE. WHERE DISCLAIMERS OF WARRANTIES ARE NOT + ALLOWED IN FULL OR IN PART, THIS DISCLAIMER MAY NOT APPLY TO YOU. + + b. TO THE EXTENT POSSIBLE, IN NO EVENT WILL THE LICENSOR BE LIABLE + TO YOU ON ANY LEGAL THEORY (INCLUDING, WITHOUT LIMITATION, + NEGLIGENCE) OR OTHERWISE FOR ANY DIRECT, SPECIAL, INDIRECT, + INCIDENTAL, CONSEQUENTIAL, PUNITIVE, EXEMPLARY, OR OTHER LOSSES, + COSTS, EXPENSES, OR DAMAGES ARISING OUT OF THIS PUBLIC LICENSE OR + USE OF THE LICENSED MATERIAL, EVEN IF THE LICENSOR HAS BEEN + ADVISED OF THE POSSIBILITY OF SUCH LOSSES, COSTS, EXPENSES, OR + DAMAGES. WHERE A LIMITATION OF LIABILITY IS NOT ALLOWED IN FULL OR + IN PART, THIS LIMITATION MAY NOT APPLY TO YOU. + + c. The disclaimer of warranties and limitation of liability provided + above shall be interpreted in a manner that, to the extent + possible, most closely approximates an absolute disclaimer and + waiver of all liability. + + +Section 6 -- Term and Termination. + + a. This Public License applies for the term of the Copyright and + Similar Rights licensed here. However, if You fail to comply with + this Public License, then Your rights under this Public License + terminate automatically. + + b. Where Your right to use the Licensed Material has terminated under + Section 6(a), it reinstates: + + 1. automatically as of the date the violation is cured, provided + it is cured within 30 days of Your discovery of the + violation; or + + 2. upon express reinstatement by the Licensor. + + For the avoidance of doubt, this Section 6(b) does not affect any + right the Licensor may have to seek remedies for Your violations + of this Public License. + + c. For the avoidance of doubt, the Licensor may also offer the + Licensed Material under separate terms or conditions or stop + distributing the Licensed Material at any time; however, doing so + will not terminate this Public License. + + d. Sections 1, 5, 6, 7, and 8 survive termination of this Public + License. + + +Section 7 -- Other Terms and Conditions. + + a. The Licensor shall not be bound by any additional or different + terms or conditions communicated by You unless expressly agreed. + + b. Any arrangements, understandings, or agreements regarding the + Licensed Material not stated herein are separate from and + independent of the terms and conditions of this Public License. + + +Section 8 -- Interpretation. + + a. For the avoidance of doubt, this Public License does not, and + shall not be interpreted to, reduce, limit, restrict, or impose + conditions on any use of the Licensed Material that could lawfully + be made without permission under this Public License. + + b. To the extent possible, if any provision of this Public License is + deemed unenforceable, it shall be automatically reformed to the + minimum extent necessary to make it enforceable. If the provision + cannot be reformed, it shall be severed from this Public License + without affecting the enforceability of the remaining terms and + conditions. + + c. No term or condition of this Public License will be waived and no + failure to comply consented to unless expressly agreed to by the + Licensor. + + d. Nothing in this Public License constitutes or may be interpreted + as a limitation upon, or waiver of, any privileges and immunities + that apply to the Licensor or You, including from the legal + processes of any jurisdiction or authority. + +======================================================================= + +Creative Commons is not a party to its public +licenses. Notwithstanding, Creative Commons may elect to apply one of +its public licenses to material it publishes and in those instances +will be considered the “Licensor.” The text of the Creative Commons +public licenses is dedicated to the public domain under the CC0 Public +Domain Dedication. Except for the limited purpose of indicating that +material is shared under a Creative Commons public license or as +otherwise permitted by the Creative Commons policies published at +creativecommons.org/policies, Creative Commons does not authorize the +use of the trademark "Creative Commons" or any other trademark or logo +of Creative Commons without its prior written consent including, +without limitation, in connection with any unauthorized modifications +to any of its public licenses or any other arrangements, +understandings, or agreements concerning use of licensed material. For +the avoidance of doubt, this paragraph does not form part of the +public licenses. + +Creative Commons may be contacted at creativecommons.org. \ No newline at end of file diff --git a/README.md b/README.md index 2a77a70b0..d7537bb05 100644 --- a/README.md +++ b/README.md @@ -33,3 +33,15 @@ You can see the required repositories for each book in either the [conf.yaml](ht 3. Run the appropriate `build_docs` script, as listed in the [doc_build_aliases.sh file](https://github.com/elastic/docs/blob/master/doc_build_aliases.sh). +## License + +Shield: [![CC BY-NC-ND 4.0][cc-by-nc-nd-shield]][cc-by-nc-nd] + +This work is licensed under a +[Creative Commons Attribution-NonCommercial-NoDerivs 4.0 International License][cc-by-nc-nd]. + +[![CC BY-NC-ND 4.0][cc-by-nc-nd-image]][cc-by-nc-nd] + +[cc-by-nc-nd]: http://creativecommons.org/licenses/by-nc-nd/4.0/ +[cc-by-nc-nd-image]: https://licensebuttons.net/l/by-nc-nd/4.0/88x31.png +[cc-by-nc-nd-shield]: https://img.shields.io/badge/License-CC%20BY--NC--ND%204.0-lightgrey.svg \ No newline at end of file diff --git a/docs/en/glossary/glossary.asciidoc b/docs/en/glossary/glossary.asciidoc index c3bd93de9..5392e6999 100644 --- a/docs/en/glossary/glossary.asciidoc +++ b/docs/en/glossary/glossary.asciidoc @@ -832,6 +832,10 @@ A network endpoint which is monitored to track the performance and availability applications and services. //Source: Observability +[[glossary-multifactor]] multifactor authentication (MFA):: +A security process that requires you to provide two or more verification methods to gain access to web-based user interfaces. +//Souce: Cloud + [[glossary-multi-field]] multi-field:: A <> that's <> in multiple ways. See the {ref}/multi-fields.html[`fields` mapping parameter]. diff --git a/docs/en/glossary/terms/multifactor.asciidoc b/docs/en/glossary/terms/multifactor.asciidoc new file mode 100644 index 000000000..8eaf3c1d4 --- /dev/null +++ b/docs/en/glossary/terms/multifactor.asciidoc @@ -0,0 +1,3 @@ +[[glossary-multifactor]] multifactor authentication (MFA):: +A security process that requires you to provide two or more verification methods to gain access to web-based user interfaces. +//Souce: Cloud \ No newline at end of file diff --git a/docs/en/install-upgrade/air-gapped-install.asciidoc b/docs/en/install-upgrade/air-gapped-install.asciidoc index e35b778e3..fcb8fe9a9 100644 --- a/docs/en/install-upgrade/air-gapped-install.asciidoc +++ b/docs/en/install-upgrade/air-gapped-install.asciidoc @@ -72,7 +72,8 @@ Specifically: * To be able to use {kib} sample data, install or update hundreds of prebuilt alert rules, and explore available data integrations, you need to set up and configure the <>. * To provide detection rule updates for {endpoint-sec} agents, you need to set up and configure the <>. * To access {ents} capabilities (in addition to the general search capabilities of {es}), you need to set up and configure <>. -* To access the APM integration, you need to set up and configure <>. +* To access the APM integration, you need to set up and configure <>. +* To install and use the Elastic documentation for {kib} AI assistants, you need to set up and configure the <>. [discrete] [[air-gapped-beats]] @@ -120,7 +121,7 @@ Air-gapped setup of the APM server is possible in two ways: [[air-gapped-elastic-maps-service]] ==== 1.8. {ems} -To learn about air-gapped setup of the {ems}, refer to {kibana-ref}/maps-connect-to-ems.html#elastic-maps-server[Host {ems} locally] in the {kib} documentation. +Refer to {kibana-ref}/maps-connect-to-ems.html[Connect to {ems}] in the {kib} documentation to learn how to configure your firewall to connect to {ems}, host it locally, or disable it completely. [discrete] [[air-gapped-enterprise-search]] @@ -163,6 +164,12 @@ Some {ml} features, like natural language processing (NLP), require you to deplo * {ml-docs}/ml-nlp-elser.html#air-gapped-install[Deploy ELSER in an air-gapped environment]. * {eland-docs}/machine-learning.html#ml-nlp-pytorch-air-gapped[Install trained models in an air-gapped environment with Eland]. +[discrete] +[[air-gapped-kibana-product-documentation]] +==== 1.14 {kib} Product documentation for AI Assistants + +Detailed install and configuration instructions are available in the {kibana-ref}/ai-assistant-settings-kb.html[{kib} AI Assistants settings documentation]. + [discrete] [[air-gapped-kubernetes-and-openshift]] === 2. Kubernetes & OpenShift Install diff --git a/docs/en/install-upgrade/images/install-stack-metrics-dashboard.png b/docs/en/install-upgrade/images/install-stack-metrics-dashboard.png new file mode 100644 index 000000000..890b4d465 Binary files /dev/null and b/docs/en/install-upgrade/images/install-stack-metrics-dashboard.png differ diff --git a/docs/en/install-upgrade/images/stack-install-final-state.png b/docs/en/install-upgrade/images/stack-install-final-state.png new file mode 100644 index 000000000..1b651cace Binary files /dev/null and b/docs/en/install-upgrade/images/stack-install-final-state.png differ diff --git a/docs/en/install-upgrade/index.asciidoc b/docs/en/install-upgrade/index.asciidoc index f3192cd5f..962c0f702 100644 --- a/docs/en/install-upgrade/index.asciidoc +++ b/docs/en/install-upgrade/index.asciidoc @@ -17,6 +17,10 @@ include::overview.asciidoc[] include::installing-stack.asciidoc[] +include::installing-stack-demo-self.asciidoc[] + +include::installing-stack-demo-secure.asciidoc[] + include::air-gapped-install.asciidoc[] include::upgrading-stack.asciidoc[] diff --git a/docs/en/install-upgrade/installing-stack-demo-secure.asciidoc b/docs/en/install-upgrade/installing-stack-demo-secure.asciidoc new file mode 100644 index 000000000..7dd3ad4b9 --- /dev/null +++ b/docs/en/install-upgrade/installing-stack-demo-secure.asciidoc @@ -0,0 +1,980 @@ +//For testing on currently available builds: +//:version: 8.11. + +[[install-stack-demo-secure]] +=== Tutorial 2: Securing a self-managed {stack} + +This tutorial is a follow-on to <>. The first tutorial describes how to configure a multi-node {es} cluster and then set up {kib}, followed by {fleet-server} and {agent}. In a production environment, it's recommended after completing the {kib} setup to proceed directly to this tutorial to configure your SSL certificates. These steps guide you through that process, and then describe how to configure {fleet-server} and {agent} with the certificates in place. + +**Securing the {stack}** + +Beginning with Elastic 8.0, security is enabled in the {stack} by default, meaning that traffic between {es} nodes and between {kib} and {es} is SSL-encrypted. While this is suitable for testing non-production viability of the Elastic platform, most production networks have requirements for the use of trusted CA-signed certificates. These steps demonstrate how to update the out-of-the-box self-signed certificates with your own trusted CA-signed certificates. + +For traffic to be encrypted between {es} cluster nodes and between {kib} and {es}, SSL certificates must be created for the transport ({es} inter-node communication) and HTTP (for the {es} REST API) layers. Similarly, when setting up {fleet-server} you'll generate and configure a new certificate bundle, and then {elastic-agent} uses the generated certificates to communicate with both {fleet-server} and {es}. The process to set things up is as follows: + +* <> +* <> +* <> +* <> +* <> +* <> +* <> +* <> +* <> + +It should take between one and two hours to complete these steps. + +[discrete] +[[install-stack-demo-secure-prereqs]] +== Prerequisites and assumptions + +Before starting, you'll need to have set up an on-premises {es} cluster with {kib}, following the steps in <>. + +The examples in this guide use RPM packages to install the {stack} components on hosts running Red Hat Enterprise Linux 8. The steps for other install methods and operating systems are similar, and can be found in the documentation linked from each section. + +Special considerations such as firewalls and proxy servers are not covered here. + +[discrete] +[[install-stack-demo-secure-ca]] +== Step 1: Generate a new self-signed CA certificate + +In a production environment you would typically use the CA certificate from your own organization, along with the certificate files generated for the hosts where the {stack} is being installed. For demonstration purposes, we'll use the Elastic certificate utility to configure a self-signed CA certificate. + +. On the first node in your {es} cluster, stop the {es} service: ++ +["source","shell"] +---- +sudo systemctl stop elasticsearch.service +---- + +. Generate a CA certificate using the provided certificate utility, `elasticsearch-certutil`. Note that the location of the utility depends on the installation method you used to install {es}. Refer to {ref}/certutil.html[elasticsearch-certutil] for the command details and to {ref}/update-node-certs-different.html[Update security certificates with a different CA] for details about the procedure as a whole. ++ +Run the following command. When prompted, specify a unique name for the output file, such as `elastic-stack-ca.zip`: ++ +["source","shell"] +---- +sudo /usr/share/elasticsearch/bin/elasticsearch-certutil ca -pem +---- + +. Move the output file to the `/etc/elasticsearch/certs` directory. This directory is created automatically when you install {es}. ++ +["source","shell"] +---- +sudo mv /usr/share/elasticsearch/elastic-stack-ca.zip /etc/elasticsearch/certs/ +---- + +. Unzip the file: ++ +["source","shell"] +---- +sudo unzip -d /etc/elasticsearch/certs /etc/elasticsearch/certs/elastic-stack-ca.zip +---- + +. View the files that were unpacked into a new `ca` directory: ++ +["source","shell"] +---- +sudo ls /etc/elasticsearch/certs/ca/ +---- ++ +`ca.crt`:: The generated certificate (or you can substitute this with your own certificate, signed by your organizations's certificate authority) +`ca.key`:: The certificate authority's private key + ++ +These steps to generate new self-signed CA certificates need to be done only on the first {es} node. The other {es} nodes will use the same `ca.crt` and `ca.key` files. + +. From the `/etc/elasticsearch/certs/ca/` directory, import the newly created CA certificate into the {es} truststore. This step ensures that your cluster trusts the new CA certificate. ++ +NOTE: On a new installation a new keystore and truststore are created automatically. If you're running these steps on an existing {es} installation and you know the password to the keystore and the truststore, follow the instructions in {ref}/update-node-certs-different.html[Update security certificates with a different CA] to import the CA certificate. ++ +Run the `keytool` command as shown, replacing `` with a unique password for the truststore, and store the password securely: ++ +["source","shell"] +---- +sudo /usr/share/elasticsearch/jdk/bin/keytool -importcert -trustcacerts -noprompt -keystore /etc/elasticsearch/certs/elastic-stack-ca.p12 -storepass -alias new-ca -file /etc/elasticsearch/certs/ca/ca.crt +---- + +. Ensure that the new key was added to the keystore: ++ +["source","shell"] +---- +sudo /usr/share/elasticsearch/jdk/bin/keytool -keystore /etc/elasticsearch/certs/elastic-stack-ca.p12 -list +---- ++ +NOTE: The keytool utility is provided as part of the {es} installation and is located at: `/usr/share/elasticsearch/jdk/bin/keytool` on RPM installations. ++ +Enter your password when prompted. The result should show the details for your newly added key: ++ +["source","shell"] +---- +Keystore type: jks +Keystore provider: SUN +Your keystore contains 1 entry +new-ca, Jul 12, 2023, trustedCertEntry, +Certificate fingerprint (SHA-256): F0:86:6B:57:FC... +---- + +[discrete] +[[install-stack-demo-secure-transport]] +== Step 2: Generate a new certificate for the transport layer + +This guide assumes the use of self-signed certificates, but the process to import CA-signed certificates is the same. If you're using a CA provided by your organization, you need to generate Certificate Signing Requests (CSRs) and then use the signed certificates in this step. Once the certificates are generated, whether self-signed or CA-signed, the steps are the same. + +. From the {es} installation directory, using the newly-created CA certificate and private key, create a new certificate for your elasticsearch node: ++ +["source","shell"] +---- +sudo /usr/share/elasticsearch/bin/elasticsearch-certutil cert --ca-cert /etc/elasticsearch/certs/ca/ca.crt --ca-key /etc/elasticsearch/certs/ca/ca.key +---- ++ +When prompted, choose an output file name (you can use the default `elastic-certificates.p12`) and a password for the certificate. + +. Move the generated file to the `/etc/elasticsearch/certs` directory: ++ +["source","shell"] +---- +sudo mv /usr/share/elasticsearch/elastic-certificates.p12 /etc/elasticsearch/certs/ +---- + ++ +[IMPORTANT] +==== +If you're running these steps on a production cluster that already contains data: + +* In a cluster with multiple {es} nodes, before proceeding you first need to perform a {ref}/restart-cluster.html#restart-cluster-rolling[Rolling restart] beginning with the node where you're updating the keystore. Stop at the `Perform any needed changes` step, and then proceed to the next step in this guide. +* In a single node cluster, always stop {es} before proceeding. +==== + ++ +. Because you've created a new truststore and keystore, you need to update the `/etc/elasticsearch/elasticsearch.yml` settings file with the new truststore and keystore filenames. ++ +Open the {es} configuration file in a text editor and adjust the following values to reflect the newly created keystore and truststore filenames and paths: ++ +["source","shell"] +---- +xpack.security.transport.ssl: + ... + keystore.path: /etc/elasticsearch/certs/elastic-certificates.p12 + truststore.path: /etc/elasticsearch/certs/elastic-stack-ca.p12 +---- + +[discrete] +[[install-stack-demo-secure-transport-es-keystore]] +=== Update the {es} keystore + +{es} uses a separate keystore to hold the passwords of the keystores and truststores holding the CA and node certificates created in the previous steps. Access to this keystore is through the use of a utility called `elasticsearch-keystore`. + +. From the {es} installation directory, list the contents of the existing keystore: ++ +["source","shell"] +---- +sudo /usr/share/elasticsearch/bin/elasticsearch-keystore list +---- ++ +The results should be like the following: ++ +["source","yaml"] +---- +keystore.seed +xpack.security.http.ssl.keystore.secure_password +xpack.security.transport.ssl.keystore.secure_password +xpack.security.transport.ssl.truststore.secure_password +---- ++ +Notice that there are entries for: ++ +* The `transport.ssl.truststore` that holds the CA certificate +* The `transport.ssl.keystore` that holds the CA-signed certificates +* The `http.ssl.keystore` for the HTTP layer ++ +These entries were created at installation and need to be replaced with the passwords to the newly-created truststore and keystores. + +. Remove the existing keystore values for the default transport keystore and truststore: ++ +["source","shell"] +---- +sudo /usr/share/elasticsearch/bin/elasticsearch-keystore remove xpack.security.transport.ssl.keystore.secure_password + +sudo /usr/share/elasticsearch/bin/elasticsearch-keystore remove xpack.security.transport.ssl.truststore.secure_password +---- + +. Update the `elasticsearch-keystore` with the passwords for the new keystore and truststore created in the previous steps. This ensures that {es} can read the new stores: ++ +["source","shell"] +---- +sudo /usr/share/elasticsearch/bin/elasticsearch-keystore add xpack.security.transport.ssl.keystore.secure_password + +sudo /usr/share/elasticsearch/bin/elasticsearch-keystore add xpack.security.transport.ssl.truststore.secure_password +---- + +[discrete] +[[install-stack-demo-secure-http]] +== Step 3: Generate new certificate(s) for the HTTP layer + +Now that communication between {es} nodes (the transport layer) has been secured with SSL certificates, the same must be done for the communications that use the REST API, including {kib}, clients, and any other components on the HTTP layer. + +. Similar to the process for the transport layer, on the first node in your {es} cluster use the certificate utility to generate a CA certificate for HTTP communications: ++ +["source","shell"] +---- +sudo /usr/share/elasticsearch/bin/elasticsearch-certutil http +---- ++ +Respond to the command prompts as follows: + +* When asked if you want to generate a CSR, enter `n`. +* When asked if you want to use an existing CA, enter `y`. ++ +NOTE: If you're using your organization's CA certificate, specify that certificate and key in the following two steps. ++ +* Provide the absolute path to your newly created CA certificate: `/etc/elasticsearch/certs/ca/ca.crt`. +* Provide the absolute path to your newly created CA key: `/etc/elasticsearch/certs/ca/ca.key`. +* Enter an expiration value for your certificate. You can enter the validity period in years, months, or days. For example, enter `1y` for one year. +* When asked if you want to generate one certificate per node, enter `y`. ++ +You'll be guided through the creation of certificates for each node. Each certificate will have its own private key, and will be issued for a specific hostname or IP address. + +.. Enter the hostname for your first {es} node, for example `mynode-es1`. ++ +["source","shell"] +---- +mynode-es1 +---- +.. When prompted, confirm that the settings are correct. +.. Add the network IP address that clients can use to connect to the first {es} node. This is the same value that's described in Step 2 of <>, for example `10.128.0.84`: ++ +["source","shell"] +---- +10.128.0.84 +---- +.. When prompted, confirm that the settings are correct. +.. When prompted, choose to generate additional certificates, and then repeat the previous steps to add hostname and IP settings for each node in your {es} cluster. +.. Provide a password for the generated `http.p12` keystore file. +.. The generated files will be included in a zip archive. At the prompt, provide a path and filename for where the archive should be created. ++ +For this example we use: `/etc/elasticsearch/certs/elasticsearch-ssl-http.zip`: ++ +["source","shell"] +---- +What filename should be used for the output zip file? [/usr/share/elasticsearch/elasticsearch-ssl-http.zip] /etc/elasticsearch/certs/elasticsearch-ssl-http.zip +---- + +. Earlier, when you generated the certificate for the transport layer, the default filename was `elastic-certificates.p12`. Now, when generating a certificate for the HTTP layer, the default filename is `http.p12`. This matches the name of the existing HTTP layer certificate file from when the initial {es} cluster was first installed. ++ +Just to avoid any possible name collisions, rename the existing `http.p12` file to distinguish it from the newly-created keystore: ++ +["source","shell"] +---- +sudo mv /etc/elasticsearch/certs/http.p12 /etc/elasticsearch/certs/http-old.p12 +---- + +. Unzip the generated `elasticsearch-ssl-http.zip` archive: ++ +["source","shell"] +---- +sudo unzip -d /usr/share/elasticsearch/ /etc/elasticsearch/certs/elasticsearch-ssl-http.zip +---- + +. When the archive is unpacked, the certificate files are located in separate directories for each {es} node and for the {kib} node. ++ +You can run a recursive `ls` command to view the file structure: ++ +["source","shell"] +---- +ls -lR /usr/share/elasticsearch/{elasticsearch,kibana} +---- ++ +["source","shell"] +---- +elasticsearch: +total 0 +drwxr-xr-x. 2 root root 56 Dec 12 19:13 mynode-es1 +drwxr-xr-x. 2 root root 72 Dec 12 19:04 mynode-es2 +drwxr-xr-x. 2 root root 72 Dec 12 19:04 mynode-es3 + +elasticsearch/mynode-es1: +total 8 +-rw-r--r--. 1 root root 1365 Dec 12 19:04 README.txt +-rw-r--r--. 1 root root 845 Dec 12 19:04 sample-elasticsearch.yml + +elasticsearch/mynode-es2: +total 12 +-rw-r--r--. 1 root root 3652 Dec 12 19:04 http.p12 +-rw-r--r--. 1 root root 1365 Dec 12 19:04 README.txt +-rw-r--r--. 1 root root 845 Dec 12 19:04 sample-elasticsearch.yml + +elasticsearch/mynode-es3: +total 12 +-rw-r--r--. 1 root root 3652 Dec 12 19:04 http.p12 +-rw-r--r--. 1 root root 1365 Dec 12 19:04 README.txt +-rw-r--r--. 1 root root 845 Dec 12 19:04 sample-elasticsearch.yml + +kibana: +total 12 +-rw-r--r--. 1 root root 1200 Dec 12 19:04 elasticsearch-ca.pem +-rw-r--r--. 1 root root 1306 Dec 12 19:04 README.txt +-rw-r--r--. 1 root root 1052 Dec 12 19:04 sample-kibana.yml +---- + +. Replace your existing keystore with the new keystore. The location of your certificate directory may be different than what is shown here, depending on the installation method you chose. ++ +Run the `mv` command, replacing `` with the hostname of your initial {es} node: ++ +["source","shell"] +---- +sudo mv /usr/share/elasticsearch/elasticsearch//http.p12 /etc/elasticsearch/certs/ +---- + +. Because this is a new keystore, the {es} configuration file needs to be updated with the path to its location. Open `/etc/elasticsearch/elasticsearch.yml` and update the HTTP SSL settings with the new path: ++ +["source","yaml"] +---- +xpack.security.http.ssl: + enabled: true + #keystore.path: certs/http.p12 + keystore.path: /etc/elasticsearch/certs/http.p12 +---- + +. Since you also generated a new keystore password, the {es} keystore needs to be updated as well. From the {es} installation directory, first remove the existing HTTP keystore entry: ++ +["source","shell"] +---- +sudo /usr/share/elasticsearch/bin/elasticsearch-keystore remove xpack.security.http.ssl.keystore.secure_password +---- + +. Add the updated HTTP keystore password, using the password you generated for this keystore: ++ +["source","shell"] +---- +sudo /usr/share/elasticsearch/bin/elasticsearch-keystore add xpack.security.http.ssl.keystore.secure_password +---- + +. Because we've added files to the {es} configuration directory during this tutorial, we need to ensure that the permissions and ownership are correct before restarting {es}. + +.. Change the files to be owned by `root:elasticsearch`: ++ +["source","shell"] +---- +sudo chown -R root:elasticsearch /etc/elasticsearch/certs/ +---- + +.. Set the files in `/etc/elasticsearch/certs` to have read and write permissions by the owner (`root`) and read permission by the `elastic` user: ++ +["source","shell"] +---- +sudo chmod 640 /etc/elasticsearch/certs/elastic-certificates.p12 +sudo chmod 640 /etc/elasticsearch/certs/elastic-stack-ca.p12 +sudo chmod 640 /etc/elasticsearch/certs/http_ca.crt +sudo chmod 640 /etc/elasticsearch/certs/http.p12 +---- + +.. Change the `/etc/elasticsearch/certs` and `/etc/elasticsearch/certs/ca` directories to be executable by the owner: ++ +["source","shell"] +---- +sudo chmod 750 /etc/elasticsearch/certs +sudo chmod 750 /etc/elasticsearch/certs/ca +---- + +. Restart the {es} service: ++ +["source","shell"] +---- +sudo systemctl start elasticsearch.service +---- + +. Run the status command to confirm that {es} is running: ++ +["source","shell"] +---- +sudo systemctl status elasticsearch.service +---- ++ +In the event of any problems, you can also monitor the {es} logs for any issues by tailing the {es} log file: ++ +["source","shell"] +---- +sudo tail -f /var/log/elasticsearch/elasticsearch-demo.log +---- ++ +A line in the log file like the following indicates that SSL has been properly configured: ++ +["source","shell"] +---- +[2023-07-12T13:11:29,154][INFO ][o.e.x.s.Security ] [es-ssl-test] Security is enabled +---- + +[discrete] +[[install-stack-demo-secure-second-node]] +== Step 4: Configure security on additional {es} nodes + +Now that the security is configured for the first {es} node, some steps need to be repeated on each additional {es} node. + +. To avoid filename collisions, on each additional {es} node rename the existing `http.p12` file in the `/etc/elasticsearch/certs/` directory: ++ +["source","shell"] +---- +mv http.p12 http-old.p12 +---- + +. Copy the CA and truststore files that you generated on the first {es} node so that they can be reused on all other nodes: + +* Copy the `/ca` directory (that contains `ca.crt` and `ca.key`) from `/etc/elasticsearch/certs/` on the first {es} node to the same path on all other {es} nodes. + +* Copy the `elastic-stack-ca.p12` file from `/etc/elasticsearch/certs/` to the `/etc/elasticsearch/certs/` directory on all other {es} nodes. + +* Copy the `http.p12` file from each node directory in `/usr/share/elasticsearch/elasticsearch` (that is, `elasticsearch/mynode-es1`, `elasticsearch/mynode-es2` and `elasticsearch/mynode-es3`) to the `/etc/elasticsearch/certs/` directory on each corresponding cluster node. + +. On each {es} node, repeat the steps to generate a new certificate for the transport layer: + +.. Stop the {es} service: ++ +["source","shell"] +---- +sudo systemctl stop elasticsearch.service +---- + +.. From the `/etc/elasticsearch/certs` directory, create a new certificate for the {es} node: ++ +["source","shell"] +---- +sudo /usr/share/elasticsearch/bin/elasticsearch-certutil cert --ca-cert /etc/elasticsearch/certs/ca/ca.crt --ca-key /etc/elasticsearch/certs/ca/ca.key +---- ++ +When prompted, choose an output file name and specify a password for the certificate. For this example, we'll use `/etc/elasticsearch/certs/elastic-certificates.p12`. + +.. Update the `/etc/elasticsearch/elasticsearch.yml` settings file with the new truststore and keystore filename and path: ++ +["source","shell"] +---- +xpack.security.transport.ssl: + ... + keystore.path: /etc/elasticsearch/certs/elastic-certificates.p12 + truststore.path: /etc/elasticsearch/certs/elastic-stack-ca.p12 +---- + +.. List the content of the {es} keystore: ++ +["source","shell"] +---- +/usr/share/elasticsearch/bin/elasticsearch-keystore list +---- ++ +The results should be like the following: ++ +["source","yaml"] +---- +keystore.seed +xpack.security.http.ssl.keystore.secure_password +xpack.security.transport.ssl.keystore.secure_password +xpack.security.transport.ssl.truststore.secure_password +---- + +.. Remove the existing keystore values for the default transport keystore and truststore: ++ +["source","shell"] +---- +sudo /usr/share/elasticsearch/bin/elasticsearch-keystore remove xpack.security.transport.ssl.keystore.secure_password + +sudo /usr/share/elasticsearch/bin/elasticsearch-keystore remove xpack.security.transport.ssl.truststore.secure_password +---- + +.. Update the `elasticsearch-keystore` with the passwords for the new keystore and truststore: ++ +["source","shell"] +---- +sudo /usr/share/elasticsearch/bin/elasticsearch-keystore add xpack.security.transport.ssl.keystore.secure_password + +sudo /usr/share/elasticsearch/bin/elasticsearch-keystore add xpack.security.transport.ssl.truststore.secure_password +---- + +. For the HTTP layer, the certificates have been generated already on the first {es} node. Each additional {es} node just needs to be configured to use the new certificates. + +.. Update the `/etc/elasticsearch/elasticsearch.yml` settings file with the new truststore and keystore filenames: ++ +["source","shell"] +---- +xpack.security.http.ssl: + enabled: true + #keystore.path: certs/http.p12 + keystore.path: /etc/elasticsearch/certs/http.p12 +---- + +.. Remove the existing HTTP keystore entry: ++ +["source","shell"] +---- +sudo /usr/share/elasticsearch/bin/elasticsearch-keystore remove xpack.security.http.ssl.keystore.secure_password +---- + +.. Add the updated HTTP keystore password: ++ +["source","shell"] +---- +sudo /usr/share/elasticsearch/bin/elasticsearch-keystore add xpack.security.http.ssl.keystore.secure_password +---- + +.. Change the certificate files to be owned by the `root:elasticsearch` group: ++ +["source","shell"] +---- +sudo chown -R root:elasticsearch /etc/elasticsearch/certs/ +---- + +.. Set the files in `/etc/elasticsearch/certs` to have read and write permissions by the owner (`root`) and read permission by the `elastic` user: ++ +["source","shell"] +---- +chmod 640 * +---- + +.. Change the `/etc/elasticsearch/certs` and `/etc/elasticsearch/certs/ca` directories to be executable by the owner: ++ +["source","shell"] +---- +chmod 750 /etc/elasticsearch/certs +chmod 750 /etc/elasticsearch/certs/ca +---- + +. Restart the {es} service. ++ +["source","shell"] +---- +sudo systemctl start elasticsearch.service +---- + +. Run the status command to confirm that {es} is running. ++ +["source","shell"] +---- +sudo systemctl status elasticsearch.service +---- + +[discrete] +[[install-stack-demo-secure-kib-es]] +== Step 5: Generate server-side and client-side certificates for {kib} + +Now that the transport and HTTP layers are configured with encryption using the new certificates, there are two more tasks that must be accomplished for end-to-end connectivity to {es}: Set up certificates for encryption between {kib} and {es}, and between the client browser and {kib}. For additional details about any of these steps, refer to {kibana-ref}/elasticsearch-mutual-tls.html[Mutual TLS authentication between {kib} and {es}] and {ref}/security-basic-setup-https.html#encrypt-kibana-browser[Encrypt traffic between your browser and {kib}]. + +. In Step 3, when you generated a new certificate for the HTTP layer, the process created an archive `elasticsearch-ssl-http.zip`. ++ +From the `kibana` directory in the expanded archive, copy the `elasticsearch-ca.pem` private key file to the {kib} host machine. + +. On the {kib} host machine, copy `elasticsearch-ca.pem` to the {kib} configuration directory (depending on the installation method that you used, the location of the configuration directory may be different from what's shown): ++ +["source","shell"] +---- +mv elasticsearch-ca.pem /etc/kibana +---- + +. Stop the {kib} service: ++ +["source","shell"] +---- +sudo systemctl stop kibana.service +---- + +. Update the `/etc/kibana/kibana.yml` settings file to reflect the location of the `elasticsearch-ca.pem`: ++ +["source","sh",subs="attributes"] +---- +elasticsearch.ssl.certificateAuthorities: [/etc/kibana/elasticsearch-ca.pem] +---- + +. Log in to the first Elasticsearch node and use the certificate utility to generate a certificate bundle for the Kibana server. This certificate will be used to encrypt the traffic between Kibana and the client's browser. In the command, replace and with the name and IP address of your Kibana server host: ++ +["source","shell"] +---- +sudo /usr/share/elasticsearch/bin/elasticsearch-certutil cert --name kibana-server --ca-cert /etc/elasticsearch/certs/ca/ca.crt --ca-key /etc/elasticsearch/certs/ca/ca.key --dns --ip --pem +---- ++ +When prompted, specify a unique name for the output file, such as `kibana-cert-bundle.zip`. + +. Copy the generated archive over to your Kibana host and unpack it: ++ +["source","shell"] +---- +sudo unzip kibana-cert-bundle.zip +---- ++ +The unpacked archive will create a directory, `kibana-server`, containing the new {kib} key and certificate: ++ +["source","shell"] +---- +ls -l kibana-server/ +total 8 +-rw-r--r--. 1 root root 1208 May 3 16:08 kibana-server.crt +-rw-r--r--. 1 root root 1675 May 3 16:08 kibana-server.key +---- + +. Copy the certificate and key into `/etc/kibana`: ++ +["source","shell"] +---- +sudo cp kibana-server.crt /etc/kibana/ +sudo cp kibana-server.key /etc/kibana/ +---- + +. Update the permissions on the certificate files to ensure that they’re readable. From inside the `/etc/kibana` directory, run: ++ +["source","shell"] +---- +sudo chmod 640 *.crt +sudo chmod 640 *.key +---- + +. Open `/etc/kibana/kibana.yml` and make the following changes: ++ +["source","sh",subs="attributes"] +---- +server.ssl.certificate: /etc/kibana/kibana-server.crt +server.ssl.key: /etc/kibana/kibana-server.key +server.ssl.enabled: true +---- ++ +Keep the file open for the next step. + +. To ensure that {kib} sessions are not invalidated, set up an encryption key by assigning any string of 32 characters or longer to the `xpack.security.encryptionKey` setting (this string will be configured in `kibana.yml` and does not need to be remembered). To generate a random string, you can use the following bash commands: ++ +["source","shell"] +---- +cat /dev/urandom | tr -dc 'a-z0-9' | fold -w 32 | head -n 1 +---- ++ +Using your own string or the output of the above command sequence, add the encryption key setting to `/etc/kibana/kibana.yml`: ++ +["source","sh",subs="attributes"] +---- +xpack.security.encryptionKey: previously_create_string +---- ++ +Save and close the file. + +. Restart the {kib} service: ++ +["source","shell"] +---- +sudo systemctl start kibana.service +---- + +. Confirm that {kib} is running: ++ +["source","shell"] +---- +sudo systemctl status kibana.service +---- ++ +If everything is configured correctly, connection to {es} will be established and {kib} will start normally. + +. You can also view the {kib} log file to gather more detail: ++ +["source","shell"] +---- +tail -f /var/log/kibana/kibana.log +---- ++ +In the log file you should find a `Kibana is now available` message. + +. You should now have an end-to-end ecnrypted deployment with {es} and {kib} that provides encryption between both the cluster nodes and {kib}, and HTTPS access to {kib}. ++ +Open a web browser to the external IP address of the Kibana host machine: `https://:5601`. Note that the URL should use the `https` and not the `http` protocol. + +. Log in using the `elastic` user and password that you configured in Step 1 of <>. + +Congratulations! You've successfully updated the SSL certificates between {es} and {kib}. + +[discrete] +[[install-stack-demo-secure-fleet]] +== Step 6: Install {fleet} with SSL certificates configured + +Now that {kib} is up and running, you can proceed to install {fleet-server}, which will manage the {agent} that we'll set up in a later step. + +If you'd like to learn more about these steps, refer to {fleet-guide}/add-fleet-server-on-prem.html[Deploy on-premises and self-managed] in the {fleet} and {agent} Guide. You can find detailed steps to generate and configure certificates in {fleet-guide}/secure-connections.html[Configure SSL/TLS for self-managed Fleet Servers]. + +. Log in to the first {es} node and use the certificate utility to generate a certificate bundle for {fleet-server}. In the command, replace `` and `IP address` with the name and IP address of your {fleet-server} host: ++ +["source","shell"] +---- +sudo /usr/share/elasticsearch/bin/elasticsearch-certutil cert --name fleet-server --ca-cert /etc/elasticsearch/certs/ca/ca.crt --ca-key /etc/elasticsearch/certs/ca/ca.key --dns --ip --pem +---- ++ +When prompted, specify a unique name for the output file, such as `fleet-cert-bundle.zip`. + +. On your {fleet-server} host, create a directory for the certificate files: ++ +["source","shell"] +---- +sudo mkdir /etc/fleet +---- + +. Copy the generated archive over to your {fleet-server} host and unpack it into `/etc/fleet/`: +** `/etc/fleet/fleet-server.crt` +** `/etc/fleet/fleet-server.key`` + +. From the first {es} node, copy the `ca.crt` file, and paste it into the `/etc/fleet/` directory on the {fleet-server} host. Just to help identify the file we'll also rename it to `es-ca.crt`: +** `/etc/fleet/es-ca.crt` + +. Update the permissions on the certificate files to ensure that they're readable. From inside the `/etc/fleet` directory, run: ++ +["source","shell"] +---- +sudo chmod 640 *.crt +sudo chmod 640 *.key +---- + +. Now that the certificate files are in place, on the {fleet-server} host create a working directory for the installation package: ++ +["source","shell"] +---- +mkdir fleet-install-files +---- + +. Change into the new directory: ++ +["source","shell"] +---- +cd fleet-install-files +---- + +. In the terminal, run the `ifconfig` command and copy the value for the host inet IP address (for example, `10.128.0.84`). You'll need this value later. + +. Back in your web browser, open the {kib} menu and go to **Management -> Fleet**. {fleet} opens with a message that you need to add a {fleet-server}. + +. Click **Add Fleet Server**. The **Add a Fleet Server** flyout opens. + +. In the flyout, select the **Advanced** tab. + +. On the **Create a policy for Fleet Server** step, keep the default {fleet-server} policy name and all advanced options at their defaults. ++ +Leave the option to collect system logs and metrics selected. Click *Create policy*. The policy takes a minute or so to create. + +. On the **Choose a deployment mode for security** step, select the **Production** option. This enables you to provide your own certificates. + +. On the **Add your Fleet Server host** step: + +.. Specify a name for your {fleet-server} host, for example `Fleet Server`. +.. Specify the host URL and where {agents} will reach {fleet-server}, including the default port `8220`. For example, `https://10.128.0.203:8220`. ++ +The URL is the inet value that you copied from the `ifconfig` output. ++ +For details about default port assignments, refer to {fleet-guide}/add-fleet-server-on-prem.html#default-port-assignments-on-prem[Default port assignments] in the on-premises {fleet-server} install documentation. + +.. Click **Add host**. + +. On the **Generate a service token** step, generate the token and save the output. The token will also be propagated automatically to the command to install {fleet-server}. + +. On the **Install Fleet Server to a centralized host** step, for this example we select the **Linux Tar** tab, but you can instead select the tab appropriate to the host operating system where you're setting up {fleet-server}. ++ +Note that TAR/ZIP packages are recommended over RPM/DEB system packages, since only the former support upgrading {fleet-server} using {fleet}. + +. Run the first three commands one-by-one in the terminal on your {fleet-server} host. ++ +These commands will, respectively: + +.. Download the {fleet-server} package from the {artifact-registry}. +.. Unpack the package archive. +.. Change into the directory containing the install binaries. + +. Before running the provided `elastic-agent install` command, you'll need to make a few changes: + +.. Update the paths to the correct file locations: +** The {es} CA file (`es-ca.crt`) +** The {fleet-server} certificate (`fleet-server.crt`) +** The {fleet-server} key (`fleet-server.key`) + +.. The `fleet-server-es-ca-trusted-fingerprint` also needs to be updated. On any of your {es} hosts, run the following command to get the correct fingerprint to use: ++ +["source","shell"] +---- +grep -v ^- /etc/elasticsearch/certs/ca/ca.crt | base64 -d | sha256sum +---- ++ +Save the fingerprint value. You'll need it in a later step. ++ +Replace the `fleet-server-es-ca-trusted-fingerprint` setting with the returned value. Your updated command should be similar to the following: ++ +["source","shell"] +---- +sudo ./elastic-agent install -url=https://10.128.0.208:8220 \ + --fleet-server-es=https://10.128.0.84:9200 \ + --fleet-server-service-token=AAEAAWVsYXN0aWMvZmxlZXQtc2VydmPyL6Rva2VuLTE5OTg4NzAxOTM4NDU6X1I0Q1RrRHZTSWlyNHhkSXQwNEJoQQ \ + --fleet-server-policy=fleet-server-policy \ + --fleet-server-es-ca-trusted-fingerprint=92b51cf91e7fa311f8c84849224d448ca44824eb \ + --certificate-authorities=/etc/fleet/es-ca.crt \ + --fleet-server-cert=/etc/fleet/fleet-server.crt \ + --fleet-server-cert-key=/etc/fleet/fleet-server.key \ + --fleet-server-port=8220 +---- ++ +For details about all of the install command options, refer to {fleet-guide}/elastic-agent-cmd-options.html#elastic-agent-install-command[`elastic-agent install`] in the {agent} command reference. + +. After you've made the required updates, run the `elastic-agent install` command to install {fleet-server}. ++ +When prompted, confirm that {agent} should run as a service. If everything goes well, the install will complete successfully: ++ +["source","shell"] +---- +Elastic Agent has been successfully installed. +---- ++ +TIP: Wondering why the command refers to {agent} rather than {fleet-server}? {fleet-server} is actually a subprocess that runs inside {agent} with a special {fleet-server} policy. Refer to {fleet-guide}/fleet-server.html[What is {fleet-server}] to learn more. + +. Return to the {kib} **Add a Fleet Server** flyout and wait for confirmation that {fleet-server} has connected. + +. Once the connection is confirmed, ignore the *Continue enrolling Elastic Agent* option and close the flyout. + +{fleet-server} is now fully set up! + +Before proceeding to install {agent}, there are a few steps needed to update the `kibana.yml` settings file with the {es} CA fingerprint: + +.. On your {kib} host, stop the {kib} service: ++ +["source","shell"] +---- +sudo systemctl stop kibana.service +---- +.. Open `/etc/kibana/kibana.yml` for editing. +.. Find the `xpack.fleet.outputs` setting. +.. Update `ca_trusted_fingerprint` to the value you captured earlier, when you ran the `grep` command on the {es} `ca.crt` file. ++ +The updated entry in `kibana.yml` should be like the following: ++ +["source","yaml"] +---- +xpack.fleet.outputs: [{id: fleet-default-output, name: default, is_default: true, is_default_monitoring: true, type: elasticsearch, hosts: [`https://10.128.0.84:9200`], ca_trusted_fingerprint: 92b51cf91e7fa311f8c84849224d448ca44824eb}] +---- +.. Save your changes. +.. Restart {kib}: ++ +["source","shell"] +---- +sudo systemctl start kibana.service +---- ++ +{kib} is now configured with the correct fingerprint for {agent} to access {es}. You're now ready to set up {agent}! + +[discrete] +[[install-stack-demo-secure-agent]] +== Step 7: Install {agent} + +Next, we'll install {agent} on another host and use the System integration to monitor system logs and metrics. You can find additional details about these steps in {fleet-guide}/secure-connections.html[Configure SSL/TLS for self-managed Fleet Servers]. + +. Log in to the host where you'd like to set up {agent}. + +. Create a directory for the {es} certificate file: ++ +["source","shell"] +---- +sudo mkdir /etc/agent +---- +. From the first {es} node, copy the `ca.crt` file, and paste it into the `/etc/agent/` directory on the {fleet-server} host. Just to help identify the file we'll also rename it to `es-ca.crt`: +** `/etc/fleet/es-ca.crt` + +. Create a working directory for the installation package: ++ +["source","shell"] +---- +mkdir agent-install-files +---- + +. Change into the new directory: ++ +["source","shell"] +---- +cd agent-install-files +---- + +. Open {kib} and go to **Management -> Fleet**. + +. On the **Agents** tab, you should see your new {fleet-server} policy running with a healthy status. + +. Click **Add agent**. The **Add agent** flyout opens. + +. In the flyout, choose an agent policy name, for example `Demo Agent Policy`. + +. Leave **Collect system logs and metrics** enabled. This will add the link:https://docs.elastic.co/integrations/system[System integration] to the {agent} policy. + +. Click **Create policy**. + +. For the **Enroll in Fleet?** step, leave **Enroll in Fleet** selected. + +. On the **Install Elastic Agent on your host** step, for this example we select the **Linux Tar** tab, but you can instead select the tab appropriate to the host operating system where you're setting up {fleet-server}. ++ +As with {fleet-server}, note that TAR/ZIP packages are recommended over RPM/DEB system packages, since only the former support upgrading {agent} using {fleet}. + +. Run the first three commands one-by-one in the terminal on your {agent} host. ++ +These commands will, respectively: + +.. Download the {agent} package from the {artifact-registry}. +.. Unpack the package archive. +.. Change into the directory containing the install binaries. + +. Before running the provided `elastic-agent install` command, you'll need to make a few changes: + +.. For the `--url` parameter, confirm that the port number is `8220` (this is the default port for on-premises {fleet-server}). ++ +.. Add a `--certificate-authorities` parameter with the full path of your CA certificate file. For example, `--certificate-authorities=/etc/agent/es-ca.crt`. ++ +The result should be like the following: ++ +["source","shell"] +---- +sudo ./elastic-agent install \ +--url=https://10.128.0.203:8220 \ +--enrollment-token=VWCobFhKd0JuUnppVYQxX0VKV5E6UmU3BGk0ck9RM2HzbWEmcS4Bc1YUUM== \ +--certificate-authorities=/etc/agent/es-ca.crt +---- + +. Run the `elastic-agent install` command. ++ +At the prompt, enter `Y` to install {agent} and run it as a service. wait for the installation to complete. ++ +If everything goes well, the install will complete successfully: ++ +["source","shell"] +---- +Elastic Agent has been successfully installed. +---- + +. In the {kib} **Add agent** flyout, wait for confirmation that {agent} has connected. + +. Wait for the **Confirm incoming data** step to complete. This may take a couple of minutes. + +. Once data is confirmed to be flowing, close the flyout. + +Your new {agent} is now installed an enrolled with {fleet-server}. + +[discrete] +[[install-stack-demo-secure-view-data]] +== Step 8: View your system data + +Now that all of the components have been installed, it's time to view your system data. + +View your system log data: + +. Open the {kib} menu and go to **Analytics -> Dashboard**. +. In the query field, search for `Logs System`. +. Select the `[Logs System] Syslog dashboard` link. The {kib} Dashboard opens with visualizations of Syslog events, hostnames and processes, and more. + +View your system metrics data: + +. Open the {kib} menu and return to **Analytics -> Dashboard**. +. In the query field, search for `Metrics System`. +. Select the `[Metrics System] Host overview` link. The {kib} Dashboard opens with visualizations of host metrics including CPU usage, memory usage, running processes, and more. ++ +image::images/install-stack-metrics-dashboard.png["The System metrics host overview showing CPU usage, memory usage, and other visualizations"] + +Congratulations! You've successfully configured security for {es}, {kib}, {fleet}, and {agent} using your own trusted CA-signed certificates. + +[discrete] +== What's next? + +* Do you have data ready to ingest into your newly set up {stack}? Learn how to {cloud}/ec-cloud-ingest-data.html[add data to Elasticsearch]. + +* Use https://www.elastic.co/observability[Elastic {observability}] +to unify your logs, infrastructure metrics, uptime, and application performance data. + +* Want to protect your endpoints from security threats? Try +https://www.elastic.co/security[{elastic-sec}]. Adding endpoint protection is +just another integration that you add to the agent policy! diff --git a/docs/en/install-upgrade/installing-stack-demo-self.asciidoc b/docs/en/install-upgrade/installing-stack-demo-self.asciidoc new file mode 100644 index 000000000..39420348a --- /dev/null +++ b/docs/en/install-upgrade/installing-stack-demo-self.asciidoc @@ -0,0 +1,734 @@ +//for testing on currently available builds: +//:version: 8.11.1 + +[[installing-stack-demo-self]] +=== Tutorial 1: Installing a self-managed {stack} + +This tutorial demonstrates how to install and configure the {stack} in a self-managed environment. Following these steps, you'll set up a three node {es} cluster, with {kib}, {fleet-server}, and {agent}, each on separate hosts. The {agent} will be configured with the System integration, enabling it to gather local system logs and metrics and deliver them into the {es} cluster. Finally, you'll learn how to view the system data in {kib}. + +It should take between one and two hours to complete these steps. + +* <> +* <> +* <> +* <> +* <> +* <> +* <> +* <> +* <> +* <> +* <> +* <> + +[IMPORTANT] +==== +If you're using these steps to configure a production cluster that uses trusted CA-signed certificates for secure communications, after completing Step 6 to install {kib} we recommend jumping directly to <>. + +The second tutorial includes steps to configure security across the {stack}, and then to set up {fleet-server} and {agent} with SSL certificates enabled. +==== + +[discrete] +[[install-stack-self-prereqs]] +== Prerequisites and assumptions + +To get started, you'll need the following: + +* A set of virtual or physical hosts on which to install each stack component. +* On each host, a super user account with `sudo` privileges. + +The examples in this guide use RPM packages to install the {stack} components on hosts running Red Hat Enterprise Linux 8. The steps for other install methods and operating systems are similar, and can be found in the documentation linked from each section. The packages that you'll install are: + +* https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{version}-x86_64.rpm + +* https://artifacts.elastic.co/downloads/kibana/kibana-{version}-x86_64.rpm + +* https://artifacts.elastic.co/downloads/beats/elastic-agent/elastic-agent-{version}-linux-x86_64.tar.gz + +NOTE: For {agent} and {fleet-server} (both of which use the elastic-agent-{version}-linux-x86_64.tar.gz package) we recommend using TAR/ZIP packages over RPM/DEB system packages, since only the former support upgrading using {fleet}. + +Special considerations such as firewalls and proxy servers are not covered here. + +For the basic ports and protocols required for the installation to work, refer to the following overview section. + +[discrete] +[[install-stack-self-overview]] +== {stack} overview + +Before starting, take a moment to familiarize yourself with the {stack} components. + +image::images/stack-install-final-state.png[Image showing the relationships between stack components] + +To learn more about the {stack} and how each of these components are related, refer to {estc-welcome-current}/stack-components.html[An overview of the {stack}]. + +[discrete] +[[install-stack-self-elasticsearch-first]] +== Step 1: Set up the first {es} node + +To begin, use RPM to install {es} on the first host. This initial {es} instance will serve as the master node. + +. Log in to the host where you'd like to set up your first {es} node. + +. Create a working directory for the installation package: ++ +["source","shell"] +---- +mkdir elastic-install-files +---- + +. Change into the new directory: ++ +["source","shell"] +---- +cd elastic-install-files +---- + +. Download the {es} RPM and checksum file from the {artifact-registry}. You can find details about these steps in the section {ref}/rpm.html#install-rpm[Download and install the RPM manually]. ++ +["source","sh",subs="attributes"] +---- +wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{version}-x86_64.rpm +wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{version}-x86_64.rpm.sha512 +---- + +. Confirm the validity of the downloaded package by checking the SHA of the downloaded RPM against the published checksum: ++ +["source","sh",subs="attributes"] +---- +shasum -a 512 -c elasticsearch-{version}-x86_64.rpm.sha512 +---- ++ +The command should return: `elasticsearch-{version}-x86_64.rpm: OK`. + +. Run the {es} install command: ++ +["source","sh",subs="attributes"] +---- +sudo rpm --install elasticsearch-{version}-x86_64.rpm +---- ++ +The {es} install process enables certain security features by default, including the following: + +* Authentication and authorization are enabled, including a built-in `elastic` superuser account. +* Certificates and keys for TLS are generated for the transport and HTTP layer, and TLS is enabled and configured with these keys and certificates. + +. Copy the terminal output from the install command to a local file. In particular, you'll need the password for the built-in `elastic` superuser account. The output also contains the commands to enable {es} to run as a service, which you'll use in the next step. + +. Run the following two commands to enable {es} to run as a service using `systemd`. This enables {es} to start automatically when the host system reboots. You can find details about this and the following steps in {ref}/starting-elasticsearch.html#start-es-deb-systemd[Running {es} with `systemd`]. ++ +["source","sh",subs="attributes"] +---- +sudo systemctl daemon-reload +sudo systemctl enable elasticsearch.service +---- + +[discrete] +[[install-stack-self-elasticsearch-config]] +== Step 2: Configure the first {es} node for connectivity + +Before moving ahead to configure additional {es} nodes, you'll need to update the {es} configuration on this first node so that other hosts are able to connect to it. This is done by updating the settings in the `elasticsearch.yml` file. For details about all available settings refer to {ref}/settings.html[Configuring {es}]. + +. In a terminal, run the `ifconfig` command and copy the value for the host inet IP address (for example, `10.128.0.84`). You'll need this value later. + +. Open the {es} configuration file in a text editor, such as `vim`: ++ +["source","sh",subs="attributes"] +---- +sudo vim /etc/elasticsearch/elasticsearch.yml +---- + +. In a multi-node {es} cluster, all of the {es} instances need to have the same name. ++ +In the configuration file, uncomment the line `#cluster.name: my-application` and give the {es} instance any name that you'd like: ++ +[source,"yaml"] +---- +cluster.name: elasticsearch-demo +---- + +. By default, {es} runs on `localhost`. In order for {es} instances on other nodes to be able to join the cluster, you'll need to set up {es} to run on a routable, external IP address. ++ +Uncomment the line `#network.host: 192.168.0.1` and replace the default address with the value that you copied from the `ifconfig` command output. For example: ++ +[source,"yaml"] +---- +network.host: 10.128.0.84 +---- + +. {es} needs to be enabled to listen for connections from other, external hosts. ++ +Uncomment the line `#transport.host: 0.0.0.0`. The `0.0.0.0` setting enables {es} to listen for connections on all available network interfaces. Note that in a production environment you might want to restrict this by setting this value to match the value set for `network.host`. ++ +[source,"yaml"] +---- +transport.host: 0.0.0.0 +---- ++ +TIP: You can find details about the `network.host` and `transport.host` settings in the {es} {ref}/modules-network.html[Networking] documentation. + +. Save your changes and close the editor. + +[discrete] +[[install-stack-self-elasticsearch-start]] +== Step 3: Start {es} + +. Now, it's time to start the {es} service: ++ +["source","sh",subs="attributes"] +---- +sudo systemctl start elasticsearch.service +---- ++ +If you need to, you can stop the service by running `sudo systemctl stop elasticsearch.service`. + +. Make sure that {es} is running properly. ++ +["source","sh",subs="attributes"] +---- +sudo curl --cacert /etc/elasticsearch/certs/http_ca.crt -u elastic:$ELASTIC_PASSWORD https://localhost:9200 +---- ++ +In the command, replace `$ELASTIC_PASSWORD` with the `elastic` superuser password that you copied from the install command output. ++ +If all is well, the command returns a response like this: ++ +["source","js",subs="attributes,callouts"] +---- +{ + "name" : "Cp9oae6", + "cluster_name" : "elasticsearch", + "cluster_uuid" : "AT69_C_DTp-1qgIJlatQqA", + "version" : { + "number" : "{version_qualified}", + "build_type" : "{build_type}", + "build_hash" : "f27399d", + "build_flavor" : "default", + "build_date" : "2016-03-30T09:51:41.449Z", + "build_snapshot" : false, + "lucene_version" : "{lucene_version}", + "minimum_wire_compatibility_version" : "1.2.3", + "minimum_index_compatibility_version" : "1.2.3" + }, + "tagline" : "You Know, for Search" +} +---- + +. Finally, check the status of {es}: ++ +[source,"shell"] +---- +sudo systemctl status elasticsearch +---- ++ +As with the previous `curl` command, the output should confirm that {es} started successfully. Type `q` to exit from the `status` command results. + +[discrete] +[[install-stack-self-elasticsearch-second]] +== Step 4: Set up a second {es} node + +To set up a second {es} node, the initial steps are similar to those that you followed for <>. + +. Log in to the host where you'd like to set up your second {es} instance. + +. Create a working directory for the installation package: ++ +["source","shell"] +---- +mkdir elastic-install-files +---- + +. Change into the new directory: ++ +["source","shell"] +---- +cd elastic-install-files +---- + +. Download the {es} RPM and checksum file: ++ +["source","sh",subs="attributes"] +---- +wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{version}-x86_64.rpm +wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{version}-x86_64.rpm.sha512 +---- + +. Check the SHA of the downloaded RPM: ++ +["source","sh",subs="attributes"] +---- +shasum -a 512 -c elasticsearch-{version}-x86_64.rpm.sha512 +---- + +. Run the {es} install command: ++ +["source","sh",subs="attributes"] +---- +sudo rpm --install elasticsearch-{version}-x86_64.rpm +---- ++ +Unlike the setup for the first {es} node, in this case you don't need to copy the output of the install command, since these settings will be updated in a later step. + +. Enable {es} to run as a service: ++ +["source","sh",subs="attributes"] +---- +sudo systemctl daemon-reload +sudo systemctl enable elasticsearch.service +---- ++ +IMPORTANT: Don't start the {es} service yet! There are a few more configuration steps to do before restarting. + +. To enable this second {es} node to connect to the first, you need to configure an enrollment token. ++ +[IMPORTANT] +==== +Be sure to run all of these configuration steps before starting the {es} service. + +You can find additional details about these steps in {ref}/rpm.html#_reconfigure_a_node_to_join_an_existing_cluster_2[Reconfigure a node to join an existing cluster] and also in {ref}/add-elasticsearch-nodes.html#_enroll_nodes_in_an_existing_cluster_5[Enroll nodes in an existing cluster]. +==== ++ +Return to your terminal shell on the first {es} node and generate a node enrollment token: ++ +[source,"shell"] +---- +sudo /usr/share/elasticsearch/bin/elasticsearch-create-enrollment-token -s node +---- + +. Copy the generated enrollment token from the command output. ++ +[TIP] +==== +Note the following tips about enrollment tokens: + +. An enrollment token has a lifespan of 30 minutes. In case the `elasticsearch-reconfigure-node` command returns an `Invalid enrollment token` error, try generating a new token. +. Be sure not to confuse an {ref}/starting-elasticsearch.html#_enroll_nodes_in_an_existing_cluster_3[{es} enrollment token] (for enrolling {es} nodes in an existing cluster) with a {kibana-ref}/start-stop.html#_run_kibana_from_the_command_line[{kib} enrollment token] (to enroll your {kib} instance with {es}, as described in the next section). These two tokens are not interchangeable. +==== + +. In the terminal shell for your second {es} node, pass the enrollment token as a parameter to the `elasticsearch-reconfigure-node` tool: ++ +[source,"shell"] +---- +sudo /usr/share/elasticsearch/bin/elasticsearch-reconfigure-node --enrollment-token +---- ++ +In the command, replace `] master node changed {previous [], current [...]} +---- ++ +Here, `hostname1` is your first {es} instance node, and `hostname2` is your second {es} instance node. ++ +The message indicates that the second {es} node has successfully contacted the initial {es} node and joined the cluster. + +. As a final check, run the following `curl` request on the new node to confirm that {es} is still running properly and viewable at the new node's `localhost` IP address. Note that you need to replace `$ELASTIC_PASSWORD` with the same `elastic` superuser password that you used on the first {es} node. ++ +["source","sh",subs="attributes"] +---- +sudo curl --cacert /etc/elasticsearch/certs/http_ca.crt -u elastic:$ELASTIC_PASSWORD https://localhost:9200 +---- ++ +["source","js",subs="attributes,callouts"] +---- +{ + "name" : "Cp9oae6", + "cluster_name" : "elasticsearch", + "cluster_uuid" : "AT69_C_DTp-1qgIJlatQqA", + "version" : { + "number" : "{version_qualified}", + "build_type" : "{build_type}", + "build_hash" : "f27399d", + "build_flavor" : "default", + "build_date" : "2016-03-30T09:51:41.449Z", + "build_snapshot" : false, + "lucene_version" : "{lucene_version}", + "minimum_wire_compatibility_version" : "1.2.3", + "minimum_index_compatibility_version" : "1.2.3" + }, + "tagline" : "You Know, for Search" +} +---- + +[discrete] +[[install-stack-self-elasticsearch-third]] +== Step 5: Set up additional {es} nodes + +To set up your next {es} node, follow exactly the same steps as you did previously in <>. The process is identical for each additional {es} node that you would like to add to the cluster. As a recommended best practice, create a new enrollment token for each new node that you add. + +[discrete] +[[install-stack-self-kibana]] +== Step 6: Install {kib} + +As with {es}, you can use RPM to install {kib} on another host. You can find details about all of the following steps in the section {kibana-ref}/rpm.html#install-rpm[Install {kib} with RPM]. + +. Log in to the host where you'd like to install {kib} and create a working directory for the installation package: ++ +["source","shell"] +---- +mkdir kibana-install-files +---- + +. Change into the new directory: ++ +["source","shell"] +---- +cd kibana-install-files +---- + +. Download the {kib} RPM and checksum file from the Elastic website. ++ +["source","sh",subs="attributes"] +---- +wget https://artifacts.elastic.co/downloads/kibana/kibana-{version}-x86_64.rpm +wget https://artifacts.elastic.co/downloads/kibana/kibana-{version}-x86_64.rpm.sha512 +---- + +. Confirm the validity of the downloaded package by checking the SHA of the downloaded RPM against the published checksum: ++ +["source","sh",subs="attributes"] +---- +shasum -a 512 -c kibana-{version}-x86_64.rpm.sha512 +---- ++ +The command should return: `kibana-{version}-x86_64.rpm: OK`. + +. Run the {kib} install command: ++ +["source","sh",subs="attributes"] +---- +sudo rpm --install kibana-{version}-x86_64.rpm +---- + +. As with each additional {es} node that you added, to enable this {kib} instance to connect to the first {es} node, you need to configure an enrollment token. ++ +Return to your terminal shell into the first {es} node. + +. Run the `elasticsearch-create-enrollment-token` command with the `-s kibana` option to generate a {kib} enrollment token: ++ +[source,"shell"] +---- +sudo /usr/share/elasticsearch/bin/elasticsearch-create-enrollment-token -s kibana +---- + +. Copy the generated enrollment token from the command output. + +. Back on the {kib} host, run the following two commands to enable {kib} to run as a service using `systemd`, enabling {kib} to start automatically when the host system reboots. ++ +["source","sh",subs="attributes"] +---- +sudo systemctl daemon-reload +sudo systemctl enable kibana.service +---- + +. Before starting the {kib} service there's one configuration change to make, to set {kib} to run on the {es} host IP address. This is done by updating the settings in the `kibana.yml` file. For details about all available settings refer to {kibana-ref}/settings.html[Configure {kib}]. + +. In a terminal, run the `ifconfig` command and copy the value for the host inet IP address. + +. Open the {kib} configuration file for editing: ++ +["source","sh",subs="attributes"] +---- +sudo vim /etc/kibana/kibana.yml +---- + +. Uncomment the line `#server.host: localhost` and replace the default address with the inet value that you copied from the `ifconfig` command. For example: ++ +[source,"yaml"] +---- +server.host: 10.128.0.28 +---- + +. Save your changes and close the editor. + +. Start the {kib} service: ++ +["source","sh",subs="attributes"] +---- +sudo systemctl start kibana.service +---- ++ +If you need to, you can stop the service by running `sudo systemctl stop kibana.service`. + +. Run the `status` command to get details about the {kib} service. ++ +["source","sh",subs="attributes"] +---- +sudo systemctl status kibana +---- + +. In the `status` command output, a URL is shown with: +** A host address to access {kib} +** A six digit verification code ++ +For example: ++ +["source","sh",subs="attributes"] +---- +Kibana has not been configured. +Go to http://10.128.0.28:5601/?code= to get started. +---- ++ +Make a note of the verification code. + +. Open a web browser to the external IP address of the {kib} host machine, for example: `http://:5601`. ++ +It can take a minute or two for {kib} to start up, so refresh the page if you don't see a prompt right away. + +. When {kib} starts you're prompted to provide an enrollment token. Paste in the {kib} enrollment token that you generated earlier. + +. Click **Configure Elastic**. + +. If you're prompted to provide a verification code, copy and paste in the six digit code that was returned by the `status` command. Then, wait for the setup to complete. + +. When you see the **Welcome to Elastic** page, provide the `elastic` as the username and provide the password that you copied in Step 1, from the `install` command output when you set up your first {es} node. + +. Click **Log in**. + +{kib} is now fully set up and communicating with your {es} cluster! + +**IMPORTANT: Stop here if you intend to configure SSL certificates.** + +[IMPORTANT] +==== +For simplicity, in this tutorial we're setting up all of the {stack} components without configuring security certificates. You can proceed to configure {fleet}, {agent}, and then confirm that your system data appears in {kib}. + +However, in a production environment, before going further to install {fleet-server} and {agent} it's recommended to update your security settings to use trusted CA-signed certificates as described in <>. + +After new security certificates are configured any {agent}s would need to be reinstalled. If you're currently setting up a production environment, we recommend that you jump directly to Tutorial 2, which includes steps to secure the {stack} using certificates and then to set up {fleet} and {agent} with those certificates already in place. +==== + +[discrete] +[[install-stack-self-fleet-server]] +== Step 7: Install {fleet-server} + +Now that {kib} is up and running, you can install {fleet-server}, which will manage the {agent} that you'll set up in a later step. If you need more detail about these steps, refer to {fleet-guide}/add-fleet-server-on-prem.html[Deploy on-premises and self-managed] in the {fleet} and {agent} Guide. + +. Log in to the host where you'd like to set up {fleet-server}. + +. Create a working directory for the installation package: ++ +["source","shell"] +---- +mkdir fleet-install-files +---- + +. Change into the new directory: ++ +["source","shell"] +---- +cd fleet-install-files +---- + +. In the terminal, run `ifconfig` and copy the value for the host inet IP address (for example, `10.128.0.84`). You'll need this value later. + +. Back to your web browser, open the {kib} menu and go to **Management -> Fleet**. {fleet} opens with a message that you need to add a {fleet-server}. + +. Click **Add Fleet Server**. The **Add a Fleet Server** flyout opens. + +. In the flyout, select the **Quick Start** tab. + +. Specify a name for your {fleet-server} host, for example `Fleet Server`. + +. Specify the host URL where {agents} will reach {fleet-server}, for example: `http://10.128.0.203:8220`. This is the inet value that you copied from the `ifconfig` output. ++ +Be sure to include the port number. Port `8220` is the default used by {fleet-server} in an on-premises environment. Refer to {fleet-guide}/add-fleet-server-on-prem.html#default-port-assignments-on-prem[Default port assignments] in the on-premises {fleet-server} install documentation for a list of port assignments. + +. Click **Generate Fleet Server policy**. A policy is created that contains all of the configuration settings for the {fleet-server} instance. + +. On the **Install Fleet Server to a centralized host** step, for this example we select the **Linux Tar** tab, but you can instead select the tab appropriate to the host operating system where you're setting up {fleet-server}. ++ +Note that TAR/ZIP packages are recommended over RPM/DEB system packages, since only the former support upgrading {fleet-server} using {fleet}. + +. Copy the generated commands and then run them one-by-one in the terminal on your {fleet-server} host. ++ +These commands will, respectively: + +.. Download the {fleet-server} package from the {artifact-registry}. +.. Unpack the package archive. +.. Change into the directory containing the install binaries. +.. Install {fleet-server}. ++ +If you'd like to learn about the install command options, refer to {fleet-guide}/elastic-agent-cmd-options.html#elastic-agent-install-command[`elastic-agent install`] in the {agent} command reference. + +. At the prompt, enter `Y` to install {agent} and run it as a service. Wait for the installation to complete. + +. In the {kib} **Add a Fleet Server** flyout, wait for confirmation that {fleet-server} has connected. + +. For now, ignore the *Continue enrolling Elastic Agent* option and close the flyout. + +{fleet-server} is now fully set up! + +[discrete] +[[install-stack-self-elastic-agent]] +== Step 8: Install {agent} + +Next, you'll install {agent} on another host and use the System integration to monitor system logs and metrics. + +. Log in to the host where you'd like to set up {agent}. + +. Create a working directory for the installation package: ++ +["source","shell"] +---- +mkdir agent-install-files +---- + +. Change into the new directory: ++ +["source","shell"] +---- +cd agent-install-files +---- + +. Open {kib} and go to **Management -> Fleet**. + +. On the **Agents** tab, you should see your new {fleet-server} policy running with a healthy status. + +. Open the **Settings** tab. + +. Reopen the **Agents** tab and select **Add agent**. The **Add agent** flyout opens. + +. In the flyout, choose a policy name, for example `Demo Agent Policy`. + +. Leave **Collect system logs and metrics** enabled. This will add the link:https://docs.elastic.co/integrations/system[System integration] to the {agent} policy. + +. Click **Create policy**. + +. For the **Enroll in Fleet?** step, leave **Enroll in Fleet** selected. + +. On the **Install Elastic Agent on your host** step, for this example we select the **Linux Tar** tab, but you can instead select the tab appropriate to the host operating system where you're setting up {fleet-server}. ++ +As with {fleet-server}, note that TAR/ZIP packages are recommended over RPM/DEB system packages, since only the former support upgrading {agent} using {fleet}. + +. Copy the generated commands. + +. In the `sudo ./elastic-agent install` command, make two changes: +.. For the `--url` parameter, check that the port number is set to `8220` (used for on-premises {fleet-server}). +.. Append an `--insecure` flag at the end. ++ +TIP: If you want to set up secure communications using SSL certificates, refer to <>. ++ +The result should be like the following: ++ +["source","shell"] +---- +sudo ./elastic-agent install --url=https://10.128.0.203:8220 --enrollment-token=VWCobFhKd0JuUnppVYQxX0VKV5E6UmU3BGk0ck9RM2HzbWEmcS4Bc1YUUM== +---- + +. Run the commands one-by-one in the terminal on your {agent} host. The commands will, respectively: + +.. Download the {agent} package from the {artifact-registry}. +.. Unpack the package archive. +.. Change into the directory containing the install binaries. +.. Install {agent}. + +. At the prompt, enter `Y` to install {agent} and run it as a service. Wait for the installation to complete. ++ +If everything goes well, the install will complete successfully: ++ +["source","shell"] +---- +Elastic Agent has been successfully installed. +---- + +. In the {kib} **Add agent** flyout, wait for confirmation that {agent} has connected. + +. Close the flyout. + +Your new {agent} is now installed an enrolled with {fleet-server}. + +[discrete] +[[install-stack-self-view-data]] +== Step 9: View your system data + +Now that all of the components have been installed, it's time to view your system data. + +View your system log data: + +. Open the {kib} menu and go to **Analytics -> Dashboard**. +. In the query field, search for `Logs System`. +. Select the `[Logs System] Syslog dashboard` link. The {kib} Dashboard opens with visualizations of Syslog events, hostnames and processes, and more. + +View your system metrics data: + +. Open the {kib} menu and return to **Analytics -> Dashboard**. +. In the query field, search for `Metrics System`. +. Select the `[Metrics System] Host overview` link. The {kib} Dashboard opens with visualizations of host metrics including CPU usage, memory usage, running processes, and others. ++ +image::images/install-stack-metrics-dashboard.png["The System metrics host overview showing CPU usage, memory usage, and other visualizations"] + +Congratulations! You've successfully set up a three node {es} cluster, with {kib}, {fleet-server}, and {agent}. + +[discrete] +[[install-stack-self-next-steps]] +== Next steps + +Now that you've successfully configured an on-premises {stack}, you can learn how to configure the {stack} in a production environment using trusted CA-signed certificates. Refer to <> to learn more. + +You can also start using your newly set up {stack} right away: + +* Do you have data ready to ingest? Learn how to {cloud}/ec-cloud-ingest-data.html[add data to Elasticsearch]. + +* Use https://www.elastic.co/observability[Elastic {observability}] +to unify your logs, infrastructure metrics, uptime, and application performance data. + +* Want to protect your endpoints from security threats? Try +https://www.elastic.co/security[{elastic-sec}]. Adding endpoint protection is +just another integration that you add to the agent policy! diff --git a/docs/en/install-upgrade/installing-stack.asciidoc b/docs/en/install-upgrade/installing-stack.asciidoc index 950937d4f..3018ed488 100644 --- a/docs/en/install-upgrade/installing-stack.asciidoc +++ b/docs/en/install-upgrade/installing-stack.asciidoc @@ -8,6 +8,8 @@ Kibana {version}, and Logstash {version}. If you're upgrading an existing installation, see <> for information about how to ensure compatibility with {version}. +For an example of installing and configuring the {stack}, you can try out our <>. After that, you can also learn how to secure your installation for production following the steps in <>. + [discrete] [[network-requirements]] === Network requirements @@ -61,7 +63,7 @@ Install the Elastic Stack products you want to use in the following order: . Elasticsearch ({ref}/install-elasticsearch.html[install instructions]) . Kibana ({kibana-ref}/install.html[install]) . Logstash ({logstash-ref}/installing-logstash.html[install]) -. Beats ({beats-ref}/getting-started.html[install instructions]) +. Elastic Agent ({fleet-guide}/elastic-agent-installation.html[install instructions]) or Beats ({beats-ref}/getting-started.html[install instructions]) . APM ({apm-guide-ref}/apm-quick-start.html[install instructions]) . Elasticsearch Hadoop ({hadoop-ref}/install.html[install instructions]) diff --git a/docs/en/install-upgrade/upgrading-elasticsearch.asciidoc b/docs/en/install-upgrade/upgrading-elasticsearch.asciidoc index dfb2efdc0..4fa2b7e24 100644 --- a/docs/en/install-upgrade/upgrading-elasticsearch.asciidoc +++ b/docs/en/install-upgrade/upgrading-elasticsearch.asciidoc @@ -1,18 +1,6 @@ [[upgrading-elasticsearch]] == Upgrade {es} -.FIPS Compliance and Java 17 -[IMPORTANT] --- -{es} {version} requires Java 17 or later. -There is not yet a FIPS-certified security module for Java 17 -that you can use when running {es} {version} in FIPS 140-2 mode. -If you run in FIPS 140-2 mode, you will either need to request -an exception from your security organization to upgrade to {es} {version}, -or remain on {es} 7.x until Java 17 is certified. -Alternatively, consider using {ess} in the FedRAMP-certified GovCloud region. --- - An {es} cluster can be upgraded one node at a time so upgrading does not interrupt service. Running multiple versions of {es} in the same cluster beyond the duration of an upgrade is @@ -22,16 +10,20 @@ running the older version. Before you start, <>. When performing a <>: -. Upgrade nodes that are **NOT** {ref}/modules-node.html#master-node[master-eligible] first. -You can retrieve a list of these nodes with `GET /_nodes/_all,master:false/_none` or by finding all the nodes configured with `node.master: false`. +. Upgrade the data nodes first, tier-by-tier, starting with the frozen tier, +then the cold tier, then the warm tier, then the hot tier, and finally any other +data nodes which are not in a tier. Complete the upgrade for all nodes in each +data tier before moving to the next. This ensures {ilm-init} can continue to +move data through the tiers during the upgrade. You can get the list of nodes in +a specific tier with a `GET /_nodes` request, for example: +`GET /_nodes/data_frozen:true/_none`. -. Upgrade nodes tier-by-tier, starting with the frozen tier. -Complete the upgrade for all nodes in each data tier before moving to the next. -Upgrade the frozen tier, then the cold tier, then the warm tier, and upgrade the hot tier last. This ensures {ilm-init} can continue to move data through the tiers during the upgrade. You can get the list of nodes in a specific tier with a `GET /_nodes` request, -for example: `GET /_nodes/data_frozen:true/_none`. +. Upgrade all remaining nodes that are neither master-eligible nor data nodes. +This includes dedicated ML nodes, dedicated ingest nodes, and dedicated +coordinating nodes. -. Upgrade the master-eligible nodes last. You can retrieve a list -of these nodes with `GET /_nodes/master:true`. +. Upgrade the master-eligible nodes last. You can retrieve a list of these nodes +with `GET /_nodes/master:true/_none`. This order ensures that all nodes can join the cluster during the upgrade. Upgraded nodes can join a cluster with an older master, but older nodes cannot diff --git a/docs/en/install-upgrade/upgrading-stack.asciidoc b/docs/en/install-upgrade/upgrading-stack.asciidoc index cadf90734..4219c92ac 100644 --- a/docs/en/install-upgrade/upgrading-stack.asciidoc +++ b/docs/en/install-upgrade/upgrading-stack.asciidoc @@ -63,7 +63,7 @@ even if you opt to do a full-cluster restart of your {es} cluster.** Alternatively, you can create a new {version} deployment and reindex from remote. For more information, see <>. -Beats and Logstash 7.n are compatible with {es} {version} +{beats} and {ls} {prev-major-last} are compatible with {es} {version} to give you flexibility in scheduling the upgrade. .Remote cluster compatibility diff --git a/docs/en/reference-architectures/hot-frozen.asciidoc b/docs/en/reference-architectures/hot-frozen.asciidoc new file mode 100644 index 000000000..68e585ca4 --- /dev/null +++ b/docs/en/reference-architectures/hot-frozen.asciidoc @@ -0,0 +1,161 @@ +[[hot-frozen-architecture]] +== Hot/Frozen - High Availability + +The Hot/Frozen High Availability architecture is cost optimized for large time-series datasets. +In this architecture, the hot tier is primarily used for indexing, searching, and continuity for automated processes. +https://www.elastic.co/guide/en/elasticsearch/reference/current/searchable-snapshots.html[Searchable snapshots] are taken from hot into a repository, such as a cloud object store or an on-premises shared filesystem, and then cached to any desired volume on the local disks of the frozen tier. +Data in the repository is indexed for fast retrieval and accessed on-demand from the frozen nodes. +Index and snapshot lifecycle management are used to automate this process. + +This architecture is ideal for time-series use cases, such as Observability or Security, that do not require updating. +All the necessary components of the {stack} are included. +This is not intended for sizing workloads, but rather as a basis to ensure that your cluster is ready to handle any desired workload with resiliency. +A very high level representation of data flow is included, and for more detail around ingest architecture see our {ingest-guide}/use-case-arch.html[ingest architectures] documentation. + +[discrete] +[[hot-frozen-use-case]] +=== Hot/Frozen use case + +This Hot/Frozen – High Availability architecture is intended for organizations that: + +* Have a requirement for cost effective long term data storage (many months or years). +* Provide insights and alerts using logs, metrics, traces, or various event types to ensure optimal performance and quick issue resolution for applications. +* Apply https://www.elastic.co/guide/en/kibana/current/xpack-ml-anomalies.html[machine learning anomaly detection] to help detect patterns in time series data to find root cause and resolve problems faster. +* Use an AI assistant (https://www.elastic.co/guide/en/observability/current/obs-ai-assistant.html[Observability], https://www.elastic.co/guide/en/security/current/security-assistant.html[Security], or https://www.elastic.co/guide/en/kibana/current/playground.html[Playground]) for investigation, incident response, reporting, query generation, or query conversion from other languages using natural language. +* Deploy an architecture model that allows for maximum flexibility between storage cost and performance. + +[IMPORTANT] +==== +**Automated operations that frequently read large data volumes require both high availability (replicas) and predictable low latency (hot, warm or cold tier).** + +* Common examples of these tasks include look-back windows on security detection/alert rules, transforms, machine learning jobs, or watches; and long running scroll queries or external extract processes. +* These operations should be completed before moving the data into a frozen tier. +==== + +[discrete] +[[hot-frozen-architecture-diagram]] +=== Architecture + +image::images/hot-frozen.png["A Hot/Frozen Highly available architecture"] + +TIP: We use an Availability Zone (AZ) concept in the architecture above. +When running in your own Data Center (DC) you can equate AZs to failure zones within a datacenter, racks, or even separate physical machines depending on your constraints. + +The diagram illustrates an {es} cluster deployed across 3 availability zones (AZ). For production we recommend a minimum of 2 availability zones and 3 availability zones for mission critical applications. See https://www.elastic.co/guide/en/cloud/current/ec-planning.html[Plan for production] for more details. A cluster that is running in {ecloud} that has data nodes in only two AZs will create a third master-eligible node in a third AZ. High availability cannot be achieved without three zones for any distributed computing technology. + +The number of data nodes shown for each tier (hot and frozen) is illustrative and would be scaled up depending on ingest volume and retention period. Hot nodes contain both primary and replica shards. By default, primary and replica shards are always guaranteed to be in different availability zones in {ess}, but when self-deploying {ref}/shard-allocation-awareness.html[shard allocation awareness] would need to be configured. Frozen nodes act as a large high-speed cache and retrieve data from the snapshot store as needed. + +Machine learning nodes are optional but highly recommended for large scale time series use cases since the amount of data quickly becomes too difficult to analyze. Applying techniques such as machine learning based anomaly detection or Search AI with large language models helps to dramatically speed up problem identification and resolution. + +[discrete] +[[hot-frozen-hardware]] +=== Recommended hardware specifications + +With {ecloud} you can deploy clusters in AWS, Azure, and Google Cloud. Available hardware types and configurations vary across all three cloud providers but each provides instance types that meet our recommendations for the node types used in this architecture. For more details on these instance types, see our documentation on {ecloud} hardware for https://www.elastic.co/guide/en/cloud/current/ec-default-aws-configurations.html[AWS], https://www.elastic.co/guide/en/cloud/current/ec-default-azure-configurations.html[Azure], and https://www.elastic.co/guide/en/cloud/current/ec-default-gcp-configurations.html[GCP]. The **Physical** column below is guidance, based on the cloud node types, when self-deploying {es} in your own data center. + +In the links provided above, Elastic has performance tested hardware for each of the cloud providers to find the optimal hardware for each node type. We use ratios to represent the best mix of CPU, RAM, and disk for each type. In some cases the CPU to RAM ratio is key, in others the disk to memory ratio and type of disk is critical. Significantly deviating from these ratios may seem like a way to save on hardware costs, but may result in an {es} cluster that does not scale and perform well. + +This table shows our specific recommendations for nodes in a Hot/Frozen architecture. + +|=== +| **Type** | **AWS** | **Azure** | **GCP** | **Physical** +|image:images/hot.png["Hot data node"] | +c6gd | +f32sv2| + + +N2| +16-32 vCPU + +64 GB RAM + +2-6 TB NVMe SSD + +|image:images/frozen.png["Frozen data node"] +| +i3en +| +e8dsv4 +| +N2| +8 vCPU + +64 GB RAM + +6-20+ TB NVMe SSD + +Depending on days cached +|image:images/machine-learning.png["Machine learning node"] +| +m6gd +| +f16sv2 +| +N2| +16 vCPU + +64 GB RAM + +256 GB SSD +|image:images/master.png["Master node"] +| +c5d +| +f16sv2 +| +N2| +8 vCPU + +16 GB RAM + +256 GB SSD +|image:images/kibana.png["Kibana node"] +| +c6gd +| +f16sv2 +| +N2| +8-16 vCPU + +8 GB RAM + +256 GB SSD +|=== + +[discrete] +[[hot-frozen-considerations]] +=== Important considerations + + +**Updating data:** + +* Typically, time series logging use cases are append-only and there is rarely a need to update documents. The frozen tier is read-only. + +**Multi-AZ frozen tier:** + +* Three availability zones is ideal, but at least two availability zones are recommended to ensure that there will be data nodes available in the event of an AZ failure. + +**Shard management:** + +* The most important foundational step to maintaining performance as you scale is proper shard management. This includes even shard distribution amongst nodes, shard size, and shard count. For a complete understanding of what shards are and how they should be used, refer to https://www.elastic.co/guide/en/elasticsearch/reference/current/size-your-shards.html[Size your shards]. + +**Snapshots:** + +* If auditable or business critical events are being logged, a backup is necessary. The choice to back up data will depend on each individual business's needs and requirements. Refer to our https://www.elastic.co/guide/en/elasticsearch/reference/current/snapshots-register-repository.html[snapshot repository] documentation to learn more. +* To automate snapshots and attach to Index lifecycle management policies, refer to https://www.elastic.co/guide/en/elasticsearch/reference/current/snapshots-take-snapshot.html#automate-snapshots-slm[SLM (Snapshot lifecycle management)]. + +**Kibana:** + +* If self-deploying outside of {ess}, ensure that {kib} is configured for https://www.elastic.co/guide/en/kibana/current/production.html#high-availability[high availability]. + +[discrete] +[[hot-frozen-estimate]] +=== How many nodes of each do you need? +It depends on: + +* The type of data being ingested (such as logs, metrics, traces) +* The retention period of searchable data (such as 30 days, 90 days, 1 year) +* The amount of data you need to ingest each day +* The number of dashboards, queries, query types and how frequent they are run. + +You can https://www.elastic.co/contact[contact us] for an estimate and recommended configuration based on your specific scenario. + +[discrete] +[[hot-frozen-resources]] +=== Resources and references + +* https://www.elastic.co/guide/en/elasticsearch/reference/current/scalability.html[{es} - Get ready for production] + +* https://www.elastic.co/guide/en/cloud/current/ec-prepare-production.html[{ess} - Preparing a deployment for production] + +* https://www.elastic.co/guide/en/elasticsearch/reference/current/size-your-shards.html[Size your shards] diff --git a/docs/en/reference-architectures/images/frozen.png b/docs/en/reference-architectures/images/frozen.png new file mode 100644 index 000000000..5e7d53bff Binary files /dev/null and b/docs/en/reference-architectures/images/frozen.png differ diff --git a/docs/en/reference-architectures/images/hot-frozen.png b/docs/en/reference-architectures/images/hot-frozen.png new file mode 100644 index 000000000..1e12b4b25 Binary files /dev/null and b/docs/en/reference-architectures/images/hot-frozen.png differ diff --git a/docs/en/reference-architectures/images/hot.png b/docs/en/reference-architectures/images/hot.png new file mode 100644 index 000000000..ebfab3925 Binary files /dev/null and b/docs/en/reference-architectures/images/hot.png differ diff --git a/docs/en/reference-architectures/images/kibana.png b/docs/en/reference-architectures/images/kibana.png new file mode 100644 index 000000000..78a39ae74 Binary files /dev/null and b/docs/en/reference-architectures/images/kibana.png differ diff --git a/docs/en/reference-architectures/images/machine-learning.png b/docs/en/reference-architectures/images/machine-learning.png new file mode 100644 index 000000000..3508f0478 Binary files /dev/null and b/docs/en/reference-architectures/images/machine-learning.png differ diff --git a/docs/en/reference-architectures/images/master.png b/docs/en/reference-architectures/images/master.png new file mode 100644 index 000000000..9de92464c Binary files /dev/null and b/docs/en/reference-architectures/images/master.png differ diff --git a/docs/en/reference-architectures/index.asciidoc b/docs/en/reference-architectures/index.asciidoc new file mode 100644 index 000000000..397114105 --- /dev/null +++ b/docs/en/reference-architectures/index.asciidoc @@ -0,0 +1,9 @@ +include::{docs-root}/shared/versions/stack/{source_branch}.asciidoc[] +include::{docs-root}/shared/attributes.asciidoc[] + +[[reference-architectures]] += Reference architectures + +include::reference-architectures-overview.asciidoc[] + +include::hot-frozen.asciidoc[] diff --git a/docs/en/reference-architectures/reference-architectures-overview.asciidoc b/docs/en/reference-architectures/reference-architectures-overview.asciidoc new file mode 100644 index 000000000..4c0977786 --- /dev/null +++ b/docs/en/reference-architectures/reference-architectures-overview.asciidoc @@ -0,0 +1,32 @@ +[[reference-architectures-overview]] += Reference architectures + +Elasticsearch reference architectures are blueprints for deploying Elasticsearch clusters tailored to different use cases. Whether you're handling logs or metrics these reference architectures focus on scalability, reliability, and cost efficiency. Use these guidelines to deploy Elasticsearch for your use case. + +These architectures are designed by architects and engineers to provide standardized, proven solutions that help you to follow best practices when deploying {es}. + +TIP: These architectures are specific to running your deployment on-premises or on cloud. If you are using Elastic serverless your {es} clusters are autoscaled and fully managed by Elastic. For all the deployment options, refer to https://www.elastic.co/guide/en/elasticsearch/reference/current/elasticsearch-intro-deploy.html[Run Elasticsearch]. + +These reference architectures are recommendations and should be adapted to fit your specific environment and needs. Each solution can vary based on the unique requirements and conditions of your deployment. In these architectures we discuss about how to deploy cluster components. For information about designing ingest architectures to feed content into your cluster, refer to https://www.elastic.co/guide/en/ingest/current/use-case-arch.html[Ingest architectures] + +[discrete] +[[reference-architectures-time-series-2]] +=== Architectures + +[cols="50, 50"] +|=== +| *Architecture* | *When to use* +| <> + +A high availability architecture that is cost optimized for large time-series datasets. + +a| +* Have a requirement for cost effective long term data storage (many months or years). +* Provide insights and alerts using logs, metrics, traces, or various event types to ensure optimal performance and quick issue resolution for applications. +* Apply Machine Learning and Search AI to assist in dealing with the large amount of data. +* Deploy an architecture model that allows for maximum flexibility between storage cost and performance. +| Additional architectures are on the way. + +Stay tuned for updates. | + +|=== diff --git a/docs/en/stack/ml/anomaly-detection/anomaly-detection-scale.asciidoc b/docs/en/stack/ml/anomaly-detection/anomaly-detection-scale.asciidoc index 5326b8567..d338f9101 100644 --- a/docs/en/stack/ml/anomaly-detection/anomaly-detection-scale.asciidoc +++ b/docs/en/stack/ml/anomaly-detection/anomaly-detection-scale.asciidoc @@ -65,7 +65,7 @@ size of a {ml} node. Creating {ml-jobs} with model memory limits larger than the maximum node size can support is not allowed, as autoscaling cannot add a node big enough to run the job. On a self-managed deployment, you can set `xpack.ml.max_model_memory_limit` according to the available resources of the -{ml} node. This prevents you from you creating jobs with model memory limits too +{ml} node. This prevents you from creating jobs with model memory limits too high to open in your cluster. [discrete] diff --git a/docs/en/stack/ml/anomaly-detection/anomaly-how-tos.asciidoc b/docs/en/stack/ml/anomaly-detection/anomaly-how-tos.asciidoc index eb8b1d4aa..783c4e688 100644 --- a/docs/en/stack/ml/anomaly-detection/anomaly-how-tos.asciidoc +++ b/docs/en/stack/ml/anomaly-detection/anomaly-how-tos.asciidoc @@ -10,9 +10,11 @@ The guides in this section describe some best practices for generating useful {ml} results and insights from your data. * <> -* <> +* <> * <> * <> +* <> +* <> * <> * <> * <> diff --git a/docs/en/stack/ml/anomaly-detection/geographic-anomalies.asciidoc b/docs/en/stack/ml/anomaly-detection/geographic-anomalies.asciidoc index 02bcae275..99f52ff3f 100644 --- a/docs/en/stack/ml/anomaly-detection/geographic-anomalies.asciidoc +++ b/docs/en/stack/ml/anomaly-detection/geographic-anomalies.asciidoc @@ -211,6 +211,11 @@ behavior: image::images/ecommerce-anomaly-explorer-geopoint.jpg[A screenshot of an anomalous event in the eCommerce data in Anomaly Explorer] // NOTE: This is an autogenerated screenshot. Do not edit it directly. +A "typical" value indicates a centroid of a cluster of previously observed +locations that is closest to the "actual" location at that time. For example, +there may be one centroid near the user's home and another near the user's +work place since there are many records associated with these distinct locations. + Likewise, there are time periods in the web logs sample data where there are both unusually high sums of data transferred and unusual geographical coordinates: diff --git a/docs/en/stack/ml/anomaly-detection/images/anomaly-explorer-alerts.png b/docs/en/stack/ml/anomaly-detection/images/anomaly-explorer-alerts.png new file mode 100644 index 000000000..e0cbc214f Binary files /dev/null and b/docs/en/stack/ml/anomaly-detection/images/anomaly-explorer-alerts.png differ diff --git a/docs/en/stack/ml/anomaly-detection/images/categorization-wizard.png b/docs/en/stack/ml/anomaly-detection/images/categorization-wizard.png new file mode 100644 index 000000000..a3f9a0de2 Binary files /dev/null and b/docs/en/stack/ml/anomaly-detection/images/categorization-wizard.png differ diff --git a/docs/en/stack/ml/anomaly-detection/images/ml-category-analyzer.png b/docs/en/stack/ml/anomaly-detection/images/ml-category-analyzer.png new file mode 100644 index 000000000..34c0d2032 Binary files /dev/null and b/docs/en/stack/ml/anomaly-detection/images/ml-category-analyzer.png differ diff --git a/docs/en/stack/ml/anomaly-detection/images/ml-category-anomalies.png b/docs/en/stack/ml/anomaly-detection/images/ml-category-anomalies.png new file mode 100644 index 000000000..a50f96ff2 Binary files /dev/null and b/docs/en/stack/ml/anomaly-detection/images/ml-category-anomalies.png differ diff --git a/docs/en/stack/ml/anomaly-detection/images/ml-population-anomalies.png b/docs/en/stack/ml/anomaly-detection/images/ml-population-anomalies.png new file mode 100644 index 000000000..8ff3c3996 Binary files /dev/null and b/docs/en/stack/ml/anomaly-detection/images/ml-population-anomalies.png differ diff --git a/docs/en/stack/ml/anomaly-detection/images/ml-population-anomaly.png b/docs/en/stack/ml/anomaly-detection/images/ml-population-anomaly.png new file mode 100644 index 000000000..da367f8d5 Binary files /dev/null and b/docs/en/stack/ml/anomaly-detection/images/ml-population-anomaly.png differ diff --git a/docs/en/stack/ml/anomaly-detection/images/ml-population-wizard.png b/docs/en/stack/ml/anomaly-detection/images/ml-population-wizard.png new file mode 100644 index 000000000..14e86d686 Binary files /dev/null and b/docs/en/stack/ml/anomaly-detection/images/ml-population-wizard.png differ diff --git a/docs/en/stack/ml/anomaly-detection/index.asciidoc b/docs/en/stack/ml/anomaly-detection/index.asciidoc index 0a7f3338b..1db6e4475 100644 --- a/docs/en/stack/ml/anomaly-detection/index.asciidoc +++ b/docs/en/stack/ml/anomaly-detection/index.asciidoc @@ -36,6 +36,10 @@ include::{es-repo-dir}/ml/anomaly-detection/ml-configuring-transform.asciidoc[le include::{es-repo-dir}/ml/anomaly-detection/ml-configuring-detector-custom-rules.asciidoc[leveloffset=+2] +include::ml-detect-categories.asciidoc[leveloffset=+2] + +include::ml-population-analysis.asciidoc[leveloffset=+2] + include::ml-revert-model-snapshot.asciidoc[leveloffset=+2] include::geographic-anomalies.asciidoc[leveloffset=+2] @@ -52,40 +56,43 @@ include::ml-ad-resources.asciidoc[leveloffset=+1] include::ml-limitations.asciidoc[leveloffset=+2] -include::{es-repo-dir}/ml/anomaly-detection/functions/ml-functions.asciidoc[leveloffset=+2] +include::ml-ad-troubleshooting.asciidoc[leveloffset=+2] -include::ootb-ml-jobs.asciidoc[leveloffset=+2] +include::{es-repo-dir}/ml/anomaly-detection/functions/ml-functions.asciidoc[leveloffset=+1] -include::ml-ad-troubleshooting.asciidoc[leveloffset=+2] +include::{es-repo-dir}/ml/anomaly-detection/functions/ml-count-functions.asciidoc[leveloffset=+2] + +include::{es-repo-dir}/ml/anomaly-detection/functions/ml-geo-functions.asciidoc[leveloffset=+2] + +include::{es-repo-dir}/ml/anomaly-detection/functions/ml-info-functions.asciidoc[leveloffset=+2] + +include::{es-repo-dir}/ml/anomaly-detection/functions/ml-metric-functions.asciidoc[leveloffset=+2] -include::ootb-ml-jobs-apache.asciidoc[] +include::{es-repo-dir}/ml/anomaly-detection/functions/ml-rare-functions.asciidoc[leveloffset=+2] -include::ootb-ml-jobs-apm.asciidoc[] +include::{es-repo-dir}/ml/anomaly-detection/functions/ml-sum-functions.asciidoc[leveloffset=+2] -include::ootb-ml-jobs-auditbeat.asciidoc[] +include::{es-repo-dir}/ml/anomaly-detection/functions/ml-time-functions.asciidoc[leveloffset=+2] -include::ootb-ml-jobs-logs-ui.asciidoc[] -include::ootb-ml-jobs-metricbeat.asciidoc[] +include::ootb-ml-jobs.asciidoc[leveloffset=+1] -include::ootb-ml-jobs-metrics-ui.asciidoc[] +include::ootb-ml-jobs-apache.asciidoc[leveloffset=+2] -include::ootb-ml-jobs-nginx.asciidoc[] +include::ootb-ml-jobs-apm.asciidoc[leveloffset=+2] -include::ootb-ml-jobs-siem.asciidoc[] +include::ootb-ml-jobs-auditbeat.asciidoc[leveloffset=+2] -include::ootb-ml-jobs-uptime.asciidoc[] +include::ootb-ml-jobs-logs-ui.asciidoc[leveloffset=+2] -include::{es-repo-dir}/ml/anomaly-detection/functions/ml-count-functions.asciidoc[] +include::ootb-ml-jobs-metricbeat.asciidoc[leveloffset=+2] -include::{es-repo-dir}/ml/anomaly-detection/functions/ml-geo-functions.asciidoc[] +include::ootb-ml-jobs-metrics-ui.asciidoc[leveloffset=+2] -include::{es-repo-dir}/ml/anomaly-detection/functions/ml-info-functions.asciidoc[] +include::ootb-ml-jobs-nginx.asciidoc[leveloffset=+2] -include::{es-repo-dir}/ml/anomaly-detection/functions/ml-metric-functions.asciidoc[] +include::ootb-ml-jobs-siem.asciidoc[leveloffset=+2] -include::{es-repo-dir}/ml/anomaly-detection/functions/ml-rare-functions.asciidoc[] +include::ootb-ml-jobs-uptime.asciidoc[leveloffset=+2] -include::{es-repo-dir}/ml/anomaly-detection/functions/ml-sum-functions.asciidoc[] -include::{es-repo-dir}/ml/anomaly-detection/functions/ml-time-functions.asciidoc[] diff --git a/docs/en/stack/ml/anomaly-detection/ml-ad-job-types.asciidoc b/docs/en/stack/ml/anomaly-detection/ml-ad-job-types.asciidoc index 35d109e74..4dc04db87 100644 --- a/docs/en/stack/ml/anomaly-detection/ml-ad-job-types.asciidoc +++ b/docs/en/stack/ml/anomaly-detection/ml-ad-job-types.asciidoc @@ -82,7 +82,9 @@ event is anomalous if the request rate of an IP address is unusually high or low compared to the request rate of all IP addresses in the population. The population job builds a model of the typical number of requests for the IP addresses collectively and compares the behavior of each IP address against that -collective model to detect outliers. +collective model to detect outliers. + +Refer to <> to learn more. [discrete] @@ -114,6 +116,8 @@ job can detect anomalous behavior, such as an unusual number of events in a category by using the `count` function or messages that rarely occur by using the `rare` function. +Refer to <> to learn more. + [discrete] [[rare-jobs]] diff --git a/docs/en/stack/ml/anomaly-detection/ml-ad-run-jobs.asciidoc b/docs/en/stack/ml/anomaly-detection/ml-ad-run-jobs.asciidoc index 56e3a7473..e56648d71 100644 --- a/docs/en/stack/ml/anomaly-detection/ml-ad-run-jobs.asciidoc +++ b/docs/en/stack/ml/anomaly-detection/ml-ad-run-jobs.asciidoc @@ -33,8 +33,8 @@ a {dfeed} will be required. You can create {anomaly-jobs} by using the {ref}/ml-put-job.html[create {anomaly-jobs} API]. {kib} also provides wizards to simplify the process, which vary depending on whether you are using -the {ml-app} app, {security-app} or {observability} apps. In *{ml-app}* > -*Anomaly Detection*: +the {ml-app} app, {security-app} or {observability} apps. To open *Anomaly Detection*, +find *{ml-app}* in the main menu, or use the {kibana-ref}/kibana-concepts-analysts.html#_finding_your_apps_and_objects[global search field]. [role="screenshot"] image::images/ml-create-job.png[Create New Job] diff --git a/docs/en/stack/ml/anomaly-detection/ml-ad-troubleshooting.asciidoc b/docs/en/stack/ml/anomaly-detection/ml-ad-troubleshooting.asciidoc index 25a5addc7..1f827a261 100644 --- a/docs/en/stack/ml/anomaly-detection/ml-ad-troubleshooting.asciidoc +++ b/docs/en/stack/ml/anomaly-detection/ml-ad-troubleshooting.asciidoc @@ -1,15 +1,183 @@ [role="xpack"] [[ml-ad-troubleshooting]] -= Troubleshooting {ml} {anomaly-detect} += Troubleshooting {ml} {anomaly-detect} and frequently asked questions ++++ -Troubleshooting +Troubleshooting and FAQ ++++ Use the information in this section to troubleshoot common problems and find answers for frequently asked questions. + [discrete] [[ml-ad-restart-failed-jobs]] -== Restart failed {anomaly-jobs} +== How to restart failed {anomaly-jobs} + +include::ml-restart-failed-jobs.asciidoc[] + +[discrete] +[[faq-methods]] +== What {ml} methods are used for {anomaly-detect}? + +For detailed information, refer to the paper https://www.ijmlc.org/papers/398-LC018.pdf[Anomaly Detection in Application Performance Monitoring Data] by Thomas Veasey and Stephen Dodson, as well as our webinars on https://www.elastic.co/elasticon/conf/2018/sf/the-math-behind-elastic-machine-learning[The Math behind Elastic Machine Learning] and +https://www.elastic.co/elasticon/conf/2017/sf/machine-learning-and-statistical-methods-for-time-series-analysis[Machine Learning and Statistical Methods for Time Series Analysis]. + +Further papers cited in the C++ code: + +* http://arxiv.org/pdf/1109.2378.pdf[Modern hierarchical, agglomerative clustering algorithms] +* https://www.cs.umd.edu/~mount/Projects/KMeans/pami02.pdf[An Efficient k-Means Clustering Algorithm: Analysis and Implementation] +* http://www.stat.columbia.edu/~madigan/PAPERS/techno.pdf[Large-Scale Bayesian Logistic Regression for Text Categorization] +* https://www.cs.cmu.edu/~dpelleg/download/xmeans.pdf[X-means: Extending K-means with Efficient Estimation of the Number of Clusters] + + +[discrete] +[[faq-features]] +== What are the input features used by the model? + +All input features are specified by the user, for example, using +https://www.elastic.co/guide/en/machine-learning/current/ml-functions.html[diverse statistical functions] +like count or mean over the data of interest. + + +[discrete] +[[faq-data]] +== Does the data used by the model only include customers' data? + +Yes. Only the data specified in the {anomaly-job} configuration are used for +detection. + + +[discrete] +[[faq-output-score]] +== What does the model output score represent? How is it generated and calibrated? + +The ensemble model generates a probability value, which is then mapped to an +anomaly severity score between 0 and 100. The lower the probability of observed +data, the higher the severity score. Refer to this +<> for details. Calibration (also called as +normalization) happens on two levels: + +. Within the same metric/partition, the scores are re-normalized “back in time” +within the window specified by the `renormalization_window_days` parameter. +This is the reason, for example, that both `record_score` and +`initial_record_score` exist. +. Over multiple partitions, scores are renormalized as described in +https://www.elastic.co/blog/changes-to-elastic-machine-learning-anomaly-scoring-in-6-5[this blog post]. + + +[discrete] +[[faq-model-update]] +== Is the model static or updated periodically? + +It's an online model and updated continuously. Old parts of the model are pruned +out based on the parameter `model_prune_window` (usually 30 days). + + +[discrete] +[[faq-model-performance]] +== Is the performance of the model monitored? + +There is a set of benchmarks to monitor the performance of the {anomaly-detect} +algorithms and to ensure no regression occurs as the methods are continuously +developed and refined. They are called "data scenarios" and consist of 3 things: + +* a dataset (stored as an {es} snapshot), +* a {ml} config ({anomaly-detect}, {dfanalysis}, {transform}, or {infer}), +* an arbitrary set of static assertions (bucket counts, anomaly scores, accuracy +value, and so on). + +Performance metrics are collected from each and every scenario run and they are +persisted in an Elastic Cloud cluster. This information is then used to track +the performance over time, across the different builds, mainly to detect any +regressions in the performance (both result quality and compute time). + +On the customer side, the situation is different. There is no conventional way +to monitor the model performance as it's unsupervised. Usually, +operationalization of the model output include one or several of the following +steps: + +* Creating alerts for influencers, buckets, or records based on a certain +anomaly score. +* Use the forecasting feature to predict the development of the metric of +interest in the future. +* Use one or a combination of multiple {anomaly-jobs} to identify the +significant anomaly influencers. + + +[discrete] +[[faq-model-accuracy]] +== How to measure the accuracy of the unsupervised {ml} model? + +For each record in a given time series, anomaly detection models provide an +anomaly severity score, 95% confidence intervals, and an actual value. This data +is stored in an index and can be retrieved using the Get Records API. With this +information, you can use standard measures to assess prediction accuracy, +interval calibration, and so on. Elasticsearch aggregations can be used to +compute these statistics. + +The purpose of {anomaly-detect} is to achieve the best ranking of periods where +an anomaly happened. A practical way to evaluate this is to keep track of real +incidents and see how well they correlate with the predictions of +{anomaly-detect}. + + +[discrete] +[[faq-model-drift]] +== Can the {anomaly-detect} model experience model drift? + +Elasticsearch's {anomaly-detect} model continuously learns and adapts to changes +in the time series. These changes can take the form of slow drifts as well as +sudden jumps. Therefore, we take great care to manage the adaptation to changing +data characteristics. There is always a fine trade-off between fitting anomalous +periods (over-fitting) and not learning new normal behavior. The following are +the main approaches Elastic uses to manage this trade-off: + +* Learning the optimal decay rate based on measuring the bias in the forecast +and the moments of the error distribution and error distribution moments. +* Allowing continuous small drifts in periodic patterns. This is achieved by +continuously minimizing the mean prediction error over the last iteration with +respect to a small bounded time shift. +* If the predictions are significantly wrong over a long period of time, the +algorithm tests whether the time series has undergone a sudden change. +Hypothesis Testing is used to test for different types of changes, such as +scaling of values, shifting of values, and large time shifts in periodic +patterns such as daylight saving time. +* Running continuous hypothesis tests on time windows of various lengths to test +for significant evidence of new or changed periodic patterns, and update the +model if the null hypothesis of unchanged features is rejected. +* Accumulating error statistics on calendar days and continuously test whether +predictive calendar features need to be added or removed from the model. + + +[discrete] +[[faq-minimum-data]] +== What is the minimum amount of data for an {anomaly-job}? + +Elastic {ml} needs a minimum amount of data to be able to build an effective +model for {anomaly-detect}. + +* For sampled metrics such as `mean`, `min`, `max`, +and `median`, the minimum data amount is either eight non-empty bucket spans or +two hours, whichever is greater. +* For all other non-zero/null metrics and count-based quantities, it's four +non-empty bucket spans or two hours, whichever is greater. +* For the `count` and `sum` functions, empty buckets matter and therefore it is +the same as sampled metrics - eight buckets or two hours. +* For the `rare` function, it's typically around 20 bucket spans. It can be faster +for population models, but it depends on the number of people that interact per +bucket. + +Rules of thumb: + +* more than three weeks for periodic data or a few hundred buckets for +non-periodic data +* at least as much data as you want to forecast + + +[discrete] +[[faq-data-integrity]] +== Are there any checks or processes to ensure data integrity? -include::ml-restart-failed-jobs.asciidoc[] \ No newline at end of file +The Elastic {ml} algorithms are programmed to work with missing and noisy data +and use denoising and data reputation techniques based on the learned +statistical properties. \ No newline at end of file diff --git a/docs/en/stack/ml/anomaly-detection/ml-buckets.asciidoc b/docs/en/stack/ml/anomaly-detection/ml-buckets.asciidoc index 06e7159e4..e466d6a78 100644 --- a/docs/en/stack/ml/anomaly-detection/ml-buckets.asciidoc +++ b/docs/en/stack/ml/anomaly-detection/ml-buckets.asciidoc @@ -43,6 +43,22 @@ typical and actual values, and probability. The **Anomaly explanation** section helps you to interpret a given anomaly by providing further insights about its type, impact, and score. +If you have <> applied to an {anomaly-job} and an +alert has occured for the rule, you can view how the alert correlates with the +{anomaly-detect} results in the **Anomaly Explorer** by using the +**Anomaly timeline** swimlane and the **Alerts** panel. The **Alerts** panel +contains a line chart with the alerts count over time. The cursor on the line +chart is in sync with the anomaly swimlane making it easier to review anomalous +buckets with the spike produced by the alerts. The panel also contains +aggregated information for each alert rule associated with the job selection +such as the total number of active, recovered, and untracked alerts for the +selected job and time range. An alert context menu is displayed when an anomaly +swimlane cell is selected with alerts in the chosen time range. The context menu +contains the alert counters for the selected time buckets. + +[role="screenshot"] +image::images/anomaly-explorer-alerts.png["Alerts table in the Anomaly Explorer"] + If you have more than one {anomaly-job}, you can also obtain _overall bucket_ results, which combine and correlate anomalies from multiple jobs into an overall score. When you view the results for job groups in {kib}, it provides diff --git a/docs/en/stack/ml/anomaly-detection/ml-detect-categories.asciidoc b/docs/en/stack/ml/anomaly-detection/ml-detect-categories.asciidoc new file mode 100644 index 000000000..d1a70e821 --- /dev/null +++ b/docs/en/stack/ml/anomaly-detection/ml-detect-categories.asciidoc @@ -0,0 +1,253 @@ +[[ml-configuring-categories]] += Detecting anomalous categories of data + +Categorization is a {ml} process that tokenizes a text field, clusters similar data together, and classifies it into categories. +It works best on machine-written messages and application output that typically consist of repeated elements. +<> enable you to find anomalous behavior in your categorized data. +Categorization is not natural language processing (NLP). +When you create a categorization {anomaly-job}, the {ml} model learns what volume and pattern is normal for each category over time. +You can then detect anomalies and surface rare events or unusual types of messages by using <> or <> functions. +Categorization works well on finite set of possible messages, for example: + +[source,js] +---------------------------------- +{"@timestamp":1549596476000, +"message":"org.jdbi.v2.exceptions.UnableToExecuteStatementException: com.mysql.jdbc.exceptions.MySQLTimeoutException: Statement cancelled due to timeout or client request [statement:\"SELECT id, customer_id, name, force_disabled, enabled FROM customers\"]", +"type":"logs"} +---------------------------------- +//NOTCONSOLE + + +[discrete] +[[categ-recommendations]] +== Recommendations + +* Categorization is tuned to work best on data like log messages by taking token order into account, including stop words, and not considering synonyms in its analysis. +Use machine-written messages for categorization analysis. +* Complete sentences in human communication or literary text (for example email, wiki pages, prose, or other human-generated content) can be extremely diverse in structure. +Since categorization is tuned for machine data, it gives poor results for human-generated data. +It would create so many categories that they couldn’t be handled effectively. +Avoid using human-generated data for categorization analysis. + +[discrete] +[[creating-categorization-jobs]] +== Creating categorization jobs + +. In {kib}, navigate to *Jobs*. To open *Jobs*, find **{ml-app} > Anomaly Detection** in the main menu, or use the {kibana-ref}/kibana-concepts-analysts.html#_finding_your_apps_and_objects[global search field]. +. Click **Create job**, select the {data-view} you want to analyze. +. Select the **Categorization** wizard from the list. +. Choose a categorization detector - it's the `count` function in this example - and the field you want to categorize - the `message` field in this example. ++ +-- +[role="screenshot"] +image::images/categorization-wizard.png[Creating a categorization job in Kibana] +-- +. Click **Next**. +. Provide a job ID and click **Next**. +. If the validation is successful, click **Next** to review the summary of the job creation. +. Click **Create job**. + +This example job generates categories from the contents of the `message` field and uses the `count` function to determine when certain categories are occurring at anomalous rates. + +[%collapsible] +.API example +==== +[source,console] +---------------------------------- +PUT _ml/anomaly_detectors/it_ops_app_logs +{ + "description" : "IT ops application logs", + "analysis_config" : { + "categorization_field_name": "message",<1> + "bucket_span":"30m", + "detectors" :[{ + "function":"count", + "by_field_name": "mlcategory"<2> + }] + }, + "data_description" : { + "time_field":"@timestamp" + } +} +---------------------------------- +// TEST[skip:needs-licence] +<1> This field is used to derive categories. +<2> The categories are used in a detector by setting `by_field_name`, `over_field_name`, or `partition_field_name` to the keyword `mlcategory`. +If you do not specify this keyword in one of those properties, the API request fails. +==== + + +[discrete] +[[categorization-job-results]] +=== Viewing the job results + +Use the **Anomaly Explorer** in {kib} to view the analysis results: + +[role="screenshot"] +image::images/ml-category-anomalies.png["Categorization results in the Anomaly Explorer"] + +For this type of job, the results contain extra information for each anomaly: the name of the category (for example, `mlcategory 2`) and examples of the messages in that category. +You can use these details to investigate occurrences of unusually high message counts. + + +[discrete] +[[advanced-categorization-options]] +=== Advanced configuration options + +If you use the advanced {anomaly-job} wizard in {kib} or the {ref}/ml-put-job.html[create {anomaly-jobs} API], there are additional configuration options. +For example, the optional `categorization_examples_limit` property specifies the maximum number of examples that are stored in memory and in the results data store for each category. +The default value is `4`. +Note that this setting does not affect the categorization; it just affects the list of visible examples. +If you increase this value, more examples are available, but you must have more storage available. +If you set this value to `0`, no examples are stored. + +Another advanced option is the `categorization_filters` property, which can contain an array of regular expressions. +If a categorization field value matches the regular expression, the portion of the field that is matched is not taken into consideration when defining categories. +The categorization filters are applied in the order they are listed in the job configuration, which enables you to disregard multiple sections of the categorization field value. +In this example, you might create a filter like `[ "\\[statement:.*\\]"]` to remove the SQL statement from the categorization algorithm. + + +[discrete] +[[ml-per-partition-categorization]] +== Per-partition categorization + +If you enable per-partition categorization, categories are determined independently for each partition. +For example, if your data includes messages from multiple types of logs from different applications, you can use a field like the ECS {ecs-ref}/ecs-event.html[`event.dataset` field] as the `partition_field_name` and categorize the messages for each type of log separately. + +If your job has multiple detectors, every detector that uses the `mlcategory` keyword must also define a `partition_field_name`. +You must use the same `partition_field_name` value in all of these detectors. +Otherwise, when you create or update a job and enable per-partition categorization, it fails. + +When per-partition categorization is enabled, you can also take advantage of a `stop_on_warn` configuration option. +If the categorization status for a partition changes to `warn`, it doesn't categorize well and can cause unnecessary resource usage. +When you set `stop_on_warn` to `true`, the job stops analyzing these problematic partitions. +You can thus avoid an ongoing performance cost for partitions that are unsuitable for categorization. + + +[discrete] +[[ml-configuring-analyzer]] +== Customizing the categorization analyzer + +Categorization uses English dictionary words to identify log message categories. +By default, it also uses English tokenization rules. +For this reason, if you use the default categorization analyzer, only English language log messages are supported, as described in the <>. + +If you use the categorization wizard in {kib}, you can see which categorization analyzer it uses and highlighted examples of the tokens that it identifies. +You can also change the tokenization rules by customizing the way the categorization field values are interpreted: + +[role="screenshot"] +image::images/ml-category-analyzer.png["Editing the categorization analyzer in Kibana"] + +The categorization analyzer can refer to a built-in {es} analyzer or a combination of zero or more character filters, a tokenizer, and zero or more token filters. +In this example, adding a {ref}/analysis-pattern-replace-charfilter.html[`pattern_replace` character filter] achieves the same behavior as the `categorization_filters` job configuration option described earlier. +For more details about these properties, refer to the {ref}/ml-put-job.html#ml-put-job-request-body[`categorization_analyzer` API object]. + +If you use the default categorization analyzer in {kib} or omit the `categorization_analyzer` property from the API, the following default values are used: + +[source,console] +-------------------------------------------------- +POST _ml/anomaly_detectors/_validate +{ + "analysis_config" : { + "categorization_analyzer" : { + "char_filter" : [ + "first_line_with_letters" + ], + "tokenizer" : "ml_standard", + "filter" : [ + { "type" : "stop", "stopwords": [ + "Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday", "Sunday", + "Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun", + "January", "February", "March", "April", "May", "June", "July", "August", "September", "October", "November", "December", + "Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec", + "GMT", "UTC" + ] } + ] + }, + "categorization_field_name": "message", + "detectors" :[{ + "function":"count", + "by_field_name": "mlcategory" + }] + }, + "data_description" : { + } +} +-------------------------------------------------- + +If you specify any part of the `categorization_analyzer`, however, any omitted sub-properties are _not_ set to default values. + +The `ml_standard` tokenizer and the day and month stopword filter are almost equivalent to the following analyzer, which is defined using only built-in {es} {ref}/analysis-tokenizers.html[tokenizers] and {ref}/analysis-tokenfilters.html[token filters]: + +[source,console] +---------------------------------- +PUT _ml/anomaly_detectors/it_ops_new_logs +{ + "description" : "IT Ops Application Logs", + "analysis_config" : { + "categorization_field_name": "message", + "bucket_span":"30m", + "detectors" :[{ + "function":"count", + "by_field_name": "mlcategory", + "detector_description": "Unusual message counts" + }], + "categorization_analyzer":{ + "char_filter" : [ + "first_line_with_letters" <1> + ], + "tokenizer": { + "type" : "simple_pattern_split", + "pattern" : "[^-0-9A-Za-z_./]+" <2> + }, + "filter": [ + { "type" : "pattern_replace", "pattern": "^[0-9].*" }, <3> + { "type" : "pattern_replace", "pattern": "^[-0-9A-Fa-f.]+$" }, <4> + { "type" : "pattern_replace", "pattern": "^[^0-9A-Za-z]+" }, <5> + { "type" : "pattern_replace", "pattern": "[^0-9A-Za-z]+$" }, <6> + { "type" : "stop", "stopwords": [ + "", + "Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday", "Sunday", + "Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun", + "January", "February", "March", "April", "May", "June", "July", "August", "September", "October", "November", "December", + "Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec", + "GMT", "UTC" + ] } + ] + } + }, + "analysis_limits":{ + "categorization_examples_limit": 5 + }, + "data_description" : { + "time_field":"time", + "time_format": "epoch_ms" + } +} +---------------------------------- +// TEST[skip:needs-licence] + +<1> Only consider the first line of the message with letters for categorization purposes. +<2> Tokens consist of hyphens, digits, letters, underscores, dots and slashes. +<3> By default, categorization ignores tokens that begin with a digit. +<4> By default, categorization ignores tokens that are hexadecimal numbers. +<5> Underscores, hyphens, and dots are removed from the beginning of tokens. +<6> Underscores, hyphens, and dots are also removed from the end of tokens. + +The key difference between the default `categorization_analyzer` and this example analyzer is that using the `ml_standard` tokenizer is several times faster. +The `ml_standard` tokenizer also tries to preserve URLs, Windows paths and email addresses as single tokens. +Another difference in behavior is that the custom analyzer does not include accented letters in tokens whereas the `ml_standard` tokenizer does. +This could be fixed by using more complex regular expressions. + +If you are categorizing non-English messages in a language where words are separated by spaces, you might get better results if you change the day or month words in the stop token filter to the appropriate words in your language. +If you are categorizing messages in a language where words are not separated by spaces, you must use a different tokenizer as well in order to get sensible categorization results. + +It is important to be aware that analyzing for categorization of machine generated log messages is a little different from tokenizing for search. +Features that work well for search, such as stemming, synonym substitution, and lowercasing are likely to make the results of categorization worse. +However, to drill down from {ml} results to work correctly, the tokens the categorization analyzer produces must be similar to those produced by the search analyzer. +If they are sufficiently similar, when you search for the tokens that the categorization analyzer produces then you find the original document that the categorization field value came from. + + + + + diff --git a/docs/en/stack/ml/anomaly-detection/ml-jobs-from-visuals.asciidoc b/docs/en/stack/ml/anomaly-detection/ml-jobs-from-visuals.asciidoc index 18b164cf9..2a3857027 100644 --- a/docs/en/stack/ml/anomaly-detection/ml-jobs-from-visuals.asciidoc +++ b/docs/en/stack/ml/anomaly-detection/ml-jobs-from-visuals.asciidoc @@ -40,7 +40,7 @@ NOTE: You need to have a compatible visualization on **Dashboard** to create an which is based on the {kib} sample flight data set. Select the `Flight count` visualization from the dashboard. -. Go to **Analytics > Dashboard** and select a dashboard with a compatible +. Go to **Analytics > Dashboard** from the main menu, or use the {kibana-ref}/kibana-concepts-analysts.html#_finding_your_apps_and_objects[global search field]. Select a dashboard with a compatible visualization. . Open the **Options (...) menu** for the panel, then select **More**. . Select **Create {anomaly-job}**. The option is only displayed if the diff --git a/docs/en/stack/ml/anomaly-detection/ml-limitations.asciidoc b/docs/en/stack/ml/anomaly-detection/ml-limitations.asciidoc index d9bd9ddd5..aca674bc9 100644 --- a/docs/en/stack/ml/anomaly-detection/ml-limitations.asciidoc +++ b/docs/en/stack/ml/anomaly-detection/ml-limitations.asciidoc @@ -45,6 +45,14 @@ Linux and MacOS environments, the CPU scheduling priority of native analysis processes is reduced to favor the {es} JVM. This improvement does not apply to Windows environments. +[discrete] +[[ml-license-ccs-limitations]] +=== License limitations for {ml-jobs} with CCS + +You must have an appropriate license to initiate {ml-jobs-cap} on datasets from remote clusters +accessed through Cross-Cluster Search (CCS). Refer to the +[Subscriptions](https://www.elastic.co/subscriptions) page for details on features available with +different subscription levels. [discrete] [[ad-config-limitations]] @@ -61,6 +69,14 @@ You can change this default behavior by setting the `size` parameter. If you send pre-aggregated data to a job for analysis, you must ensure that the `size` is configured correctly. Otherwise, some data might not be analyzed. +[discrete] +=== Scripted metric aggregations are not supported + +Using +{ref}/search-aggregations-metrics-scripted-metric-aggregation.html[scripted metric aggregations] +in {dfeeds} is not supported. Refer to the <> page +to learn more about aggregations in {dfeeds}. + [discrete] === Fields named "by", "count", or "over" cannot be used to split data @@ -94,18 +110,6 @@ Analyzing large arrays results in long strings which may require more system resources. Consider using a query in the {dfeed} that filters on the relevant items of the array. - -[discrete] -[[ml-frozen-limitations]] -=== Frozen indices are not supported - -{ref}/frozen-indices.html[Frozen indices] cannot be used in {anomaly-jobs} or -{dfeeds}. This limitation applies irrespective of whether you create the jobs in -{kib} or by using APIs. This limitation exists because it's currently not -possible to specify the `ignore_throttled` query parameter for search requests -in {dfeeds} or jobs. See -{ref}/searching_a_frozen_index.html[Searching a frozen index]. - [discrete] [[ml-frozen-tier-limitations]] === {anomaly-jobs-cap} on frozen tier data cannot be created in {kib} @@ -254,24 +258,53 @@ that it does not block pop-up windows or create an exception for your {kib} URL. //See x-pack-elasticsearch/#844 and x-pack-kibana/#1461 In {kib}, **Anomaly Explorer** and **Single Metric Viewer** charts are not -displayed for anomalies that were due to categorization (if model plot is not -enabled), `time_of_day` functions, `time_of_week` functions, or `lat_long` -geographic functions. - -If model plot is not enabled, the charts are not displayed for detectors that -use script fields either (except for scripts that define metric fields). In that -case, the original source data cannot be easily searched because it has been -transformed by the script. - -If your <> and -model plot is not enabled for the {anomaly-job}, neither the **Anomaly -Explorer** nor the **Single Metric Viewer** can plot and display an anomaly -chart for the job. In these cases, the charts are not visible and an explanatory -message is shown. +displayed: + +* for anomalies that were due to categorization (if model plot is not +enabled), +* if the {dfeed} uses scripted fields and model plot is not enabled (except +for scripts that define metric fields), +* if the {dfeed} uses +{ref}/search-aggregations-bucket-composite-aggregation.html[composite aggregations] +that have composite sources other than `terms` and `date_histogram`, +* if your <> and +model plot is not enabled, +* `freq_rare` functions, +* `info_content`, `high_info_content`, `low_info_content` functions, +* `lat_long` geographic functions +* `time_of_day`, `time_of_week` functions, +* `varp`, `high_varp`, `low_varp` functions. + +Refer to the table below for a more detailed view of supported detector +functions. The charts can also look odd in circumstances where there is very little data to plot. For example, if there is only one data point, it is represented as a single dot. If there are only two data points, they are joined by a line. +The following table shows which detector functions are supported in the Single +Metric Viewer. + +.Detector function support in the Anomaly Explorer and the Single Metric Viewer +[options="header"] +|=================================================================================================================================================================================================== +| Detector functions | Function description | Supported +| count, high_count, low_count, non_zero_count, low_non_zero_count | <> | yes +| count, high_count, low_count, non_zero_count, low_non_zero_count with summary_count_field_name that is not doc_count (model plot not enabled)| <> | yes +| non_zero_count with summary_count_field that is not doc_count using cardinality aggregation in datafeed config (model plot not enabled) | <> | yes +| distinct_count, high_distinct_count, low_distinct_count | <> | yes +| mean, high_mean, low_mean | <> | yes +| min | <> | yes +| max | <> | yes +| metric | <> | yes +| median, high_median, low_median | <> | yes +| sum, high_sum ,low_sum, non_null_sum, high_non_null_sum, low_non_null_sum | <> | yes +| varp, high_varp, low_varp | <> | yes (only if model plot is enabled) +| lat_long | <> | no (but map is displayed in the Anomaly Explorer) +| info_content, high_info_content, low_info_content | <> | yes (only if model plot is enabled) +| rare | <> | yes +| freq_rare | <> | no +| time_of_day, time_of_week | <> | no +|=================================================================================================================================================================================================== [discrete] @@ -344,4 +377,4 @@ Rollup indices and {data-sources} with rolled up indices cannot be used in an index that uses the experimental {ref}/xpack-rollup.html[{rollup-features}], the {anomaly-job} creation wizards fail. If you use APIs to create {anomaly-jobs} that use {rollup-features}, the job results might not display -properly in the *Single Metric Viewer* or *Anomaly Explorer* in {kib}. \ No newline at end of file +properly in the *Single Metric Viewer* or *Anomaly Explorer* in {kib}. diff --git a/docs/en/stack/ml/anomaly-detection/ml-population-analysis.asciidoc b/docs/en/stack/ml/anomaly-detection/ml-population-analysis.asciidoc new file mode 100644 index 000000000..3ccd95e75 --- /dev/null +++ b/docs/en/stack/ml/anomaly-detection/ml-population-analysis.asciidoc @@ -0,0 +1,96 @@ +[[ml-configuring-populations]] += Performing population analysis + +Population analysis is a method of detecting anomalies by comparing the behavior of entities or events within a specified population. +In this approach, {ml} analytics create a profile of what is considered "typical" behavior for users, machines, or other entities over a specified time period. +An entity is considered as anomalous when its behavior deviates from that of the population, indicating abnormal activity compared to the rest of the population. + +This type of analysis is most effective when the behavior within a group is generally homogeneous, allowing for the identification of unusual patterns. +However, it is less useful when members of the population show vastly different behaviors. +In such cases, you can segment your data into groups with similar behaviors and run separate jobs for each. +This can be done by using a query filter in the datafeed or by applying the `partition_field_name` to split the analysis across different groups. + +Population analysis is resource-efficient and scales well, enabling the analysis of populations consisting of hundreds of thousands or even millions of entities with a lower resource footprint than analyzing each series individually. + + + +[discrete] +[[population-recommendations]] +== Recommendations + +* Use population analysis when the behavior within a group is mostly homogeneous, as it helps identify anomalous patterns effectively. +* Leverage population analysis when dealing with large-scale datasets. +* Avoid using population analysis when members of the population exhibit vastly different behaviors, as it may not be effective. + + +[discrete] +[[creating-population-jobs]] +== Creating population jobs + +. In {kib}, navigate to *Jobs*. To open *Jobs*, find **{ml-app} > Anomaly Detection** in the main menu, or use the {kibana-ref}/kibana-concepts-analysts.html#_finding_your_apps_and_objects[global search field]. +. Click **Create job**, select the {data-source} you want to analyze. +. Select the **Population** wizard from the list. +. Choose a population field - it's the `clientip` field in this example - and the metric you want to use for the analysis - `Mean(bytes)` in this example. ++ +-- +[role="screenshot"] +image::images/ml-population-wizard.png[Creating a population job in Kibana] +-- +. Click **Next**. +. Provide a job ID and click **Next**. +. If the validation is successful, click **Next** to review the summary of the job creation. +. Click **Create job**. + +[%collapsible] +.API example +==== +To specify the population, use the `over_field_name` property. For example: + +[source,console] +---------------------------------- +PUT _ml/anomaly_detectors/population +{ + "description" : "Population analysis", + "analysis_config" : { + "bucket_span":"15m", + "influencers": [ + "clientip" + ], + "detectors": [ + { + "function": "mean", + "field_name": "bytes", + "over_field_name": "clientip" <1> + } + ] + }, + "data_description" : { + "time_field":"timestamp", + "time_format": "epoch_ms" + } +} +---------------------------------- +// TEST[skip:needs-licence] + +<1> This `over_field_name` property indicates that the metrics for each client (as identified by their IP address) are analyzed relative to other clients in each bucket. +==== + +[discrete] +[[population-job-results]] +=== Viewing the job results + +Use the **Anomaly Explorer** in {kib} to view the analysis results: + +[role="screenshot"] +image::images/ml-population-anomalies.png["Population results in the Anomaly Explorer"] + +The results are often quite sparse. +There might be just a few data points for the selected time period. +Population analysis is particularly useful when you have many entities and the data for specific entitles is sporadic or sparse. +If you click on a section in the timeline or swim lanes, you can see more details about the anomalies: + +[role="screenshot"] +image::images/ml-population-anomaly.png["Anomaly details for a specific user"] + +In this example, the client IP address `167.145.234.154` received a high volume of bytes on the date and time shown. +This event is anomalous because the mean is four times higher than the expected behavior of the population. \ No newline at end of file diff --git a/docs/en/stack/ml/anomaly-detection/ml-revert-model-snapshot.asciidoc b/docs/en/stack/ml/anomaly-detection/ml-revert-model-snapshot.asciidoc index 08f494edc..478465d57 100644 --- a/docs/en/stack/ml/anomaly-detection/ml-revert-model-snapshot.asciidoc +++ b/docs/en/stack/ml/anomaly-detection/ml-revert-model-snapshot.asciidoc @@ -7,7 +7,7 @@ resilience. It makes it possible to reset the model to a previous state in case of a system failure or if the model changed significantly due to a one-off event. -. In {kib}, navigate to **{ml-app} > Anomaly Detection > Jobs**. +. In {kib}, navigate to *Jobs*. To open *Jobs*, find **{ml-app} > Anomaly Detection** in the main menu, or use the {kibana-ref}/kibana-concepts-analysts.html#_finding_your_apps_and_objects[global search field]. . Locate the {anomaly-job} whose model you want to revert in the job table. . Open the job details and navigate to the **Model Snapshots** tab. + diff --git a/docs/en/stack/ml/anomaly-detection/ootb-ml-jobs-apache.asciidoc b/docs/en/stack/ml/anomaly-detection/ootb-ml-jobs-apache.asciidoc index a63d5694c..91dd0a51e 100644 --- a/docs/en/stack/ml/anomaly-detection/ootb-ml-jobs-apache.asciidoc +++ b/docs/en/stack/ml/anomaly-detection/ootb-ml-jobs-apache.asciidoc @@ -1,4 +1,4 @@ -["appendix",role="exclude",id="ootb-ml-jobs-apache"] +[[ootb-ml-jobs-apache]] = Apache {anomaly-detect} configurations // tag::apache-jobs[] diff --git a/docs/en/stack/ml/anomaly-detection/ootb-ml-jobs-apm.asciidoc b/docs/en/stack/ml/anomaly-detection/ootb-ml-jobs-apm.asciidoc index cd7cdd9d1..c685b9db0 100644 --- a/docs/en/stack/ml/anomaly-detection/ootb-ml-jobs-apm.asciidoc +++ b/docs/en/stack/ml/anomaly-detection/ootb-ml-jobs-apm.asciidoc @@ -1,4 +1,4 @@ -["appendix",role="exclude",id="ootb-ml-jobs-apm"] +[[ootb-ml-jobs-apm]] = APM {anomaly-detect} configurations This {anomaly-job} appears in the {apm-app} and the {ml-app} app when you have diff --git a/docs/en/stack/ml/anomaly-detection/ootb-ml-jobs-auditbeat.asciidoc b/docs/en/stack/ml/anomaly-detection/ootb-ml-jobs-auditbeat.asciidoc index 537c687bf..027d520d7 100644 --- a/docs/en/stack/ml/anomaly-detection/ootb-ml-jobs-auditbeat.asciidoc +++ b/docs/en/stack/ml/anomaly-detection/ootb-ml-jobs-auditbeat.asciidoc @@ -1,4 +1,4 @@ -["appendix",role="exclude",id="ootb-ml-jobs-auditbeat"] +[[ootb-ml-jobs-auditbeat]] = {auditbeat} {anomaly-detect} configurations // tag::auditbeat-jobs[] diff --git a/docs/en/stack/ml/anomaly-detection/ootb-ml-jobs-logs-ui.asciidoc b/docs/en/stack/ml/anomaly-detection/ootb-ml-jobs-logs-ui.asciidoc index c15e3f815..2faa037a1 100644 --- a/docs/en/stack/ml/anomaly-detection/ootb-ml-jobs-logs-ui.asciidoc +++ b/docs/en/stack/ml/anomaly-detection/ootb-ml-jobs-logs-ui.asciidoc @@ -1,4 +1,4 @@ -["appendix",role="exclude",id="ootb-ml-jobs-logs-ui"] +[[ootb-ml-jobs-logs-ui]] = Logs {anomaly-detect} configurations These {anomaly-jobs} appear by default in the diff --git a/docs/en/stack/ml/anomaly-detection/ootb-ml-jobs-metricbeat.asciidoc b/docs/en/stack/ml/anomaly-detection/ootb-ml-jobs-metricbeat.asciidoc index e2a8ebee7..1b163e687 100644 --- a/docs/en/stack/ml/anomaly-detection/ootb-ml-jobs-metricbeat.asciidoc +++ b/docs/en/stack/ml/anomaly-detection/ootb-ml-jobs-metricbeat.asciidoc @@ -1,4 +1,4 @@ -["appendix",role="exclude",id="ootb-ml-jobs-metricbeat"] +[[ootb-ml-jobs-metricbeat]] = {metricbeat} {anomaly-detect} configurations These {anomaly-job} wizards appear in {kib} if you use the diff --git a/docs/en/stack/ml/anomaly-detection/ootb-ml-jobs-metrics-ui.asciidoc b/docs/en/stack/ml/anomaly-detection/ootb-ml-jobs-metrics-ui.asciidoc index fda928cb3..6974b7b6d 100644 --- a/docs/en/stack/ml/anomaly-detection/ootb-ml-jobs-metrics-ui.asciidoc +++ b/docs/en/stack/ml/anomaly-detection/ootb-ml-jobs-metrics-ui.asciidoc @@ -1,10 +1,8 @@ -["appendix",role="exclude",id="ootb-ml-jobs-metrics-ui"] +[[ootb-ml-jobs-metrics-ui]] = Metrics {anomaly-detect} configurations -These {anomaly-jobs} can be created in the -{observability-guide}/analyze-metrics.html[{metrics-app}] in {kib}. For more -information about their usage, refer to -{observability-guide}/inspect-metric-anomalies.html[Inspect metric anomalies]. +These {anomaly-jobs} can be created in the {observability-guide}/analyze-metrics.html[{infrastructure-app}] in {kib}. +For more information about their usage, refer to {observability-guide}/inspect-metric-anomalies.html[Inspect metric anomalies]. // tag::metrics-jobs[] [discrete] diff --git a/docs/en/stack/ml/anomaly-detection/ootb-ml-jobs-nginx.asciidoc b/docs/en/stack/ml/anomaly-detection/ootb-ml-jobs-nginx.asciidoc index 938f33913..c6583ce61 100644 --- a/docs/en/stack/ml/anomaly-detection/ootb-ml-jobs-nginx.asciidoc +++ b/docs/en/stack/ml/anomaly-detection/ootb-ml-jobs-nginx.asciidoc @@ -1,4 +1,4 @@ -["appendix",role="exclude",id="ootb-ml-jobs-nginx"] +[[ootb-ml-jobs-nginx]] = Nginx {anomaly-detect} configurations // tag::nginx-jobs[] diff --git a/docs/en/stack/ml/anomaly-detection/ootb-ml-jobs-siem.asciidoc b/docs/en/stack/ml/anomaly-detection/ootb-ml-jobs-siem.asciidoc index 5b972df0d..797f50312 100644 --- a/docs/en/stack/ml/anomaly-detection/ootb-ml-jobs-siem.asciidoc +++ b/docs/en/stack/ml/anomaly-detection/ootb-ml-jobs-siem.asciidoc @@ -1,4 +1,4 @@ -["appendix",role="exclude",id="ootb-ml-jobs-siem"] +[[ootb-ml-jobs-siem]] = Security {anomaly-detect} configurations // tag::siem-jobs[] @@ -116,6 +116,39 @@ for data that matches the query. |=== // end::security-cloudtrail-jobs[] +[discrete] +[[security-host-jobs]] +== Security: Host + +Anomaly detection jobs for host-based threat hunting and detection. + +In the {ml-app} app, these configurations are available only when data exists +that matches the query specified in the +https://github.com/elastic/kibana/blob/{branch}/x-pack/platform/plugins/shared/ml/server/models/data_recognizer/modules/security_host/manifest.json[manifest file]. +In the {security-app}, it looks in the {data-source} specified in the +{kibana-ref}/advanced-options.html#securitysolution-defaultindex[`securitySolution:defaultIndex` advanced setting] +for data that matches the query. + +To access the host traffic anomalies dashboard in Kibana, go to: `Security -> Dashboards -> Host Traffic Anomalies`. + +// tag::security-host-jobs[] + +|=== +|Name |Description |Job |Datafeed + +|high_count_events_for_a_host_name +|Looks for a sudden spike in host based traffic. This can be due to a range of security issues, such as a compromised system, DDoS attacks, malware infections, privilege escalation, or data exfiltration. +|https://github.com/elastic/kibana/blob/{branch}/x-pack/platform/plugins/shared/ml/server/models/data_recognizer/modules/security_host/ml/high_count_events_for_a_host_name.json[image:images/link.svg[A link icon]] +|https://github.com/elastic/kibana/blob/{branch}/x-pack/platform/plugins/shared/ml/server/models/data_recognizer/modules/security_host/ml/datafeed_high_count_events_for_a_host_name.json[image:images/link.svg[A link icon]] + +|low_count_events_for_a_host_name +|Looks for a sudden drop in host based traffic. This can be due to a range of security issues, such as a compromised system, a failed service, or a network misconfiguration. +|https://github.com/elastic/kibana/blob/{branch}/x-pack/platform/plugins/shared/ml/server/models/data_recognizer/modules/security_host/ml/low_count_events_for_a_host_name.json[image:images/link.svg[A link icon]] +|https://github.com/elastic/kibana/blob/{branch}/x-pack/platform/plugins/shared/ml/server/models/data_recognizer/modules/security_host/ml/datafeed_low_count_events_for_a_host_name.json[image:images/link.svg[A link icon]] + +|=== +// end::security-host-jobs[] + [discrete] [[security-linux-jobs]] == Security: Linux @@ -389,9 +422,13 @@ they are listed for each job. [[security-integrations-jobs]] == Security: Elastic Integrations -https://docs.elastic.co/integrations[Elastic Integrations] are a streamlined way to add Elastic assets to your environment, such as data ingestion, {transforms}, and in this case, {ml} capabilities for Security. +https://docs.elastic.co/integrations[Elastic Integrations] are a streamlined way +to add Elastic assets to your environment, such as data ingestion, {transforms}, +and in this case, {ml} capabilities for Security. -The following Integrations use {ml} to analyze patterns of user and entity behavior, and help detect and alert when there is related suspicious activity in your environment. +The following Integrations use {ml} to analyze patterns of user and entity +behavior, and help detect and alert when there is related suspicious activity in +your environment. * https://docs.elastic.co/integrations/ded[Data Exfiltration Detection] * https://docs.elastic.co/integrations/dga[Domain Generation Algorithm Detection] @@ -402,63 +439,59 @@ The following Integrations use {ml} to analyze patterns of user and entity behav *Domain Generation Algorithm (DGA) Detection* -{ml-cap} solution package to detect domain generation algorithm (DGA) activity in your network data. Refer to the {subscriptions}[subscription page] to learn more about the required subscription. +{ml-cap} solution package to detect domain generation algorithm (DGA) activity +in your network data. Refer to the {subscriptions}[subscription page] to learn +more about the required subscription. To download, refer to the https://docs.elastic.co/integrations/dga[documentation]. |=== -|Name |Description |Job |Datafeed +|Name |Description |dga_high_sum_probability |Detect domain generation algorithm (DGA) activity in your network data. -|https://github.com/elastic/integrations/blob/{branch}/packages/dga/kibana/ml_module/dga-ml.json#L23[image:images/link.svg[A link icon]] -|https://github.com/elastic/integrations/blob/{branch}/packages/dga/kibana/ml_module/dga-ml.json#L58[image:images/link.svg[A link icon]] |=== +The job configurations and datafeeds can be found +https://github.com/elastic/integrations/blob/main/packages/dga/kibana/ml_module/dga-ml.json[here]. + // LotL *Living off the Land Attack (LotL) Detection* -{ml-cap} solution package to detect Living off the Land (LotL) attacks in your environment. Refer to the {subscriptions}[subscription page] to learn more about the required subscription. (Also known as ProblemChild). +{ml-cap} solution package to detect Living off the Land (LotL) attacks in your +environment. Refer to the {subscriptions}[subscription page] to learn more about +the required subscription. (Also known as ProblemChild). To download, refer to the https://docs.elastic.co/integrations/problemchild[documentation]. |=== -|Name |Description |Job |Datafeed +|Name |Description |problem_child_rare_process_by_host -|Looks for a process that has been classified as malicious on a host that does not commonly manifest malicious process activity (experimental). -|https://github.com/elastic/integrations/blob/{branch}/packages/problemchild/kibana/ml_module/problemchild-ml.json#L29[image:images/link.svg[A link icon]] -|https://github.com/elastic/integrations/blob/{branch}/packages/problemchild/kibana/ml_module/problemchild-ml.json#262[image:images/link.svg[A link icon]] +|Looks for a process that has been classified as malicious on a host that does not commonly manifest malicious process activity. |problem_child_high_sum_by_host -|Looks for a set of one or more malicious child processes on a single host (experimental). -|https://github.com/elastic/integrations/blob/{branch}/packages/problemchild/kibana/ml_module/problemchild-ml.json#L64[image:images/link.svg[A link icon]] -|https://github.com/elastic/integrations/blob/{branch}/packages/problemchild/kibana/ml_module/problemchild-ml.json#262[image:images/link.svg[A link icon]] +|Looks for a set of one or more malicious child processes on a single host. |problem_child_rare_process_by_user -|Looks for a process that has been classified as malicious where the user context is unusual and does not commonly manifest malicious process activity (experimental). -|https://github.com/elastic/integrations/blob/{branch}/packages/problemchild/kibana/ml_module/problemchild-ml.json#L106[image:images/link.svg[A link icon]] -|https://github.com/elastic/integrations/blob/{branch}/packages/problemchild/kibana/ml_module/problemchild-ml.json#262[image:images/link.svg[A link icon]] +|Looks for a process that has been classified as malicious where the user context is unusual and does not commonly manifest malicious process activity. |problem_child_rare_process_by_parent -|Looks for rare malicious child processes spawned by a parent process (experimental). -|https://github.com/elastic/integrations/blob/{branch}/packages/problemchild/kibana/ml_module/problemchild-ml.json#L141[image:images/link.svg[A link icon]] -|https://github.com/elastic/integrations/blob/{branch}/packages/problemchild/kibana/ml_module/problemchild-ml.json#262[image:images/link.svg[A link icon]] +|Looks for rare malicious child processes spawned by a parent process. |problem_child_high_sum_by_user -|Looks for a set of one or more malicious processes, started by the same user (experimental). -|https://github.com/elastic/integrations/blob/{branch}/packages/problemchild/kibana/ml_module/problemchild-ml.json#L177[image:images/link.svg[A link icon]] -|https://github.com/elastic/integrations/blob/{branch}/packages/problemchild/kibana/ml_module/problemchild-ml.json#262[image:images/link.svg[A link icon]] +|Looks for a set of one or more malicious processes, started by the same user. |problem_child_high_sum_by_parent -|Looks for a set of one or more malicious child processes spawned by the same parent process (experimental). -|https://github.com/elastic/integrations/blob/{branch}/packages/problemchild/kibana/ml_module/problemchild-ml.json#L219[image:images/link.svg[A link icon]] -|https://github.com/elastic/integrations/blob/{branch}/packages/problemchild/kibana/ml_module/problemchild-ml.json#262[image:images/link.svg[A link icon]] +|Looks for a set of one or more malicious child processes spawned by the same parent process. |=== +The job configurations and datafeeds can be found +https://github.com/elastic/integrations/blob/main/packages/problemchild/kibana/ml_module/problemchild-ml.json[here]. + // ded *Data Exfiltration Detection (DED)* @@ -468,111 +501,84 @@ To download, refer to the https://docs.elastic.co/integrations/problemchild[docu To download, refer to the https://docs.elastic.co/integrations/ded[documentation]. |=== -|Name |Description |Job |Datafeed +|Name |Description -|high-sent-bytes-destination-geo-country_iso_code +|ded_high_sent_bytes_destination_geo_country_iso_code |Detects data exfiltration to an unusual geo-location (by country iso code). -|https://github.com/elastic/integrations/blob/{branch}/packages/ded/kibana/ml_module/ded-ml.json#L44[image:images/link.svg[A link icon]] -|https://github.com/elastic/integrations/blob/{branch}/packages/ded/kibana/ml_module/ded-ml.json#L304[image:images/link.svg[A link icon]] -|high-sent-bytes-destination-ip +|ded_high_sent_bytes_destination_ip |Detects data exfiltration to an unusual geo-location (by IP address). -|https://github.com/elastic/integrations/blob/{branch}/packages/ded/kibana/ml_module/ded-ml.json#L83[image:images/link.svg[A link icon]] -|https://github.com/elastic/integrations/blob/{branch}/packages/ded/kibana/ml_module/ded-ml.json#L304[image:images/link.svg[A link icon]] -|high-sent-bytes-destination-port +|ded_high_sent_bytes_destination_port |Detects data exfiltration to an unusual destination port. -|https://github.com/elastic/integrations/blob/{branch}/packages/ded/kibana/ml_module/ded-ml.json#L119[image:images/link.svg[A link icon]] -|https://github.com/elastic/integrations/blob/{branch}/packages/ded/kibana/ml_module/ded-ml.json#L304[image:images/link.svg[A link icon]] -|high-sent-bytes-destination-region_name +|ded_high_sent_bytes_destination_region_name |Detects data exfiltration to an unusual geo-location (by region name). -|https://github.com/elastic/integrations/blob/{branch}/packages/ded/kibana/ml_module/ded-ml.json#L156[image:images/link.svg[A link icon]] -|https://github.com/elastic/integrations/blob/{branch}/packages/ded/kibana/ml_module/ded-ml.json#L304[image:images/link.svg[A link icon]] -|high-bytes-written-to-external-device +|ded_high_bytes_written_to_external_device |Detects data exfiltration activity by identifying high bytes written to an external device. -|https://github.com/elastic/integrations/blob/{branch}/packages/ded/kibana/ml_module/ded-ml.json#L194[image:images/link.svg[A link icon]] -|https://github.com/elastic/integrations/blob/{branch}/packages/ded/kibana/ml_module/ded-ml.json#L304[image:images/link.svg[A link icon]] -|rare-process-writing-to-external-device +|ded_rare_process_writing_to_external_device |Detects data exfiltration activity by identifying a file write started by a rare process to an external device. -|https://github.com/elastic/integrations/blob/{branch}/packages/ded/kibana/ml_module/ded-ml.json#L231[image:images/link.svg[A link icon]] -|https://github.com/elastic/integrations/blob/{branch}/packages/ded/kibana/ml_module/ded-ml.json#L304[image:images/link.svg[A link icon]] -|high-bytes-written-to-external-device-airdrop +|ded_high_bytes_written_to_external_device_airdrop |Detects data exfiltration activity by identifying high bytes written to an external device via Airdrop. -|https://github.com/elastic/integrations/blob/{branch}/packages/ded/kibana/ml_module/ded-ml.json#L268[image:images/link.svg[A link icon]] -|https://github.com/elastic/integrations/blob/{branch}/packages/ded/kibana/ml_module/ded-ml.json#L304[image:images/link.svg[A link icon]] |=== +The job configurations and datafeeds can be found +https://github.com/elastic/integrations/blob/main/packages/ded/kibana/ml_module/ded-ml.json[here]. + // lmd *Lateral Movement Detection (LMD)* -{ml-cap} package to detect lateral movement based on file transfer activity and Windows RDP events. Refer to the {subscriptions}[subscription page] to learn more about the required subscription. +{ml-cap} package to detect lateral movement based on file transfer activity and +Windows RDP events. Refer to the {subscriptions}[subscription page] to learn +more about the required subscription. To download, refer to the https://docs.elastic.co/integrations/lmd[documentation]. |=== -|Name |Description |Job |Datafeed +|Name |Description -|high-count-remote-file-transfer +|lmd_high_count_remote_file_transfer |Detects unusually high file transfers to a remote host in the network. -|https://github.com/elastic/integrations/blob/{branch}/packages/lmd/kibana/ml_module/lmd-ml.json#L24[image:images/link.svg[A link icon]] -|https://github.com/elastic/integrations/blob/{branch}/packages/lmd/kibana/ml_module/lmd-ml.json#L436[image:images/link.svg[A link icon]] -|high-file-size-remote-file-transfer +|lmd_high_file_size_remote_file_transfer |Detects unusually high size of files shared with a remote host in the network. -|https://github.com/elastic/integrations/blob/main/packages/lmd/kibana/ml_module/lmd-ml.json#L58[image:images/link.svg[A link icon]] -|https://github.com/elastic/integrations/blob/{branch}/packages/lmd/kibana/ml_module/lmd-ml.json#L436[image:images/link.svg[A link icon]] -|rare-file-extension-remote-transfer +|lmd_rare_file_extension_remote_transfer |Detects data exfiltration to an unusual destination port. -|https://github.com/elastic/integrations/blob/main/packages/lmd/kibana/ml_module/lmd-ml.json#L92[image:images/link.svg[A link icon]] -|https://github.com/elastic/integrations/blob/{branch}/packages/lmd/kibana/ml_module/lmd-ml.json#L436[image:images/link.svg[A link icon]] -|rare-file-path-remote-transfer +|lmd_rare_file_path_remote_transfer |Detects unusual folders and directories on which a file is transferred. -|https://github.com/elastic/integrations/blob/main/packages/lmd/kibana/ml_module/lmd-ml.json#L126[image:images/link.svg[A link icon]] -|https://github.com/elastic/integrations/blob/{branch}/packages/lmd/kibana/ml_module/lmd-ml.json#L436[image:images/link.svg[A link icon]] -|high-mean-rdp-session-duration +|lmd_high_mean_rdp_session_duration |Detects unusually high mean of RDP session duration. -|https://github.com/elastic/integrations/blob/main/packages/lmd/kibana/ml_module/lmd-ml.json#L160[image:images/link.svg[A link icon]] -|https://github.com/elastic/integrations/blob/{branch}/packages/lmd/kibana/ml_module/lmd-ml.json#L436[image:images/link.svg[A link icon]] -|high-var-rdp-session-duration +|lmd_high_var_rdp_session_duration |Detects unusually high variance in RDP session duration. -|https://github.com/elastic/integrations/blob/main/packages/lmd/kibana/ml_module/lmd-ml.json#L202[image:images/link.svg[A link icon]] -|https://github.com/elastic/integrations/blob/{branch}/packages/lmd/kibana/ml_module/lmd-ml.json#L436[image:images/link.svg[A link icon]] -|high-sum-rdp-number-of-processes +|lmd_high_sum_rdp_number_of_processes |Detects unusually high number of processes started in a single RDP session. -|https://github.com/elastic/integrations/blob/main/packages/lmd/kibana/ml_module/lmd-ml.json#L244[image:images/link.svg[A link icon]] -|https://github.com/elastic/integrations/blob/{branch}/packages/lmd/kibana/ml_module/lmd-ml.json#L436[image:images/link.svg[A link icon]] -|unusual-time-weekday-rdp-session-start +|lmd_unusual_time_weekday_rdp_session_start |Detects an RDP session started at an usual time or weekday. -|https://github.com/elastic/integrations/blob/main/packages/lmd/kibana/ml_module/lmd-ml.json#L286[image:images/link.svg[A link icon]] -|https://github.com/elastic/integrations/blob/{branch}/packages/lmd/kibana/ml_module/lmd-ml.json#L436[image:images/link.svg[A link icon]] -|high-rdp-distinct-count-source-ip-for-destination +|lmd_high_rdp_distinct_count_source_ip_for_destination |Detects a high count of source IPs making an RDP connection with a single destination IP. -|https://github.com/elastic/integrations/blob/main/packages/lmd/kibana/ml_module/lmd-ml.json#L326[image:images/link.svg[A link icon]] -|https://github.com/elastic/integrations/blob/{branch}/packages/lmd/kibana/ml_module/lmd-ml.json#L436[image:images/link.svg[A link icon]] -|high-rdp-distinct-count-destination-ip-for-source +|lmd_high_rdp_distinct_count_destination_ip_for_source |Detects a high count of destination IPs establishing an RDP connection with a single source IP. -|https://github.com/elastic/integrations/blob/main/packages/lmd/kibana/ml_module/lmd-ml.json#L360[image:images/link.svg[A link icon]] -|https://github.com/elastic/integrations/blob/{branch}/packages/lmd/kibana/ml_module/lmd-ml.json#L436[image:images/link.svg[A link icon]] -|high-mean-rdp-process-args +|lmd_high_mean_rdp_process_args |Detects unusually high number of process arguments in an RDP session. -|https://github.com/elastic/integrations/blob/main/packages/lmd/kibana/ml_module/lmd-ml.json#L394[image:images/link.svg[A link icon]] -|https://github.com/elastic/integrations/blob/{branch}/packages/lmd/kibana/ml_module/lmd-ml.json#L436[image:images/link.svg[A link icon]] |=== + +The job configurations and datafeeds can be found +https://github.com/elastic/integrations/blob/main/packages/lmd/kibana/ml_module/lmd-ml.json[here]. + // end::security-windows-jobs[] -// end::siem-jobs[] \ No newline at end of file +// end::siem-jobs[] diff --git a/docs/en/stack/ml/anomaly-detection/ootb-ml-jobs-uptime.asciidoc b/docs/en/stack/ml/anomaly-detection/ootb-ml-jobs-uptime.asciidoc index dec44e182..d1215783f 100644 --- a/docs/en/stack/ml/anomaly-detection/ootb-ml-jobs-uptime.asciidoc +++ b/docs/en/stack/ml/anomaly-detection/ootb-ml-jobs-uptime.asciidoc @@ -1,4 +1,4 @@ -["appendix",role="exclude",id="ootb-ml-jobs-uptime"] +[[ootb-ml-jobs-uptime]] = Uptime {anomaly-detect} configurations If you have appropriate {heartbeat} data in {es}, you can enable this diff --git a/docs/en/stack/ml/anomaly-detection/ootb-ml-jobs.asciidoc b/docs/en/stack/ml/anomaly-detection/ootb-ml-jobs.asciidoc index 7aa98eb05..eb8c8136f 100644 --- a/docs/en/stack/ml/anomaly-detection/ootb-ml-jobs.asciidoc +++ b/docs/en/stack/ml/anomaly-detection/ootb-ml-jobs.asciidoc @@ -1,4 +1,3 @@ -[role="xpack"] [[ootb-ml-jobs]] = Supplied {anomaly-detect} configurations ++++ diff --git a/docs/en/stack/ml/df-analytics/ml-dfa-shared.asciidoc b/docs/en/stack/ml/df-analytics/ml-dfa-shared.asciidoc index 06486283d..c8d42ec4b 100644 --- a/docs/en/stack/ml/df-analytics/ml-dfa-shared.asciidoc +++ b/docs/en/stack/ml/df-analytics/ml-dfa-shared.asciidoc @@ -1,6 +1,6 @@ tag::dfa-deploy-model[] . To deploy {dfanalytics} model in a pipeline, navigate to **Machine Learning** > -**Model Management** > **Trained models** in {kib}. +**Model Management** > **Trained models** in the main menu, or use the {kibana-ref}/kibana-concepts-analysts.html#_finding_your_apps_and_objects[global search field] in {kib}. . Find the model you want to deploy in the list and click **Deploy model** in the **Actions** menu. diff --git a/docs/en/stack/ml/get-started/ml-getting-started.asciidoc b/docs/en/stack/ml/get-started/ml-getting-started.asciidoc index b08eb21f9..a0d6d619d 100644 --- a/docs/en/stack/ml/get-started/ml-getting-started.asciidoc +++ b/docs/en/stack/ml/get-started/ml-getting-started.asciidoc @@ -47,7 +47,7 @@ user that has authority to manage {anomaly-jobs}. See <>. . {kibana-ref}/get-started.html#gs-get-data-into-kibana[Add the sample data sets that ship with {kib}]. -.. From the {kib} home page, click *Try samle data*, then open the +.. From the {kib} home page, click *Try sample data*, then open the *Other sample data sets* section. .. Pick a data set. In this tutorial, you'll use the *Sample web logs*. While diff --git a/docs/en/stack/ml/get-started/ml-gs-results.asciidoc b/docs/en/stack/ml/get-started/ml-gs-results.asciidoc index 489053679..f5697d261 100644 --- a/docs/en/stack/ml/get-started/ml-gs-results.asciidoc +++ b/docs/en/stack/ml/get-started/ml-gs-results.asciidoc @@ -34,7 +34,7 @@ request rate on your web site drops significantly. Let's start by looking at this simple job in the **Single Metric Viewer**: -. Select the *Anomaly Detection* tab in *{ml-app}* to see the list of your +. Select the *Jobs* tab in *{ml-app}* to see the list of your {anomaly-jobs}. . Click the chart icon in the *Actions* column for your `low_request_rate` job @@ -151,7 +151,7 @@ look at both high and low request rates partitioned by response code. Let's start by looking at the `response_code_rates` job in the **Anomaly Explorer**: -. Select the *Anomaly Detection* tab in *{ml-app}* to see the list of your +. Select the *Jobs* tab in *{ml-app}* to see the list of your {anomaly-jobs}. . Open the `response_code_rates` job in the Anomaly Explorer to view its results diff --git a/docs/en/stack/ml/get-started/ml-gs-visualizer.asciidoc b/docs/en/stack/ml/get-started/ml-gs-visualizer.asciidoc index a1dcc9da6..8335722b9 100644 --- a/docs/en/stack/ml/get-started/ml-gs-visualizer.asciidoc +++ b/docs/en/stack/ml/get-started/ml-gs-visualizer.asciidoc @@ -17,7 +17,7 @@ exception for your {kib} URL. -- -. Click *Machine Learning* in the {kib} main menu. +. Open *Machine Learning* from the main menu, or use the {kibana-ref}/kibana-concepts-analysts.html#_finding_your_apps_and_objects[global search field]. . Select the *{data-viz}* tab. diff --git a/docs/en/stack/ml/index-custom-title-page.html b/docs/en/stack/ml/index-custom-title-page.html index 96eb0e1ca..eaea77a96 100644 --- a/docs/en/stack/ml/index-custom-title-page.html +++ b/docs/en/stack/ml/index-custom-title-page.html @@ -151,12 +151,18 @@

  • ELSER - Elastic Learned Sparse EncodeR
    • +
  • + E5 +
  • Supported third-party models
  • Examples
    • +
  • + Search Labs tutorial +
    • @@ -180,6 +186,9 @@

  • Anomaly detection function reference
    • +
  • + Search Labs +
    • @@ -187,7 +196,7 @@

    Explore by use case

    - +

    diff --git a/docs/en/stack/ml/nlp/images/ml-nlp-bm25-elser-v2.png b/docs/en/stack/ml/nlp/images/ml-nlp-bm25-elser-v2.png new file mode 100644 index 000000000..c609b193d Binary files /dev/null and b/docs/en/stack/ml/nlp/images/ml-nlp-bm25-elser-v2.png differ diff --git a/docs/en/stack/ml/nlp/images/ml-nlp-deploy-e5-es.png b/docs/en/stack/ml/nlp/images/ml-nlp-deploy-e5-es.png new file mode 100644 index 000000000..d3d53464c Binary files /dev/null and b/docs/en/stack/ml/nlp/images/ml-nlp-deploy-e5-es.png differ diff --git a/docs/en/stack/ml/nlp/images/ml-nlp-deploy-elser-v2-es.png b/docs/en/stack/ml/nlp/images/ml-nlp-deploy-elser-v2-es.png new file mode 100644 index 000000000..e547b3f97 Binary files /dev/null and b/docs/en/stack/ml/nlp/images/ml-nlp-deploy-elser-v2-es.png differ diff --git a/docs/en/stack/ml/nlp/images/ml-nlp-deployment-id-e5.png b/docs/en/stack/ml/nlp/images/ml-nlp-deployment-id-e5.png new file mode 100644 index 000000000..0a930142d Binary files /dev/null and b/docs/en/stack/ml/nlp/images/ml-nlp-deployment-id-e5.png differ diff --git a/docs/en/stack/ml/nlp/images/ml-nlp-deployment-id-elser-v2.png b/docs/en/stack/ml/nlp/images/ml-nlp-deployment-id-elser-v2.png new file mode 100644 index 000000000..d549ea815 Binary files /dev/null and b/docs/en/stack/ml/nlp/images/ml-nlp-deployment-id-elser-v2.png differ diff --git a/docs/en/stack/ml/nlp/images/ml-nlp-deployment-id-elser.png b/docs/en/stack/ml/nlp/images/ml-nlp-deployment-id-elser.png deleted file mode 100644 index 549f474a1..000000000 Binary files a/docs/en/stack/ml/nlp/images/ml-nlp-deployment-id-elser.png and /dev/null differ diff --git a/docs/en/stack/ml/nlp/images/ml-nlp-e5-download.png b/docs/en/stack/ml/nlp/images/ml-nlp-e5-download.png new file mode 100644 index 000000000..373802cef Binary files /dev/null and b/docs/en/stack/ml/nlp/images/ml-nlp-e5-download.png differ diff --git a/docs/en/stack/ml/nlp/images/ml-nlp-elser-average-ndcg.png b/docs/en/stack/ml/nlp/images/ml-nlp-elser-average-ndcg.png deleted file mode 100644 index d0b07440d..000000000 Binary files a/docs/en/stack/ml/nlp/images/ml-nlp-elser-average-ndcg.png and /dev/null differ diff --git a/docs/en/stack/ml/nlp/images/ml-nlp-elser-bm-summary.png b/docs/en/stack/ml/nlp/images/ml-nlp-elser-bm-summary.png new file mode 100644 index 000000000..52cce7d21 Binary files /dev/null and b/docs/en/stack/ml/nlp/images/ml-nlp-elser-bm-summary.png differ diff --git a/docs/en/stack/ml/nlp/images/ml-nlp-elser-download.png b/docs/en/stack/ml/nlp/images/ml-nlp-elser-download.png deleted file mode 100644 index d1fd1a81f..000000000 Binary files a/docs/en/stack/ml/nlp/images/ml-nlp-elser-download.png and /dev/null differ diff --git a/docs/en/stack/ml/nlp/images/ml-nlp-elser-ndcg10-beir.png b/docs/en/stack/ml/nlp/images/ml-nlp-elser-ndcg10-beir.png deleted file mode 100644 index 1befc5df5..000000000 Binary files a/docs/en/stack/ml/nlp/images/ml-nlp-elser-ndcg10-beir.png and /dev/null differ diff --git a/docs/en/stack/ml/nlp/images/ml-nlp-elser-test.png b/docs/en/stack/ml/nlp/images/ml-nlp-elser-test.png deleted file mode 100644 index ee01aad1b..000000000 Binary files a/docs/en/stack/ml/nlp/images/ml-nlp-elser-test.png and /dev/null differ diff --git a/docs/en/stack/ml/nlp/images/ml-nlp-elser-v2-cp-bm-results.png b/docs/en/stack/ml/nlp/images/ml-nlp-elser-v2-cp-bm-results.png new file mode 100644 index 000000000..c0c6b3b1e Binary files /dev/null and b/docs/en/stack/ml/nlp/images/ml-nlp-elser-v2-cp-bm-results.png differ diff --git a/docs/en/stack/ml/nlp/images/ml-nlp-elser-v2-download.png b/docs/en/stack/ml/nlp/images/ml-nlp-elser-v2-download.png new file mode 100644 index 000000000..636adb6ff Binary files /dev/null and b/docs/en/stack/ml/nlp/images/ml-nlp-elser-v2-download.png differ diff --git a/docs/en/stack/ml/nlp/images/ml-nlp-elser-v2-opt-bm-results.png b/docs/en/stack/ml/nlp/images/ml-nlp-elser-v2-opt-bm-results.png new file mode 100644 index 000000000..a7c97f238 Binary files /dev/null and b/docs/en/stack/ml/nlp/images/ml-nlp-elser-v2-opt-bm-results.png differ diff --git a/docs/en/stack/ml/nlp/images/ml-nlp-elser-v2-test.png b/docs/en/stack/ml/nlp/images/ml-nlp-elser-v2-test.png new file mode 100644 index 000000000..e0b1982b0 Binary files /dev/null and b/docs/en/stack/ml/nlp/images/ml-nlp-elser-v2-test.png differ diff --git a/docs/en/stack/ml/nlp/images/ml-nlp-pipeline-copy-customize.png b/docs/en/stack/ml/nlp/images/ml-nlp-pipeline-copy-customize.png new file mode 100644 index 000000000..1f2bf99aa Binary files /dev/null and b/docs/en/stack/ml/nlp/images/ml-nlp-pipeline-copy-customize.png differ diff --git a/docs/en/stack/ml/nlp/images/ml-nlp-start-e5-es.png b/docs/en/stack/ml/nlp/images/ml-nlp-start-e5-es.png new file mode 100644 index 000000000..2187f8b78 Binary files /dev/null and b/docs/en/stack/ml/nlp/images/ml-nlp-start-e5-es.png differ diff --git a/docs/en/stack/ml/nlp/images/ml-nlp-start-elser-v2-es.png b/docs/en/stack/ml/nlp/images/ml-nlp-start-elser-v2-es.png new file mode 100644 index 000000000..5d94354c7 Binary files /dev/null and b/docs/en/stack/ml/nlp/images/ml-nlp-start-elser-v2-es.png differ diff --git a/docs/en/stack/ml/nlp/images/ml-nlp-test-ner.png b/docs/en/stack/ml/nlp/images/ml-nlp-test-ner.png index 36541436f..e0f187e68 100644 Binary files a/docs/en/stack/ml/nlp/images/ml-nlp-test-ner.png and b/docs/en/stack/ml/nlp/images/ml-nlp-test-ner.png differ diff --git a/docs/en/stack/ml/nlp/index.asciidoc b/docs/en/stack/ml/nlp/index.asciidoc index 53259476d..206f72919 100644 --- a/docs/en/stack/ml/nlp/index.asciidoc +++ b/docs/en/stack/ml/nlp/index.asciidoc @@ -4,9 +4,14 @@ include::ml-nlp-extract-info.asciidoc[leveloffset=+2] include::ml-nlp-classify-text.asciidoc[leveloffset=+2] include::ml-nlp-search-compare.asciidoc[leveloffset=+2] include::ml-nlp-deploy-models.asciidoc[leveloffset=+1] +include::ml-nlp-autoscaling.asciidoc[leveloffset=+1] include::ml-nlp-inference.asciidoc[leveloffset=+1] include::ml-nlp-apis.asciidoc[leveloffset=+1] -include::ml-nlp-elser.asciidoc[leveloffset=+1] +include::ml-nlp-built-in-models.asciidoc[leveloffset=+1] +include::ml-nlp-elser.asciidoc[leveloffset=+2] +include::ml-nlp-elastic-rerank.asciidoc[leveloffset=+2] +include::ml-nlp-e5.asciidoc[leveloffset=+2] +include::ml-nlp-lang-ident.asciidoc[leveloffset=+2] include::ml-nlp-model-ref.asciidoc[leveloffset=+1] include::ml-nlp-examples.asciidoc[leveloffset=+1] include::ml-nlp-ner-example.asciidoc[leveloffset=+2] diff --git a/docs/en/stack/ml/nlp/ml-nlp-apis.asciidoc b/docs/en/stack/ml/nlp/ml-nlp-apis.asciidoc index 9dce692c6..ab0402b66 100644 --- a/docs/en/stack/ml/nlp/ml-nlp-apis.asciidoc +++ b/docs/en/stack/ml/nlp/ml-nlp-apis.asciidoc @@ -26,3 +26,20 @@ All the trained models endpoints have the following base: * {ref}/stop-trained-model-deployment.html[Stop trained model deployment] // UPDATE * {ref}/put-trained-models-aliases.html[Update trained model aliases] + + +You can also integrate NLP models from different providers such as Cohere, +HuggingFace, or OpenAI and use them as a service through the {infer} API. + +The {infer} APIs have the following base: + +[source,js] +---- +/_inference/ +---- +// NOTCONSOLE + +* {ref}/put-inference-api.html[Create inference endpoint] +* {ref}/delete-inference-api.html[Delete inference endpoint] +* {ref}/get-inference-api.html[Get inference endpoint] +* {ref}/post-inference-api.html[Perform inference] \ No newline at end of file diff --git a/docs/en/stack/ml/nlp/ml-nlp-autoscaling.asciidoc b/docs/en/stack/ml/nlp/ml-nlp-autoscaling.asciidoc new file mode 100644 index 000000000..0906621e7 --- /dev/null +++ b/docs/en/stack/ml/nlp/ml-nlp-autoscaling.asciidoc @@ -0,0 +1,159 @@ +[[ml-nlp-auto-scale]] += Trained model autoscaling + +You can enable autoscaling for each of your trained model deployments. +Autoscaling allows {es} to automatically adjust the resources the model deployment can use based on the workload demand. + +There are two ways to enable autoscaling: + +* through APIs by enabling adaptive allocations +* in {kib} by enabling adaptive resources + +IMPORTANT: To fully leverage model autoscaling, it is highly recommended to enable {cloud}/ec-autoscaling.html[{es} deployment autoscaling]. + + +[discrete] +[[nlp-model-adaptive-allocations]] +== Enabling autoscaling through APIs - adaptive allocations + +Model allocations are independent units of work for NLP tasks. +If you set the numbers of threads and allocations for a model manually, they remain constant even when not all the available resources are fully used or when the load on the model requires more resources. +Instead of setting the number of allocations manually, you can enable adaptive allocations to set the number of allocations based on the load on the process. +This can help you to manage performance and cost more easily. +(Refer to the https://cloud.elastic.co/pricing[pricing calculator] to learn more about the possible costs.) + +When adaptive allocations are enabled, the number of allocations of the model is set automatically based on the current load. +When the load is high, a new model allocation is automatically created. +When the load is low, a model allocation is automatically removed. +You can explicitely set the minimum and maximum number of allocations; autoscaling will occur within these limits. + +You can enable adaptive allocations by using: + +* the create inference endpoint API for {ref}/infer-service-elser.html[ELSER], {ref}/infer-service-elasticsearch.html[E5 and models uploaded through Eland] that are used as {infer} services. +* the {ref}/start-trained-model-deployment.html[start trained model deployment] or {ref}/update-trained-model-deployment.html[update trained model deployment] APIs for trained models that are deployed on {ml} nodes. + +If the new allocations fit on the current {ml} nodes, they are immediately started. +If more resource capacity is needed for creating new model allocations, then your {ml} node will be scaled up if {ml} autoscaling is enabled to provide enough resources for the new allocation. +The number of model allocations can be scaled down to 0. +They cannot be scaled up to more than 32 allocations, unless you explicitly set the maximum number of allocations to more. +Adaptive allocations must be set up independently for each deployment and {ref}/put-inference-api.html[{infer} endpoint]. + + +[discrete] +[[optimize-use-case]] +=== Optimizing for typical use cases + +You can optimize your model deployment for typical use cases, such as search and ingest. +When you optimize for ingest, the throughput will be higher, which increases the number of {infer} requests that can be performed in parallel. +When you optimize for search, the latency will be lower during search processes. + +* If you want to optimize for ingest, set the number of threads to `1` (`"threads_per_allocation": 1`). +* If you want to optimize for search, set the number of threads to greater than `1`. +Increasing the number of threads will make the search processes more performant. + + +[discrete] +[[nlp-model-adaptive-resources]] +== Enabling autoscaling in {kib} - adaptive resources + +You can enable adaptive resources for your models when starting or updating the model deployment. +Adaptive resources make it possible for {es} to scale up or down the available resources based on the load on the process. +This can help you to manage performance and cost more easily. +When adaptive resources are enabled, the number of vCPUs that the model deployment uses is set automatically based on the current load. +When the load is high, the number of vCPUs that the process can use is automatically increased. +When the load is low, the number of vCPUs that the process can use is automatically decreased. + +You can choose from three levels of resource usage for your trained model deployment; autoscaling will occur within the selected level's range. + +Refer to the tables in the <> section to find out the setings for the level you selected. + + +[role="screenshot"] +image::images/ml-nlp-deployment-id-elser-v2.png["ELSER deployment with adaptive resources enabled.",width=640] + + +[discrete] +[[auto-scaling-matrix]] +== Model deployment resource matrix + +The used resources for trained model deployments depend on three factors: + +* your cluster environment (Serverless, Cloud, or on-premises) +* the use case you optimize the model deployment for (ingest or search) +* whether model autoscaling is enabled with adaptive allocations/resources to have dynamic resources, or disabled for static resources + +If you use {es} on-premises, vCPUs level ranges are derived from the `total_ml_processors` and `max_single_ml_node_processors` values. +Use the {ref}/get-ml-info.html[get {ml} info API] to check these values. +The following tables show you the number of allocations, threads, and vCPUs available in Cloud when adaptive resources are enabled or disabled. + +NOTE: On Serverless, adaptive allocations are automatically enabled for all project types. +However, the "Adaptive resources" control is not displayed in {kib} for Observability and Security projects. + + +[discrete] +=== Deployments in Cloud optimized for ingest + +In case of ingest-optimized deployments, we maximize the number of model allocations. + + +[discrete] +==== Adaptive resources enabled + +[cols="4*", options="header"] +|========== +| Level | Allocations | Threads | vCPUs +| Low | 0 to 2 if available, dynamically | 1 | 0 to 2 if available, dynamically +| Medium | 1 to 32 dynamically | 1 | 1 to the smaller of 32 or the limit set in the Cloud console, dynamically +| High | 1 to limit set in the Cloud console ^*^, dynamically | 1 | 1 to limit set in the Cloud console, dynamically +|========== + +^*^ The Cloud console doesn't directly set an allocations limit; it only sets a vCPU limit. +This vCPU limit indirectly determines the number of allocations, calculated as the vCPU limit divided by the number of threads. + +[discrete] +==== Adaptive resources disabled + +[cols="4*", options="header"] +|========== +| Level | Allocations | Threads | vCPUs +| Low | 2 if available, otherwise 1, statically | 1 | 2 if available +| Medium | the smaller of 32 or the limit set in the Cloud console, statically | 1 | 32 if available +| High | Maximum available set in the Cloud console ^*^, statically | 1 | Maximum available set in the Cloud console, statically +|========== + +^*^ The Cloud console doesn't directly set an allocations limit; it only sets a vCPU limit. +This vCPU limit indirectly determines the number of allocations, calculated as the vCPU limit divided by the number of threads. + +[discrete] +=== Deployments in Cloud optimized for search + +In case of search-optimized deployments, we maximize the number of threads. +The maximum number of threads that can be claimed depends on the hardware your architecture has. + +[discrete] +==== Adaptive resources enabled + +[cols="4*", options="header"] +|========== +| Level | Allocations | Threads | vCPUs +| Low | 1 | 2 | 2 +| Medium | 1 to 2 (if threads=16) dynamically | maximum that the hardware allows (for example, 16) | 1 to 32 dynamically +| High | 1 to limit set in the Cloud console ^*^, dynamically| maximum that the hardware allows (for example, 16) | 1 to limit set in the Cloud console, dynamically +|========== + +^*^ The Cloud console doesn't directly set an allocations limit; it only sets a vCPU limit. +This vCPU limit indirectly determines the number of allocations, calculated as the vCPU limit divided by the number of threads. + +[discrete] +==== Adaptive resources disabled + +[cols="4*", options="header"] +|========== +| Level | Allocations | Threads | vCPUs +| Low | 1 if available, statically | 2 | 2 if available +| Medium | 2 (if threads=16) statically | maximum that the hardware allows (for example, 16) | 32 if available +| High | Maximum available set in the Cloud console ^*^, statically | maximum that the hardware allows (for example, 16) | Maximum available set in the Cloud console, statically +|========== + +^*^ The Cloud console doesn't directly set an allocations limit; it only sets a vCPU limit. +This vCPU limit indirectly determines the number of allocations, calculated as the vCPU limit divided by the number of threads. \ No newline at end of file diff --git a/docs/en/stack/ml/nlp/ml-nlp-built-in-models.asciidoc b/docs/en/stack/ml/nlp/ml-nlp-built-in-models.asciidoc new file mode 100644 index 000000000..5d068b41e --- /dev/null +++ b/docs/en/stack/ml/nlp/ml-nlp-built-in-models.asciidoc @@ -0,0 +1,11 @@ +[[ml-nlp-built-in-models]] += Built-in NLP models + +There are {nlp} models that are available for use in every cluster +out-of-the-box. These models are pre-trained which means they don't require +fine-tuning on your own data, making it adaptable for various use cases out of +the box. The following models are available: + +* <> trained by Elastic +* <> +* <> \ No newline at end of file diff --git a/docs/en/stack/ml/nlp/ml-nlp-classify-text.asciidoc b/docs/en/stack/ml/nlp/ml-nlp-classify-text.asciidoc index 0afabb25d..fad14e85b 100644 --- a/docs/en/stack/ml/nlp/ml-nlp-classify-text.asciidoc +++ b/docs/en/stack/ml/nlp/ml-nlp-classify-text.asciidoc @@ -13,7 +13,13 @@ label unstructured input text: * <> * <> -include::ml-nlp-lang-ident.asciidoc[] + +[discrete] +== {lang-ident-cap} + +The {lang-ident} model is provided out-of-the box in your {es} cluster. You can +find the documentation of the model on the <> page under the +Built-in models section. [discrete] diff --git a/docs/en/stack/ml/nlp/ml-nlp-deploy-models.asciidoc b/docs/en/stack/ml/nlp/ml-nlp-deploy-models.asciidoc index 12685e9c5..06c063f1b 100644 --- a/docs/en/stack/ml/nlp/ml-nlp-deploy-models.asciidoc +++ b/docs/en/stack/ml/nlp/ml-nlp-deploy-models.asciidoc @@ -81,11 +81,11 @@ example: -- [source, shell] -------------------------------------------------- -eland_import_hub_model +eland_import_hub_model \ --cloud-id \ <1> -u -p \ <2> --hub-model-id elastic/distilbert-base-cased-finetuned-conll03-english \ <3> ---task-type ner \ <4> +--task-type ner <4> -------------------------------------------------- // NOTCONSOLE -- @@ -94,7 +94,8 @@ eland_import_hub_model <> to learn more. <3> Specify the identifier for the model in the Hugging Face model hub. <4> Specify the type of NLP task. Supported values are `fill_mask`, `ner`, -`text_classification`, `text_embedding`, and `zero_shot_classification`. +`question_answering`, `text_classification`, `text_embedding`, `text_expansion`, +`text_similarity`, and `zero_shot_classification`. For more details, refer to https://www.elastic.co/guide/en/elasticsearch/client/eland/current/machine-learning.html#ml-nlp-pytorch. @@ -103,26 +104,16 @@ https://www.elastic.co/guide/en/elasticsearch/client/eland/current/machine-learn [[ml-nlp-import-docker]] === Import with Docker -IMPORTANT: To use the Docker container, you need to clone the Eland repository: -https://github.com/elastic/eland - -If you want to use Eland without installing it, clone the Eland repository and -from the root directory run the following to build the Docker image: - -```bash -$ docker build -t elastic/eland . -``` - -You can now use the container interactively: +If you want to use Eland without installing it, run the following command: ```bash -$ docker run -it --rm --network host elastic/eland +$ docker run -it --rm --network host docker.elastic.co/eland/eland ``` The `eland_import_hub_model` script can be run directly in the docker command: ```bash -docker run -it --rm elastic/eland \ +docker run -it --rm docker.elastic.co/eland/eland \ eland_import_hub_model \ --url $ELASTICSEARCH_URL \ --hub-model-id elastic/distilbert-base-uncased-finetuned-conll03-english \ @@ -173,50 +164,35 @@ their deployment across your cluster under **{ml-app}** > *Model Management*. Alternatively, you can use the {ref}/start-trained-model-deployment.html[start trained model deployment API]. -You can deploy a model multiple times by assigning a unique deployment ID when -starting the deployment. It enables you to have dedicated deployments for -different purposes, such as search and ingest. By doing so, you ensure that the -search speed remains unaffected by ingest workloads, and vice versa. Having -separate deployments for search and ingest mitigates performance issues -resulting from interactions between the two, which can be hard to diagnose. +You can deploy a model multiple times by assigning a unique deployment ID when starting the deployment. + +You can optimize your deplyoment for typical use cases, such as search and ingest. +When you optimize for ingest, the throughput will be higher, which increases the number of {infer} requests that can be performed in parallel. +When you optimize for search, the latency will be lower during search processes. +When you have dedicated deployments for different purposes, you ensure that the search speed remains unaffected by ingest workloads, and vice versa. +Having separate deployments for search and ingest mitigates performance issues resulting from interactions between the two, which can be hard to diagnose. [role="screenshot"] -image::images/ml-nlp-deployment-id-elser.png["Model deployment on the Trained Models UI."] - -It is recommended to fine-tune each deployment based on its specific purpose. To -improve ingest performance, increase throughput by adding more allocations to -the deployment. For improved search speed, increase the number of threads per -allocation. - -NOTE: Since eland uses APIs to deploy the models, you cannot see the models in -{kib} until the saved objects are synchronized. You can follow the prompts in -{kib}, wait for automatic synchronization, or use the -{kibana-ref}/machine-learning-api-sync.html[sync {ml} saved objects API]. - -When you deploy the model, its allocations are distributed across available {ml} -nodes. Model allocations are independent units of work for NLP tasks. To -influence model performance, you can configure the number of allocations and the -number of threads used by each allocation of your deployment. - -Throughput can be scaled by adding more allocations to the deployment; it -increases the number of {infer} requests that can be performed in parallel. All -allocations assigned to a node share the same copy of the model in memory. The -model is loaded into memory in a native process that encapsulates `libtorch`, -which is the underlying {ml} library of PyTorch. The number of allocations -setting affects the amount of model allocations across all the {ml} nodes. Model -allocations are distributed in such a way that the total number of used threads -does not exceed the allocated processors of a node. - -The threads per allocation setting affects the number of threads used by each -model allocation during {infer}. Increasing the number of threads generally -increases the speed of {infer} requests. The value of this setting must not -exceed the number of available allocated processors per node. - -You can view the allocation status in {kib} or by using the -{ref}/get-trained-models-stats.html[get trained model stats API]. If you to -change the number of allocations, you can use the -{ref}/update-trained-model-deployment.html[update trained model stats API] after -the allocation status is `started`. +image::images/ml-nlp-deployment-id-elser-v2.png["Model deployment on the Trained Models UI.",width=640] + +Each deployment will be fine-tuned automatically based on its specific purpose you choose. + +NOTE: Since eland uses APIs to deploy the models, you cannot see the models in {kib} until the saved objects are synchronized. +You can follow the prompts in {kib}, wait for automatic synchronization, or use the {kibana-ref}/machine-learning-api-sync.html[sync {ml} saved objects API]. + +You can define the resource usage level of the NLP model during model deployment. +The resource usage levels behave differently depending on <> being enabled or disabled. +When adaptive resources are disabled but {ml} autoscaling is enabled, vCPU usage of Cloud deployments derived from the Cloud console and functions as follows: + +* Low: This level limits resources to two vCPUs, which may be suitable for development, testing, and demos depending on your parameters. +It is not recommended for production use +* Medium: This level limits resources to 32 vCPUs, which may be suitable for development, testing, and demos depending on your parameters. +It is not recommended for production use. +* High: This level may use the maximum number of vCPUs available for this deployment from the Cloud console. +If the maximum is 2 vCPUs or fewer, this level is equivalent to the medium or low level. + +For the resource levels when adaptive resources are enabled, refer to <<>. + [discrete] [[infer-request-queues]] @@ -255,7 +231,7 @@ The simplest method to test your model against new data is to use the field of an existing index in your cluster to test the model: [role="screenshot"] -image::images/ml-nlp-test-ner.png[Testing a sentence with two named entities against a NER trained model in the *{ml}* app] +image::images/ml-nlp-test-ner.png["Testing a sentence with two named entities against a NER trained model in the *{ml}* app",] Alternatively, you can use the {ref}/infer-trained-model.html[infer trained model API]. @@ -305,4 +281,4 @@ If you are satisfied with the results, you can add these NLP tasks in your <>. :!keywords: -:!description: \ No newline at end of file +:!description: diff --git a/docs/en/stack/ml/nlp/ml-nlp-e5.asciidoc b/docs/en/stack/ml/nlp/ml-nlp-e5.asciidoc new file mode 100644 index 000000000..28c5303f5 --- /dev/null +++ b/docs/en/stack/ml/nlp/ml-nlp-e5.asciidoc @@ -0,0 +1,331 @@ +[[ml-nlp-e5]] += E5 – EmbEddings from bidirEctional Encoder rEpresentations +++++ +E5 +++++ + +:frontmatter-description: E5 is a multilingual dense vector model. +:frontmatter-tags-products: [ml] +:frontmatter-tags-content-type: [how-to] +:frontmatter-tags-user-goals: [analyze] + +EmbEddings from bidirEctional Encoder rEpresentations - or E5 - is a {nlp} +model that enables you to perform multi-lingual semantic search by using dense +vector representations. This model is recommended for non-English language +documents and queries. If you want to perform semantic search on English +language documents, use the <> model. + +{ref}/semantic-search.html[Semantic search] provides you search results based on +contextual meaning and user intent, rather than exact keyword matches. + +E5 has two versions: one cross-platform version which runs on any hardware +and one version which is optimized for Intel® silicon. The +**Model Management** > **Trained Models** page shows you which version of E5 is +recommended to deploy based on your cluster's hardware. However, the +recommended way to use E5 is through the +{ref}/infer-service-elasticsearch.html[{infer} API] as a service which makes it +easier to download and deploy the model and you don't need to select from +different versions. + +Refer to the model cards of the +https://huggingface.co/elastic/multilingual-e5-small[multilingual-e5-small] and +the +https://huggingface.co/elastic/multilingual-e5-small-optimized[multilingual-e5-small-optimized] +models on HuggingFace for further information including licensing. + + +[discrete] +[[e5-req]] +== Requirements + +To use E5, you must have the {subscriptions}[appropriate subscription] level +for semantic search or the trial period activated. + +Enabling trained model autoscaling for your E5 deployment is recommended. +Refer to <> to learn more. + + +[discrete] +[[download-deploy-e5]] +== Download and deploy E5 + +The easiest and recommended way to download and deploy E5 is to use the {ref}/inference-apis.html[{infer} API]. + +1. In {kib}, navigate to the **Dev Console**. +2. Create an {infer} endpoint with the `elasticsearch` service by running the following API request: ++ +-- +[source,console] +---------------------------------- +PUT _inference/text_embedding/my-e5-model +{ + "service": "elasticsearch", + "service_settings": { + "num_allocations": 1, + "num_threads": 1, + "model_id": ".multilingual-e5-small" + } +} +---------------------------------- +-- +The API request automatically initiates the model download and then deploy the model. + +Refer to the {ref}/infer-service-elasticsearch.html[`elasticsearch` {infer} service documentation] to learn more about the available settings. + +After you created the E5 {infer} endpoint, it's ready to be used for semantic search. +The easiest way to perform semantic search in the {stack} is to {ref}/semantic-search-semantic-text.html[follow the `semantic_text` workflow]. + + +[discrete] +[[alternative-download-deploy-e5]] +=== Alternative methods to download and deploy E5 + +You can also download and deploy the E5 model either from **{ml-app}** > **Trained Models**, from **Search** > **Indices**, or by using the trained models API in Dev Console. + +NOTE: For most cases, the preferred version is the **Intel and Linux optimized** model, it is recommended to download and deploy that version. + + +.Using the Trained Models page +[%collapsible%closed] +===== +[discrete] +[[trained-model-e5]] +==== Using the Trained Models page + +1. In {kib}, navigate to **{ml-app}** > **Trained Models** from the main menu, or use the {kibana-ref}/kibana-concepts-analysts.html#_finding_your_apps_and_objects[global search field]. E5 can be found in +the list of trained models. There are two versions available: one portable +version which runs on any hardware and one version which is optimized for Intel® +silicon. You can see which model is recommended to use based on your hardware +configuration. +2. Click the **Add trained model** button. Select the E5 model version you want +to use in the opening modal window. The model that is recommended for you based +on your hardware configuration is highlighted. Click **Download**. You can check +the download status on the **Notifications** page. ++ +-- +[role="screenshot"] +image::images/ml-nlp-e5-download.png[alt="Downloading E5",align="center"] +-- ++ +-- +Alternatively, click the **Download model** button under **Actions** in the +trained model list. +-- +3. After the download is finished, start the deployment by clicking the +**Start deployment** button. +4. Provide a deployment ID, select the priority, and set the number of +allocations and threads per allocation values. ++ +-- +[role="screenshot"] +image::images/ml-nlp-deployment-id-e5.png[alt="Deploying E5",align="center"] +-- +5. Click Start. +===== + + +.Using the search indices UI +[%collapsible%closed] +===== +[discrete] +[[elasticsearch-e5]] +==== Using the search indices UI + +Alternatively, you can download and deploy the E5 model to an {infer} pipeline +using the search indices UI. + +1. In {kib}, navigate to **Search** > **Indices**. +2. Select the index from the list that has an {infer} pipeline in which you want +to use E5. +3. Navigate to the **Pipelines** tab. +4. Under **{ml-app} {infer-cap} Pipelines**, click the **Deploy** button in the +**Improve your results with E5** section to begin downloading the E5 model. This +may take a few minutes depending on your network. ++ +-- +[role="screenshot"] +image::images/ml-nlp-deploy-e5-es.png[alt="Deploying E5 in Elasticsearch",align="center"] +-- +5. Once the model is downloaded, click the **Start single-threaded** button to +start the model with basic configuration or select the **Fine-tune performance** +option to navigate to the **Trained Models** page where you can configure the +model deployment. ++ +-- +[role="screenshot"] +image::images/ml-nlp-start-e5-es.png[alt="Start E5 in Elasticsearch",align="center"] +-- + +When your E5 model is deployed and started, it is ready to be used in a +pipeline. +===== + + +.Using the trained models API in Dev Console +[%collapsible%closed] +===== +[discrete] +[[dev-console-e5]] +==== Using the trained models API in Dev Console + +1. In {kib}, navigate to the **Dev Console**. +2. Create the E5 model configuration by running the following API call: ++ +-- +[source,console] +---------------------------------- +PUT _ml/trained_models/.multilingual-e5-small +{ + "input": { + "field_names": ["text_field"] + } +} +---------------------------------- + +The API call automatically initiates the model download if the model is not +downloaded yet. +-- +3. Deploy the model by using the +{ref}/start-trained-model-deployment.html[start trained model deployment API] +with a delpoyment ID: ++ +-- +[source,console] +---------------------------------- +POST _ml/trained_models/.multilingual-e5-small/deployment/_start?deployment_id=for_search +---------------------------------- +-- +===== + + +[discrete] +[[air-gapped-install-e5]] +== Deploy the E5 model in an air-gapped environment + +If you want to install E5 in an air-gapped environment, you have the following +options: +* put the model artifacts into a directory inside the config directory on all +master-eligible nodes (for `multilingual-e5-small` and +`multilingual-e5-small-linux-x86-64`) +* install the model by using HuggingFace (for `multilingual-e5-small` model +only). + + +[discrete] +[[e5-model-artifacts]] +=== Model artifact files + +For the `multilingual-e5-small` model, you need the following files in your +system: +``` +https://ml-models.elastic.co/multilingual-e5-small.metadata.json +https://ml-models.elastic.co/multilingual-e5-small.pt +https://ml-models.elastic.co/multilingual-e5-small.vocab.json +``` + +For the optimized version, you need the following files in your system: +``` +https://ml-models.elastic.co/multilingual-e5-small_linux-x86_64.metadata.json +https://ml-models.elastic.co/multilingual-e5-small_linux-x86_64.pt +https://ml-models.elastic.co/multilingual-e5-small_linux-x86_64.vocab.json +``` + + +[discrete] +=== Using file-based access + +For a file-based access, follow these steps: + +. Download the <>. +. Put the files into a `models` subdirectory inside the `config` directory of +your {es} deployment. +. Point your {es} deployment to the model directory by adding the following line +to the `config/elasticsearch.yml` file: ++ +-- +``` +xpack.ml.model_repository: file://${path.home}/config/models/` +``` +-- +. Repeat step 2 and step 3 on all master-eligible nodes. +. {ref}/restart-cluster.html#restart-cluster-rolling[Restart] the +master-eligible nodes one by one. +. Navigate to the **Trained Models** page from the main menu, or use the {kibana-ref}/kibana-concepts-analysts.html#_finding_your_apps_and_objects[global search field] in {kib}. E5 can be found in the +list of trained models. +. Click the **Add trained model** button, select the E5 model version you +downloaded in step 1 and want to deploy and click **Download**. The selected +model will be downloaded from the model directory where you put in step 2. +. After the download is finished, start the deployment by clicking the +**Start deployment** button. +. Provide a deployment ID, select the priority, and set the number of +allocations and threads per allocation values. +. Click **Start**. + + +[discrete] +=== Using the HuggingFace repository + +You can install the `multilingual-e5-small` model in a restricted or closed +network by pointing the `eland_import_hub_model` script to the model's local +files. + +For an offline install, the model first needs to be cloned locally, Git and +https://git-lfs.com/[Git Large File Storage] are required to be installed in +your system. + +1. Clone the E5 model from Hugging Face by using the model URL. ++ +-- +[source,bash] +---- +git clone https://huggingface.co/intfloat/multilingual-e5-small +---- +The command results in a local copy of the model in the `multilingual-e5-small` +directory. +-- + +2. Use the `eland_import_hub_model` script with the `--hub-model-id` set to the +directory of the cloned model to install it: ++ +-- +[source,bash] +---- +eland_import_hub_model \ + --url 'XXXX' \ + --hub-model-id /PATH/TO/MODEL \ + --task-type text_embedding \ + --es-username elastic --es-password XXX \ + --es-model-id multilingual-e5-small +---- + +If you use the Docker image to run `eland_import_hub_model` you must bind mount +the model directory, so the container can read the files. +[source,bash] +---- +docker run --mount type=bind,source=/PATH/TO/MODELS,destination=/models,readonly -it --rm docker.elastic.co/eland/eland \ + eland_import_hub_model \ + --url 'XXXX' \ + --hub-model-id /models/multilingual-e5-small \ + --task-type text_embedding \ + --es-username elastic --es-password XXX \ + --es-model-id multilingual-e5-small +---- +Once it's uploaded to {es}, the model will have the ID specified by +`--es-model-id`. If it is not set, the model ID is derived from +`--hub-model-id`; spaces and path delimiters are converted to double +underscores `__`. +-- + + +[discrete] +[[terms-of-use-e5]] +== Disclaimer + +Customers may add third party trained models for management in Elastic. These +models are not owned by Elastic. While Elastic will support the integration with +these models in the performance according to the documentation, you understand +and agree that Elastic has no control over, or liability for, the third party +models or the underlying training data they may utilize. + +This e5 model, as defined, hosted, integrated and used in conjunction with our +other Elastic Software is covered by our standard warranty. diff --git a/docs/en/stack/ml/nlp/ml-nlp-elastic-rerank.asciidoc b/docs/en/stack/ml/nlp/ml-nlp-elastic-rerank.asciidoc new file mode 100644 index 000000000..982831bb7 --- /dev/null +++ b/docs/en/stack/ml/nlp/ml-nlp-elastic-rerank.asciidoc @@ -0,0 +1,365 @@ +[[ml-nlp-rerank]] += Elastic Rerank + +Elastic Rerank is a state-of-the-art cross-encoder reranking model trained by Elastic that helps you improve search relevance with a few simple API calls. +Elastic Rerank is Elastic's first semantic reranking model and is available out-of-the-box in supporting Elastic deployments using the {es} Inference API. + +Use Elastic Rerank to improve existing search applications including: + +* Traditional BM25 scoring +* Hybrid semantic search +* Retrieval Augmented Generation (RAG) + +The model can significantly improve search result quality by reordering results based on deeper semantic understanding of queries and documents. + +When reranking BM25 results, it provides an average 40% improvement in ranking quality on a diverse benchmark of retrieval tasks— matching the performance of models 11x its size. + +[discrete] +[[ml-nlp-rerank-availability]] +== Availability and requirements + +experimental[] + +[discrete] +[[ml-nlp-rerank-availability-serverless]] +=== Elastic Cloud Serverless + +Elastic Rerank is available in {es} Serverless projects as of November 25, 2024. + +[discrete] +[[ml-nlp-rerank-availability-elastic-stack]] +=== Elastic Cloud Hosted and self-managed deployments + +Elastic Rerank is available in Elastic Stack version 8.17+: + +* To use Elastic Rerank, you must have the appropriate subscription level or the trial period activated. +* A 4GB ML node ++ +[IMPORTANT] +==== +Deploying the Elastic Rerank model in combination with ELSER (or other hosted models) requires at minimum an 8GB ML node. The current maximum size for trial ML nodes is 4GB (defaults to 1GB). +==== + +[discrete] +[[ml-nlp-rerank-deploy]] +== Download and deploy + +To download and deploy Elastic Rerank, use the {ref}/infer-service-elasticsearch.html[create inference API] to create an {es} service `rerank` endpoint. + +[TIP] +==== +Refer to this https://github.com/elastic/elasticsearch-labs/blob/main/notebooks/search/12-semantic-reranking-elastic-rerank.ipynb[Python notebook] for an end-to-end example using Elastic Rerank. +==== + +[discrete] +[[ml-nlp-rerank-deploy-steps]] +=== Create an inference endpoint + +. In {kib}, navigate to the *Dev Console*. + +. Create an {infer} endpoint with the Elastic Rerank service by running: ++ +[source,console] +---------------------------------- +PUT _inference/rerank/my-rerank-model +{ + "service": "elasticsearch", + "service_settings": { + "adaptive_allocations": { + "enabled": true, + "min_number_of_allocations": 1, + "max_number_of_allocations": 10 + }, + "num_threads": 1, + "model_id": ".rerank-v1" + } +} +---------------------------------- ++ +NOTE: The API request automatically downloads and deploys the model. This example uses <> through adaptive allocation. + +[NOTE] +==== +You might see a 502 bad gateway error in the response when using the {kib} Console. +This error usually just reflects a timeout, while the model downloads in the background. +You can check the download progress in the {ml-app} UI. +If using the Python client, you can set the `timeout` parameter to a higher value. +==== + +After creating the Elastic Rerank {infer} endpoint, it's ready to use with a {ref}/retriever.html#text-similarity-reranker-retriever-example-elastic-rerank[`text_similarity_reranker`] retriever. + +[discrete] +[[ml-nlp-rerank-deploy-verify]] +== Deploy in an air-gapped environment + +If you want to deploy the Elastic Rerank model in a restricted or closed network, you have two options: + +* Create your own HTTP/HTTPS endpoint with the model artifacts on it +* Put the model artifacts into a directory inside the config directory on all master-eligible nodes. + +[discrete] +[[ml-nlp-rerank-model-artifacts]] +=== Model artifact files + +For the cross-platform version, you need the following files in your system: +``` +https://ml-models.elastic.co/rerank-v1.metadata.json +https://ml-models.elastic.co/rerank-v1.pt +https://ml-models.elastic.co/rerank-v1.vocab.json +``` + +// For the optimized version, you need the following files in your system: +// ``` +// https://ml-models.elastic.co/rerank-v1_linux-x86_64.metadata.json +// https://ml-models.elastic.co/rerank-v1_linux-x86_64.pt +// https://ml-models.elastic.co/rerank-v1_linux-x86_64.vocab.json +// ``` + +[discrete] +=== Using an HTTP server + +INFO: If you use an existing HTTP server, note that the model downloader only +supports passwordless HTTP servers. + +You can use any HTTP service to deploy the model. This example uses the official +Nginx Docker image to set a new HTTP download service up. + +. Download the <>. +. Put the files into a subdirectory of your choice. +. Run the following commands: ++ +-- +[source, shell] +-------------------------------------------------- +export ELASTIC_ML_MODELS="/path/to/models" +docker run --rm -d -p 8080:80 --name ml-models -v ${ELASTIC_ML_MODELS}:/usr/share/nginx/html nginx +-------------------------------------------------- + +Don't forget to change `/path/to/models` to the path of the subdirectory where +the model artifact files are located. + +These commands start a local Docker image with an Nginx server with the +subdirectory containing the model files. As the Docker image has to be +downloaded and built, the first start might take a longer period of time. +Subsequent runs start quicker. +-- +. Verify that Nginx runs properly by visiting the following URL in your +browser: ++ +-- +``` +http://{IP_ADDRESS_OR_HOSTNAME}:8080/rerank-v1.metadata.json +``` + +If Nginx runs properly, you see the content of the metdata file of the model. +-- +. Point your {es} deployment to the model artifacts on the HTTP server +by adding the following line to the `config/elasticsearch.yml` file: ++ +-- +``` +xpack.ml.model_repository: http://{IP_ADDRESS_OR_HOSTNAME}:8080 +``` + +If you use your own HTTP or HTTPS server, change the address accordingly. It is +important to specificy the protocol ("http://" or "https://"). Ensure that all +master-eligible nodes can reach the server you specify. +-- +. Repeat step 5 on all master-eligible nodes. +. {ref}/restart-cluster.html#restart-cluster-rolling[Restart] the +master-eligible nodes one by one. +. Create an inference endpoint to deploy the model per <>. + +The HTTP server is only required for downloading the model. After the download +has finished, you can stop and delete the service. You can stop the Docker image +used in this example by running the following command: + +[source, shell] +-------------------------------------------------- +docker stop ml-models +-------------------------------------------------- + +[discrete] +=== Using file-based access + +For a file-based access, follow these steps: + +. Download the <>. +. Put the files into a `models` subdirectory inside the `config` directory of +your {es} deployment. +. Point your {es} deployment to the model directory by adding the +following line to the `config/elasticsearch.yml` file: ++ +-- +``` +xpack.ml.model_repository: file://${path.home}/config/models/ +``` +-- +. Repeat step 2 and step 3 on all master-eligible nodes. +. {ref}/restart-cluster.html#restart-cluster-rolling[Restart] the +master-eligible nodes one by one. +. Create an inference endpoint to deploy the model per <>. + +[discrete] +[[ml-nlp-rerank-limitations]] +== Limitations + +* English language only +* Maximum context window of 512 tokens ++ +When using the {ref}/semantic-text.html[`semantic_text` field type], text is divided into chunks. By default, each chunk contains 250 words (approximately 400 tokens). Be cautious when increasing the chunk size - if the combined length of your query and chunk text exceeds 512 tokens, the model won't have access to the full content. ++ +When the combined inputs exceed the 512 token limit, a balanced truncation strategy is used. If both the query and input text are longer than 255 tokens each then both are truncated, otherwise the longest is truncated. + +[discrete] +[[ml-nlp-rerank-perf-considerations]] +== Performance considerations + +It's important to note that if you rerank to depth `n` then you will need to run `n` inferences per query. This will include the document text and will therefore be significantly more expensive than inference for query embeddings. Hardware can be scaled to run these inferences in parallel, but we would recommend shallow reranking for CPU inference: no more than top-30 results. You may find that the preview version is cost prohibitive for high query rates and low query latency requirements. We plan to address performance issues for GA. + +[discrete] +[[ml-nlp-rerank-model-specs]] +== Model specifications + +* Purpose-built for English language content + +* Relatively small: 184M parameters (86M backbone + 98M embedding layer) + +* Matches performance of billion-parameter reranking models + +* Built directly into {es} - no external services or dependencies needed + +[discrete] +[[ml-nlp-rerank-arch-overview]] +== Model architecture + +Elastic Rerank is built on the https://arxiv.org/abs/2111.09543[DeBERTa v3] language model architecture. + +The model employs several key architectural features that make it particularly effective for reranking: + +* *Disentangled attention mechanism* enables the model to: +** Process word content and position separately +** Learn more nuanced relationships between query and document text +** Better understand the semantic importance of word positions and relationships + +* *ELECTRA-style pre-training* uses: +** A GAN-like approach to token prediction +** Simultaneous training of token generation and detection +** Enhanced parameter efficiency compared to traditional masked language modeling + +[discrete] +[[ml-nlp-rerank-arch-training]] +== Training process + +Here is an overview of the Elastic Rerank model training process: + +* *Initial relevance extraction* +** Fine-tunes the pre-trained DeBERTa [CLS] token representation +** Uses a GeLU activation and dropout layer +** Preserves important pre-trained knowledge while adapting to the reranking task + +* *Trained by distillation* +** Uses an ensemble of bi-encoder and cross-encoder models as a teacher +** Bi-encoder provides nuanced negative example assessment +** Cross-encoder helps differentiate between positive and negative examples +** Combines strengths of both model types + +[discrete] +[[ml-nlp-rerank-arch-data]] +=== Training data + +The training data consists of: + +* Open domain Question-Answering datasets +* Natural document pairs (like article headings and summaries) +* 180,000 synthetic query-passage pairs with varying relevance +* Total of approximately 3 million queries + +The data preparation process includes: + +* Basic cleaning and fuzzy deduplication +* Multi-stage prompting for diverse topics (on the synthetic portion of the training data only) +* Varied query types: +** Keyword search +** Exact phrase matching +** Short and long natural language questions + +[discrete] +[[ml-nlp-rerank-arch-sampling]] +=== Negative sampling + +The model uses an advanced sampling strategy to ensure high-quality rankings: + +* Samples from top 128 documents per query using multiple retrieval methods +* Uses five negative samples per query - more than typical approaches +* Applies probability distribution shaped by document scores for sampling + +* Deep sampling benefits: +** Improves model robustness across different retrieval depths +** Enhances score calibration +** Provides better handling of document diversity + +[discrete] +[[ml-nlp-rerank-arch-optimization]] +=== Training optimization + +The training process incorporates several key optimizations: + +Uses cross-entropy loss function to: + +* Model relevance as probability distribution +* Learn relationships between all document scores +* Fit scores through maximum likelihood estimation + +Implemented parameter averaging along optimization trajectory: + +* Eliminates need for traditional learning rate scheduling and provides improvement in the final model quality + +[discrete] +[[ml-nlp-rerank-performance]] +== Performance + +Elastic Rerank shows significant improvements in search quality across a wide range of retrieval tasks. + +[discrete] +[[ml-nlp-rerank-performance-overview]] +=== Overview + +* Average 40% improvement in ranking quality when reranking BM25 results +* 184M parameter model matches performance of 2B parameter alternatives +* Evaluated across 21 different datasets using the BEIR benchmark suite + +[discrete] +[[ml-nlp-rerank-performance-benchmarks]] +=== Key benchmark results + +* Natural Questions: 90% improvement +* MS MARCO: 85% improvement +* Climate-FEVER: 80% improvement +* FiQA-2018: 76% improvement + +For detailed benchmark information, including complete dataset results and methodology, refer to the https://www.elastic.co/search-labs/blog/elastic-semantic-reranker-part-2[Introducing Elastic Rerank blog]. + +// [discrete] +// [[ml-nlp-rerank-benchmarks-hw]] +// === Hardware benchmarks +// Note: these are more for GA timeframe + +[discrete] +[[ml-nlp-rerank-resources]] +== Further resources + +*Documentation*: + +* {ref}/semantic-reranking.html#semantic-reranking-in-es[Semantic re-ranking in {es} overview] +* {ref}/infer-service-elasticsearch.html#inference-example-elastic-reranker[Inference API example] + +*Blogs*: + +* https://www.elastic.co/search-labs/blog/elastic-semantic-reranker-part-1[Part 1] +* https://www.elastic.co/search-labs/blog/elastic-semantic-reranker-part-2[Part 2] +* https://www.elastic.co/search-labs/blog/elastic-semantic-reranker-part-3[Part 3] + +*Python notebooks*: + +* https://github.com/elastic/elasticsearch-labs/blob/main/notebooks/search/12-semantic-reranking-elastic-rerank.ipynb[End-to-end example using Elastic Rerank in Python] diff --git a/docs/en/stack/ml/nlp/ml-nlp-elser.asciidoc b/docs/en/stack/ml/nlp/ml-nlp-elser.asciidoc index caedfdf6c..ddb000c42 100644 --- a/docs/en/stack/ml/nlp/ml-nlp-elser.asciidoc +++ b/docs/en/stack/ml/nlp/ml-nlp-elser.asciidoc @@ -9,8 +9,6 @@ :frontmatter-tags-content-type: [how-to] :frontmatter-tags-user-goals: [analyze] -experimental[] - Elastic Learned Sparse EncodeR - or ELSER - is a retrieval model trained by Elastic that enables you to perform {ref}/semantic-search-elser.html[semantic search] to retrieve more relevant @@ -20,13 +18,36 @@ meaning and user intent, rather than exact keyword matches. ELSER is an out-of-domain model which means it does not require fine-tuning on your own data, making it adaptable for various use cases out of the box. +This model is recommended for English language documents and queries. If you +want to perform semantic search on non-English language documents, use the +<> model. + +IMPORTANT: While ELSER V2 is generally available, ELSER V1 is in experimental:[] +and will remain in technical preview. + + +[discrete] +[[elser-tokens]] +== Tokens - not synonyms + ELSER expands the indexed and searched passages into collections of terms that are learned to co-occur frequently within a diverse set of training data. The terms that the text is expanded into by the model _are not_ synonyms for the -search terms; they are learned associations. These expanded terms are weighted -as some of them are more significant than others. Then the {es} -{ref}/rank-features.html[rank features field type] is used to store the terms -and weights at index time, and to search against later. +search terms; they are learned associations capturing relevance. These expanded +terms are weighted as some of them are more significant than others. Then the +{es} {ref}/sparse-vector.html[sparse vector] +(or {ref}/rank-features.html[rank features]) field type is used to store the +terms and weights at index time, and to search against later. + +This approach provides a more understandable search experience compared to +vector embeddings. However, attempting to directly interpret the tokens and +weights can be misleading, as the expansion essentially results in a vector in a +very high-dimensional space. Consequently, certain tokens, especially those with +low weight, contain information that is intertwined with other low-weight tokens +in the representation. In this regard, they function similarly to a dense vector +representation, making it challenging to separate their individual +contributions. This complexity can potentially lead to misinterpretations if not +carefully considered during analysis. [discrete] @@ -45,100 +66,127 @@ more allocations or more threads per allocation, which requires bigger ML nodes. Autoscaling provides bigger nodes when required. If autoscaling is turned off, you must provide suitably sized nodes yourself. -[discrete] -[[elser-benchamrks]] -== Benchmarks - -The following sections provide information about how ELSER performs on different -hardwares and compares the model performance to {es} BM25 and other strong -baselines such as Splade or OpenAI. +Enabling trained model autoscaling for your ELSER deployment is recommended. +Refer to <> to learn more. [discrete] -[[elser-hw-benchamrks]] -=== Hardware benchmarks - -Two data sets were utilized to evaluate the performance of ELSER in different -hardware configurations: `msmarco-long-light` and `arguana`. +[[elser-v2]] +== ELSER v2 -|============================================================================================================== -| **Data set** ^| **Data set size** ^| **Average count of tokens / query** ^| **Average count of tokens / document** -| `msmarco-long-light` ^| 37367 documents ^| 9 ^| 1640 -| `arguana` ^| 8674 documents ^| 238 ^| 202 -|============================================================================================================== +Compared to the initial version of the model, ELSER v2 offers improved retrieval +accuracy and more efficient indexing. This enhancement is attributed to the +extension of the training data set, which includes high-quality question and +answer pairs and the improved FLOPS regularizer which reduces the cost of +computing the similarity between a query and a document. -The `msmarco-long-light` data set contains long documents with an average of -over 512 tokens, which provides insights into the performance implications -of indexing and {infer} time for long documents. This is a subset of the -"msmarco" dataset specifically designed for document retrieval (it shouldn't be -confused with the "msmarco" dataset used for passage retrieval, which primarily -consists of shorter spans of text). +ELSER v2 has two versions: one cross-platform version which runs on any hardware +and one version which is optimized for Intel® silicon. The +**Model Management** > **Trained Models** page shows you which version of ELSER +v2 is recommended to deploy based on your cluster's hardware. However, the +recommended way to use ELSER is through the +{ref}/infer-service-elser.html[{infer} API] as a service which makes it easier +to download and deploy the model and you don't need to select from different +versions. -The `arguana` data set is a https://github.com/beir-cellar/beir[BEIR] data set. -It consists of long queries with an average of 200 tokens per query. It can -represent an upper limit for query slowness. - -The table below present benchmarking results for ELSER using various hardware -configurations. - -|================================================================================================================================================================================== -| 3+^| `msmarco-long-light` 3+^| `arguana` | -| ^.^| inference ^.^| indexing ^.^| query latency ^.^| inference ^.^| indexing ^.^| query latency | -| **ML node 4GB - 2 vCPUs (1 allocation * 1 thread)** ^.^| 581 ms/call ^.^| 1.7 doc/sec ^.^| 713 ms/query ^.^| 1200 ms/call ^.^| 0.8 doc/sec ^.^| 169 ms/query | -| **ML node 16GB - 8 vCPUs (7 allocation * 1 thread)** ^.^| 568 ms/call ^.^| 12 doc/sec ^.^| 689 ms/query ^.^| 1280 ms/call ^.^| 5.4 doc/sec ^.^| 159 ms/query | -| **ML node 16GB - 8 vCPUs (1 allocation * 8 thread)** ^.^| 102 ms/call ^.^| 9.7 doc/sec ^.^| 164 ms/query ^.^| 220 ms/call ^.^| 4.5 doc/sec ^.^| 40 ms/query | -| **ML node 32 GB - 16 vCPUs (15 allocation * 1 thread)** ^.^| 565 ms/call ^.^| 25.2 doc/sec ^.^| 608 ms/query ^.^| 1260 ms/call ^.^| 11.4 doc/sec ^.^| 138 ms/query | -|================================================================================================================================================================================== +If you want to learn more about the ELSER V2 improvements, refer to +https://www.elastic.co/search-labs/blog/introducing-elser-v2-part-1[this blog post]. [discrete] -[[elser-qualitative-benchmarks]] -=== Qualitative benchmarks +[[upgrade-elser-v2]] +=== Upgrading to ELSER v2 -The metric that is used to evaluate ELSER's ranking ability is the Normalized -Discounted Cumulative Gain (NDCG) which can handle multiple relevant documents -and fine-grained document ratings. The metric is applied to a fixed-sized list -of retrieved documents which, in this case, is the top 10 documents (NDCG@10). +ELSER v2 is not backward compatible. If you indexed your data with ELSER v1, you +need to reindex it with an ingest pipeline referencing ELSER v2 to be able to +use v2 for search. This {ref}/semantic-search-elser.html[tutorial] shows you how +to create an ingest pipeline with an {infer} processor that uses ELSER v2, and +how to reindex your data through the pipeline. -The table below shows the performance of ELSER compared to {es} BM25 with an -English analyzer broken down by the 12 data sets used for the evaluation. ELSER -has 10 wins, 1 draw, 1 loss and an average improvement in NDCG@10 of 17%. +Additionally, the `elasticearch-labs` GitHub repository contains an interactive +https://github.com/elastic/elasticsearch-labs/blob/main/notebooks/model-upgrades/upgrading-index-to-use-elser.ipynb[Python notebook] +that walks through upgrading an index to ELSER V2. -image::images/ml-nlp-elser-ndcg10-beir.png[alt="ELSER benchmarks",align="center"] -_NDCG@10 for BEIR data sets for BM25 and ELSER - higher values are better)_ -The following table compares the average performance of ELSER to some other -strong baselines. The OpenAI results are separated out because they use a -different subset of the BEIR suite. +[discrete] +[[download-deploy-elser]] +== Download and deploy ELSER -image::images/ml-nlp-elser-average-ndcg.png[alt="ELSER average performance compared to other baselines",align="center"] -_Average NDCG@10 for BEIR data sets vs. various high quality baselines (higher_ -_is better). OpenAI chose a different subset, ELSER results on this set_ -_reported separately._ +The easiest and recommended way to download and deploy ELSER is to use the {ref}/inference-apis.html[{infer} API]. -To read more about the evaluation details, refer to -https://www.elastic.co/blog/may-2023-launch-information-retrieval-elasticsearch-ai-model[this blog post]. +1. In {kib}, navigate to the **Dev Console**. +2. Create an {infer} endpoint with the ELSER service by running the following API request: ++ +-- +[source,console] +---------------------------------- +PUT _inference/sparse_embedding/my-elser-model +{ + "service": "elasticsearch", + "service_settings": { + "adaptive_allocations": { + "enabled": true, + "min_number_of_allocations": 1, + "max_number_of_allocations": 10 + }, + "num_threads": 1, + "model_id": ".elser_model_2_linux-x86_64" + } +} +---------------------------------- +-- +The API request automatically initiates the model download and then deploy the model. +This example uses <> through adaptive allocation. +Refer to the {ref}/infer-service-elser.html[ELSER {infer} service documentation] to learn more about the available settings. -[discrete] -[[download-deploy-elser]] -== Download and deploy ELSER +After you created the ELSER {infer} endpoint, it's ready to be used for semantic search. +The easiest way to perform semantic search in the {stack} is to {ref}/semantic-search-semantic-text.html[follow the `semantic_text` workflow]. -You can download and deploy ELSER either from **{ml-app}** > **Trained Models**, -from **{ents}** > **Indices**, or by using the Dev Console. +[discrete] +[[alternative-download-deploy]] +=== Alternative methods to download and deploy ELSER + +You can also download and deploy ELSER either from **{ml-app}** > **Trained Models**, from **Search** > **Indices**, or by using the trained models API in Dev Console. + +[NOTE] +==== +* For most cases, the preferred version is the **Intel and Linux optimized** +model, it is recommended to download and deploy that version. +* You can deploy the model multiple times by assigning a unique deployment ID +when starting the deployment. It enables you to have dedicated deployments for +different purposes, such as search and ingest. By doing so, you ensure that the +search speed remains unaffected by ingest workloads, and vice versa. Having +separate deployments for search and ingest mitigates performance issues +resulting from interactions between the two, which can be hard to diagnose. +==== + +.Using the Trained Models page +[%collapsible%closed] +===== [discrete] [[trained-model]] -=== Using the Trained Models page +==== Using the Trained Models page 1. In {kib}, navigate to **{ml-app}** > **Trained Models**. ELSER can be found -in the list of trained models. -2. Click the **Download model** button under **Actions**. You can check the -download status on the **Notifications** page. +in the list of trained models. There are two versions available: one portable +version which runs on any hardware and one version which is optimized for Intel® +silicon. You can see which model is recommended to use based on your hardware +configuration. +2. Click the **Add trained model** button. Select the ELSER model version you +want to use in the opening modal window. The model that is recommended for you +based on your hardware configuration is highlighted. Click **Download**. You can +check the download status on the **Notifications** page. + -- [role="screenshot"] -image::images/ml-nlp-elser-download.png[alt="Downloading ELSER",align="center"] +image::images/ml-nlp-elser-v2-download.png[alt="Downloading ELSER",align="center"] +-- ++ +-- +Alternatively, click the **Download model** button under **Actions** in the +trained model list. -- 3. After the download is finished, start the deployment by clicking the **Start deployment** button. @@ -147,33 +195,50 @@ allocations and threads per allocation values. + -- [role="screenshot"] -image::images/ml-nlp-deployment-id-elser.png[alt="Deploying ELSER",align="center"] +image::images/ml-nlp-deployment-id-elser-v2.png[alt="Deploying ELSER",align="center"] -- -5. Click Start. - +5. Click **Start**. +===== +.Using the search indices UI +[%collapsible%closed] +===== [discrete] -[[enterprise-search]] -=== Using the Indices page in {ents} +[[elasticsearch]] +==== Using the search indices UI -You can also {enterprise-search-ref}/elser-text-expansion.html#deploy-elser[download and deploy ELSER to an {infer} pipeline] directly from the -{ents} app. +Alternatively, you can download and deploy ELSER to an {infer} pipeline using +the search indices UI. -1. In {kib}, navigate to **{ents}** > **Indices**. +1. In {kib}, navigate to **Search** > **Indices**. 2. Select the index from the list that has an {infer} pipeline in which you want to use ELSER. 3. Navigate to the **Pipelines** tab. 4. Under **{ml-app} {infer-cap} Pipelines**, click the **Deploy** button to begin downloading the ELSER model. This may take a few minutes depending on your -network. Once it's downloaded, click the **Start single-threaded** button to +network. ++ +-- +[role="screenshot"] +image::images/ml-nlp-deploy-elser-v2-es.png[alt="Deploying ELSER in Elasticsearch",align="center"] +-- +5. Once the model is downloaded, click the **Start single-threaded** button to start the model with basic configuration or select the **Fine-tune performance** option to navigate to the **Trained Models** page where you can configure the model deployment. ++ +-- +[role="screenshot"] +image::images/ml-nlp-start-elser-v2-es.png[alt="Start ELSER in Elasticsearch",align="center"] +-- +===== - +.Using the trained models API in Dev Console +[%collapsible%closed] +===== [discrete] [[dev-console]] -=== Using the Dev Console +==== Using the trained models API in Dev Console 1. In {kib}, navigate to the **Dev Console**. 2. Create the ELSER model configuration by running the following API call: @@ -181,7 +246,7 @@ model deployment. -- [source,console] ---------------------------------- -PUT _ml/trained_models/.elser_model_1 +PUT _ml/trained_models/.elser_model_2 { "input": { "field_names": ["text_field"] @@ -199,14 +264,12 @@ with a delpoyment ID: -- [source,console] ---------------------------------- -POST _ml/trained_models/.elser_model_1/deployment/_start?deployment_id=for_search +POST _ml/trained_models/.elser_model_2/deployment/_start?deployment_id=for_search ---------------------------------- You can deploy the model multiple times with different deployment IDs. -- - -After the deployment is complete, ELSER is ready to use either in an ingest -pipeline or in a `text_expansion` query to perform semantic search. +===== [discrete] @@ -220,12 +283,23 @@ options: * put the model artifacts into a directory inside the config directory on all {ref}/modules-node.html#master-node[master-eligible nodes]. -You need the following files in your system: +[discrete] +[[elser-model-artifacts]] +=== Model artifact files + +For the cross-platform verison, you need the following files in your system: ``` -https://ml-models.elastic.co/elser_model_1.metadata.json -https://ml-models.elastic.co/elser_model_1.pt -https://ml-models.elastic.co/elser_model_1.vocab.json +https://ml-models.elastic.co/elser_model_2.metadata.json +https://ml-models.elastic.co/elser_model_2.pt +https://ml-models.elastic.co/elser_model_2.vocab.json +``` + +For the optimized version, you need the following files in your system: +``` +https://ml-models.elastic.co/elser_model_2_linux-x86_64.metadata.json +https://ml-models.elastic.co/elser_model_2_linux-x86_64.pt +https://ml-models.elastic.co/elser_model_2_linux-x86_64.vocab.json ``` @@ -238,7 +312,7 @@ supports passwordless HTTP servers. You can use any HTTP service to deploy ELSER. This example uses the official Nginx Docker image to set a new HTTP download service up. -. Download the model artifact files from https://ml-models.elastic.co/. +. Download the <>. . Put the files into a subdirectory of your choice. . Run the following commands: + @@ -262,7 +336,7 @@ browser: + -- ``` -http://{IP_ADDRESS_OR_HOSTNAME}:8080/elser_model_1.metadata.json +http://{IP_ADDRESS_OR_HOSTNAME}:8080/elser_model_2.metadata.json ``` If Nginx runs properly, you see the content of the metdata file of the model. @@ -281,7 +355,17 @@ master-eligible nodes can reach the server you specify. -- . Repeat step 5 on all master-eligible nodes. . {ref}/restart-cluster.html#restart-cluster-rolling[Restart] the -master-eligible nodes one by one. +master-eligible nodes one by one. +. Navigate to the **Trained Models** page from the main menu, or use the {kibana-ref}/kibana-concepts-analysts.html#_finding_your_apps_and_objects[global search field] in {kib}. ELSER can be found in the +list of trained models. +. Click the **Add trained model** button, select the ELSER model version you +downloaded in step 1 and want to deploy, and click **Download**. The selected +model will be downloaded from the HTTP/HTTPS server you configured. +. After the download is finished, start the deployment by clicking the +**Start deployment** button. +. Provide a deployment ID, select the priority, and set the number of +allocations and threads per allocation values. +. Click **Start**. The HTTP server is only required for downloading the model. After the download has finished, you can stop and delete the service. You can stop the Docker image @@ -298,7 +382,7 @@ docker stop ml-models For a file-based access, follow these steps: -. Download the model artifact files from https://ml-models.elastic.co/. +. Download the <>. . Put the files into a `models` subdirectory inside the `config` directory of your Elasticsearch deployment. . Point your Elasticsearch deployment to the model directory by adding the @@ -306,19 +390,29 @@ following line to the `config/elasticsearch.yml` file: + -- ``` -xpack.ml.model_repository: file://${path.home}/config/models/` +xpack.ml.model_repository: file://${path.home}/config/models/ ``` -- . Repeat step 2 and step 3 on all master-eligible nodes. . {ref}/restart-cluster.html#restart-cluster-rolling[Restart] the master-eligible nodes one by one. +. Navigate to the **Trained Models** page from the main menu, or use the {kibana-ref}/kibana-concepts-analysts.html#_finding_your_apps_and_objects[global search field] in {kib}. ELSER can be found in the +list of trained models. +. Click the **Add trained model** button, select the ELSER model version you +downloaded in step 1 and want to deploy and click **Download**. The selected +model will be downloaded from the model directory where you put in step 2. +. After the download is finished, start the deployment by clicking the +**Start deployment** button. +. Provide a deployment ID, select the priority, and set the number of +allocations and threads per allocation values. +. Click **Start**. [discrete] == Testing ELSER You can test the deployed model in {kib}. Navigate to **Model Management** > -**Trained Models**, locate the deployed ELSER model in the list of trained +**Trained Models** from the main menu, or use the {kibana-ref}/kibana-concepts-analysts.html#_finding_your_apps_and_objects[global search field] in {kib}. Locate the deployed ELSER model in the list of trained models, then select **Test model** from the Actions menu. You can use data from an existing index to test the model. Select the index, @@ -328,15 +422,159 @@ the documents. The results contain a list of ten random values for the selected field along with a score showing how relevant each document is to the query. The higher the -score, the more relevant the document is. +score, the more relevant the document is. You can reload example documents by +clicking **Reload examples**. [role="screenshot"] -image::images/ml-nlp-elser-test.png[alt="Testing ELSER",align="center"] +image::images/ml-nlp-elser-v2-test.png[alt="Testing ELSER",align="center"] + + +[discrete] +[[performance]] +== Performance considerations + +* ELSER works best on small-to-medium sized fields that contain natural language. +For connector or web crawler use cases, this aligns best with fields like _title_, _description_, _summary_, or _abstract_. +As ELSER encodes the first 512 tokens of a field, it may not provide as relevant of results for large fields. +For example, `body_content` on web crawler documents, or body fields resulting from extracting text from office documents with connectors. +For larger fields like these, consider "chunking" the content into multiple values, where each chunk can be under 512 tokens. +* Larger documents take longer at ingestion time, and {infer} time per document also increases the more fields in a document that need to be processed. +* The more fields your pipeline has to perform inference on, the longer it takes per document to ingest. + +To learn more about ELSER performance, refer to the <>. + + +[discrete] +[[pre-cleaning]] +== Pre-cleaning input text + +The quality of the input text significantly affects the quality of the embeddings. +To achieve the best results, it's recommended to clean the input text before generating embeddings. +The exact preprocessing you may need to do heavily depends on your text. +For example, if your text contains HTML tags, use the {ref}/htmlstrip-processor.html[HTML strip processor] in an ingest pipeline to remove unnecessary elements. +Always review and clean your input text before ingestion to eliminate any irrelevant entities that might affect the results. + + +[discrete] +[[elser-recommendations]] +== Recommendations for using ELSER + +To gain the biggest value out of ELSER trained models, consider to follow this list of recommendations. + +* If quick response time is important for your use case, keep {ml} resources available at all times by setting `min_allocations` to `1`. +* Setting `min_allocations` to `0` can save on costs for non-critical use cases or testing environments. +* Enabling <> through adaptive allocations or adaptive resources makes it possible for {es} to scale up or down the available resources of your ELSER deployment based on the load on the process. + +* Use dedicated, optimized ELSER {infer} endpoints for ingest and search use cases. +** When deploying a trained model in {kib}, you can select for which case you want to optimize your ELSER deployment. +** If you use the trained model or {infer} APIs and want to optimize your ELSER trained model deployment or {infer} endpoint for ingest, set the number of threads to `1` (`"num_threads": 1`). +** If you use the trained model or {infer} APIs and want to optimize your ELSER trained model deployment or {infer} endpoint for search, set the number of threads to greater than `1`. [discrete] [[further-readings]] == Further reading +* {ref}/semantic-search-semantic-text.html[Perform semantic search with `semantic_text` using the ELSER endpoint] * {ref}/semantic-search-elser.html[Perform semantic search with ELSER] -* https://www.elastic.co/blog/may-2023-launch-information-retrieval-elasticsearch-ai-model[Improving information retrieval in the Elastic Stack: Introducing Elastic Learned Sparse Encoder, our new retrieval model] + + +[discrete] +[[elser-benchmarks]] +== Benchmark information + +IMPORTANT: The recommended way to use ELSER is through the {ref}/infer-service-elser.html[{infer} API] as a service. + +The following sections provide information about how ELSER performs on different +hardwares and compares the model performance to {es} BM25 and other strong +baselines. + + +[discrete] +[[version-overview]] +=== Version overview + +ELSER V2 has a **optimized** version that is designed to run only on Linux with +an x86-64 CPU architecture and a **cross-platform** version that can be run on +any platform. + + +[discrete] +[[version-overview-v2]] +==== ELSER V2 + +Besides the performance improvements, the biggest change in ELSER V2 is the +introduction of the first platform specific ELSER model - that is, a model +optimized to run only on Linux with an x86-64 CPU architecture. The optimized +model is designed to work best on newer Intel CPUs, but it works on AMD CPUs as +well. It is recommended to use the new optimized Linux-x86-64 model for all new +users of ELSER as it is significantly faster than the cross-platform model which +can be run on any platform. ELSER V2 produces significantly higher quality +embeddings than ELSER V1. Regardless of which ELSER V2 model you use (optimized +or cross-platform), the particular embeddings produced are the same. + + +[discrete] +[[elser-qualitative-benchmarks]] +=== Qualitative benchmarks + +The metric that is used to evaluate ELSER's ranking ability is the Normalized +Discounted Cumulative Gain (NDCG) which can handle multiple relevant documents +and fine-grained document ratings. The metric is applied to a fixed-sized list +of retrieved documents which, in this case, is the top 10 documents (NDCG@10). + +The table below shows the performance of ELSER V2 compared to BM 25. ELSER V2 +has 10 wins, 1 draw, 1 loss and an average improvement in NDCG@10 of 18%. + +image::images/ml-nlp-bm25-elser-v2.png[alt="ELSER V2 benchmarks compared to BM25",align="center"] +_NDCG@10 for BEIR data sets for BM25 and ELSER V2 - higher values are better)_ + + +[discrete] +[[elser-hw-benchmarks]] +=== Hardware benchmarks + +IMPORTANT: While the goal is to create a model that is as performant as +possible, retrieval accuracy always take precedence over speed, this is one of +the design principles of ELSER. Consult with the tables below to learn more +about the expected model performance. The values refer to operations performed +on two data sets and different hardware configurations. Your data set has an +impact on the model performance. Run tests on your own data to have a more +realistic view on the model performance for your use case. + + +[discrete] +==== ELSER V2 + +Overall the optimized V2 model ingested at a max rate of 26 docs/s, compared +with the ELSER V1 max rate of 14 docs/s from the ELSER V1 benchamrk, resulting +in a 90% increase in throughput. + +The performance of virtual cores (that is, when the number of allocations is +greater than half of the vCPUs) has increased. Previously, the increase in +performance between 8 and 16 allocations was around 7%. It has increased to 17% +(ELSER V1 on 8.11) and 20% (for ELSER V2 optimized). These tests were performed +on a 16vCPU machine, with all documents containing exactly 256 tokens. + +IMPORTANT: The length of the documents in your particular dataset will have a +significant impact on your throughput numbers. + +Refer to +https://www.elastic.co/search-labs/blog/introducing-elser-v2-part-1[this blog post] +to learn more about ELSER V2 improved performance. + +image::images/ml-nlp-elser-bm-summary.png[alt="Summary of ELSER V1 and V2 benchmark reports",align="center"] + +**The optimized model** results show a nearly linear growth up until 8 +allocations, after which performance improvements become smaller. In this case, +the performance at 8 allocations was 22 docs/s, while the performance of 16 +allocations was 26 docs/s, indicating a 20% performance increase due to virtual +cores. + +image::images/ml-nlp-elser-v2-opt-bm-results.png[alt="ELSER V2 optimized benchmarks",align="center"] + +**The cross-platform** model performance of 8 and 16 allocations are +respectively 14 docs/s and 16 docs/s, indicating a performance improvement due +to virtual cores of 12%. + +image::images/ml-nlp-elser-v2-cp-bm-results.png[alt="ELSER V2 cross-platform benchmarks",align="center"] diff --git a/docs/en/stack/ml/nlp/ml-nlp-examples.asciidoc b/docs/en/stack/ml/nlp/ml-nlp-examples.asciidoc index 2d10177ac..f7b07b861 100644 --- a/docs/en/stack/ml/nlp/ml-nlp-examples.asciidoc +++ b/docs/en/stack/ml/nlp/ml-nlp-examples.asciidoc @@ -4,5 +4,6 @@ The following pages contain end-to-end examples of how to use the different {nlp} tasks in the {stack}. +* https://colab.research.google.com/github/elastic/elasticsearch-labs/blob/main/notebooks/search/04-multilingual.ipynb[Multilingual semantic search - notebook example] * <> -* <> \ No newline at end of file +* <> diff --git a/docs/en/stack/ml/nlp/ml-nlp-inference.asciidoc b/docs/en/stack/ml/nlp/ml-nlp-inference.asciidoc index 09194c2c7..71fd063ee 100644 --- a/docs/en/stack/ml/nlp/ml-nlp-inference.asciidoc +++ b/docs/en/stack/ml/nlp/ml-nlp-inference.asciidoc @@ -18,7 +18,7 @@ can use it to perform {nlp} tasks in ingest pipelines. == Add an {infer} processor to an ingest pipeline In {kib}, you can create and edit pipelines in **{stack-manage-app}** > -**Ingest Pipelines**. +**Ingest Pipelines**. To open **Ingest Pipelines**, find **{stack-manage-app}** in the main menu, or use the {kibana-ref}/kibana-concepts-analysts.html#_finding_your_apps_and_objects[global search field]. [role="screenshot"] image::images/ml-nlp-pipeline-lang.png[Creating a pipeline in the Stack Management app,align="center"] diff --git a/docs/en/stack/ml/nlp/ml-nlp-lang-ident.asciidoc b/docs/en/stack/ml/nlp/ml-nlp-lang-ident.asciidoc index abc3604f3..c6650e695 100644 --- a/docs/en/stack/ml/nlp/ml-nlp-lang-ident.asciidoc +++ b/docs/en/stack/ml/nlp/ml-nlp-lang-ident.asciidoc @@ -1,6 +1,5 @@ -[discrete] [[ml-nlp-lang-ident]] -== {lang-ident-cap} += {lang-ident-cap} {lang-ident-cap} enables you to determine the language of text. @@ -25,9 +24,10 @@ language traditionally uses. These languages are marked in the supported languages table (see below) with the `Latn` subtag. {lang-ident-cap} supports Unicode input. + [discrete] [[ml-lang-ident-supported-languages]] -=== Supported languages +== Supported languages The table below contains the ISO codes and the English names of the languages that {lang-ident} supports. If a language has a 2-letter `ISO 639-1` code, the @@ -83,8 +83,119 @@ script. //// + +[discrete] +[[ml-lang-ident-example]] +== Example of {lang-ident} + +In the following example, we feed the {lang-ident} trained model a short +Hungarian text that contains diacritics and a couple of English words. The +model identifies the text correctly as Hungarian with high probability. + +[source,js] +---------------------------------- +POST _ingest/pipeline/_simulate +{ + "pipeline":{ + "processors":[ + { + "inference":{ + "model_id":"lang_ident_model_1", <1> + "inference_config":{ + "classification":{ + "num_top_classes":5 <2> + } + }, + "field_map":{ + } + } + } + ] + }, + "docs":[ + { + "_source":{ <3> + "text":"Sziasztok! Ez egy rövid magyar szöveg. Nézzük, vajon sikerül-e azonosítania a language identification funkciónak? Annak ellenére is sikerülni fog, hogy a szöveg két angol szót is tartalmaz." + } + } + ] +} +---------------------------------- +//NOTCONSOLE + +<1> ID of the {lang-ident} trained model. +<2> Specifies the number of languages to report by descending order of +probability. +<3> The source object that contains the text to identify. + + +In the example above, the `num_top_classes` value indicates that only the top +five languages (that is to say, the ones with the highest probability) are +reported. + +The request returns the following response: + +[source,js] +---------------------------------- +{ + "docs" : [ + { + "doc" : { + "_index" : "_index", + "_type" : "_doc", + "_id" : "_id", + "_source" : { + "text" : "Sziasztok! Ez egy rövid magyar szöveg. Nézzük, vajon sikerül-e azonosítania a language identification funkciónak? Annak ellenére is sikerülni fog, hogy a szöveg két angol szót is tartalmaz.", + "ml" : { + "inference" : { + "top_classes" : [ <1> + { + "class_name" : "hu", + "class_probability" : 0.9999936063740517, + "class_score" : 0.9999936063740517 + }, + { + "class_name" : "lv", + "class_probability" : 2.5020248433413966E-6, + "class_score" : 2.5020248433413966E-6 + }, + { + "class_name" : "is", + "class_probability" : 1.0150420723037688E-6, + "class_score" : 1.0150420723037688E-6 + }, + { + "class_name" : "ga", + "class_probability" : 6.67935962773335E-7, + "class_score" : 6.67935962773335E-7 + }, + { + "class_name" : "tr", + "class_probability" : 5.591166324774555E-7, + "class_score" : 5.591166324774555E-7 + } + ], + "predicted_value" : "hu", <2> + "model_id" : "lang_ident_model_1" + } + } + }, + "_ingest" : { + "timestamp" : "2020-01-22T14:25:14.644912Z" + } + } + } + ] +} +---------------------------------- +//NOTCONSOLE + +<1> Contains scores for the most probable languages. +<2> The ISO identifier of the language with the highest probability. + + [discrete] [[ml-lang-ident-readings]] -=== Further reading +== Further reading * {blog-ref}multilingual-search-using-language-identification-in-elasticsearch[Multilingual search using {lang-ident} in {es}] diff --git a/docs/en/stack/ml/nlp/ml-nlp-limitations.asciidoc b/docs/en/stack/ml/nlp/ml-nlp-limitations.asciidoc index 1ae0856e9..e505bb63b 100644 --- a/docs/en/stack/ml/nlp/ml-nlp-limitations.asciidoc +++ b/docs/en/stack/ml/nlp/ml-nlp-limitations.asciidoc @@ -9,12 +9,18 @@ The following limitations and known problems apply to the {version} release of the Elastic {nlp} trained models feature. +[discrete] +[[ml-nlp-large-documents-limit-10k-10mb]] +== Document size limitations when using `semantic_text` fields + +When using semantic text to ingest documents, chunking takes place automatically. The number of chunks is limited by the {ref}/mapping-settings-limit.html[`index.mapping.nested_objects.limit`] cluster setting, which defaults to 10k. Documents that are too large will cause errors during ingestion. To avoid this issue, please split your documents into roughly 1MB parts before ingestion. + [discrete] [[ml-nlp-elser-v1-limit-512]] -== ELSER v1 semantic search is limited to 512 tokens per field that inference is applied to +== ELSER semantic search is limited to 512 tokens per field that inference is applied to -When you use ELSER v1 for semantic search, only the first 512 extracted tokens -from each field of the ingested documents that ELSER is applied to are taken -into account for the search process. If your data set contains long documents, -divide them into smaller segments before ingestion if you need the full text to -be searchable. +When you use ELSER for semantic search, only the first 512 extracted tokens from +each field of the ingested documents that ELSER is applied to are taken into +account for the search process. If your data set contains long documents, divide +them into smaller segments before ingestion if you need the full text to be +searchable. diff --git a/docs/en/stack/ml/nlp/ml-nlp-model-ref.asciidoc b/docs/en/stack/ml/nlp/ml-nlp-model-ref.asciidoc index 1fd312198..dd664147b 100644 --- a/docs/en/stack/ml/nlp/ml-nlp-model-ref.asciidoc +++ b/docs/en/stack/ml/nlp/ml-nlp-model-ref.asciidoc @@ -26,6 +26,7 @@ The current list of supported architectures is: * BERT * BART * DPR bi-encoders +* DeBERTa * DistilBERT * ELECTRA * MobileBERT @@ -77,6 +78,16 @@ purposes and to get started with the Elastic {nlp} features. * https://huggingface.co/deepset/electra-base-squad2[Electra base squad2] * https://huggingface.co/deepset/tinyroberta-squad2[TinyRoBERTa squad2] +[discrete] +[[ml-nlp-model-ref-sparse-embedding]] +== Third party sparse embedding models + +Sparse embedding models should be configured with the `text_expansion` task type. + +* https://huggingface.co/naver/splade-v3-distilbert[SPLADE-v3-DistilBERT] +* https://huggingface.co/aken12/splade-japanese-v3[aken12/splade-japanese-v3] +* https://huggingface.co/hotchpotch/japanese-splade-v2[hotchpotch/japanese-splade-v2] + [discrete] [[ml-nlp-model-ref-text-embedding]] @@ -148,8 +159,12 @@ Using `DPREncoderWrapper`: [discrete] [[ml-nlp-model-ref-text-similarity]] == Third party text similarity models + +You can use these text similarity models for {ref}/semantic-reranking.html#semantic-reranking-in-es[semantic re-ranking]. + * https://huggingface.co/cross-encoder/ms-marco-TinyBERT-L-2-v2[ms marco TinyBERT L2 v2] * https://huggingface.co/cross-encoder/ms-marco-MiniLM-L-6-v2[ms marco MiniLM L6 v2] +* https://huggingface.co/BAAI/bge-reranker-base[BAAI/bge-reranker-base] [discrete] [[ml-nlp-model-ref-zero-shot]] diff --git a/docs/en/stack/ml/nlp/ml-nlp-ner-example.asciidoc b/docs/en/stack/ml/nlp/ml-nlp-ner-example.asciidoc index f021297eb..fd20f60c5 100644 --- a/docs/en/stack/ml/nlp/ml-nlp-ner-example.asciidoc +++ b/docs/en/stack/ml/nlp/ml-nlp-ner-example.asciidoc @@ -35,7 +35,7 @@ image: [source,shell] -------------------------------------------------- -docker run -it --rm elastic/eland \ +docker run -it --rm docker.elastic.co/eland/eland \ eland_import_hub_model \ --cloud-id $CLOUD_ID \ -u -p \ @@ -294,7 +294,7 @@ You can create a tag cloud to visualize your data processed by the {infer} pipeline. A tag cloud is a visualization that scales words by the frequency at which they occur. It is a handy tool for viewing the entities found in the data. -In {kib}, open **Stack management** > **{data-sources-cap}**, and create a new +In {kib}, open **Stack management** > **{data-sources-cap}** from the main menu, or use the {kibana-ref}/kibana-concepts-analysts.html#_finding_your_apps_and_objects[global search field], and create a new {data-source} from the `les-miserables-infer` index pattern. Open **Dashboard** and create a new dashboard. Select the diff --git a/docs/en/stack/ml/nlp/ml-nlp-overview.asciidoc b/docs/en/stack/ml/nlp/ml-nlp-overview.asciidoc index 28d2a9798..66fe6a3e9 100644 --- a/docs/en/stack/ml/nlp/ml-nlp-overview.asciidoc +++ b/docs/en/stack/ml/nlp/ml-nlp-overview.asciidoc @@ -4,6 +4,30 @@ {nlp-cap} (NLP) refers to the way in which we can use software to understand natural language in spoken word or written text. +[discrete] +[[nlp-elastic-stack]] +== NLP in the {stack} + +Elastic offers a wide range of possibilities to leverage natural language +processing. + +You can **integrate NLP models from different providers** such as Cohere, HuggingFace, or OpenAI and use them as a service through the {ref}/semantic-search-semantic-text.html[semantic_text] workflow. +You can also use <> (the retrieval model trained by Elastic) and <> in the same way. + +The {ref}/inference-apis.html[{infer} API] enables you to use the same services with a more complex workflow, for greater control over your configurations settings. +This {ref}/semantic-search-inference.html[tutorial] walks you through the process of using the various services with the {infer} API. + +You can **upload and manage NLP models** using the Eland client and the <>. +Find the <>. +Refer to <> to learn more about how to use {ml} models deployed in your cluster. + +You can **store embeddings in your {es} vector database** if you generate {ref}/dense-vector.html[dense vector] or {ref}/sparse-vector.html[sparse vector] model embeddings outside of {es}. + + +[discrete] +[[what-is-nlp]] +== What is NLP? + Classically, NLP was performed using linguistic rules, dictionaries, regular expressions, and {ml} for specific tasks such as automatic categorization or summarization of text. In recent years, however, deep learning techniques have @@ -24,11 +48,14 @@ which is an underlying native library for PyTorch. Trained models must be in a TorchScript representation for use with {stack} {ml} features. As in the cases of <> and -<>, after you deploy a model to your cluster, you -can use it to make predictions (also known as _{infer}_) against incoming +<>, after you deploy a model to your cluster, +you can use it to make predictions (also known as _{infer}_) against incoming data. You can perform the following NLP operations: * <> * <> - * <> + +To delve deeper into Elastic's {ml} research and development, explore the +https://www.elastic.co/search-labs/blog/categories/ml-research[ML research] +section within Search Labs. diff --git a/docs/en/stack/ml/nlp/ml-nlp-shared.asciidoc b/docs/en/stack/ml/nlp/ml-nlp-shared.asciidoc index da0f9a683..0568cda26 100644 --- a/docs/en/stack/ml/nlp/ml-nlp-shared.asciidoc +++ b/docs/en/stack/ml/nlp/ml-nlp-shared.asciidoc @@ -1,16 +1,13 @@ tag::nlp-eland-clone-docker-build[] -You can use the {eland-docs}[Eland client] to install the {nlp} model. Eland -commands can be run in Docker. First, you need to clone the Eland repository -then create a Docker image of Eland: +You can use the {eland-docs}[Eland client] to install the {nlp} model. Use the prebuilt +Docker image to run the Eland install model commands. Pull the latest image with: [source,shell] -------------------------------------------------- -git clone git@github.com:elastic/eland.git -cd eland -docker build -t elastic/eland . +docker pull docker.elastic.co/eland/eland -------------------------------------------------- -After the script finishes, your Eland Docker client is ready to use. +After the pull completes, your Eland Docker client is ready to use. end::nlp-eland-clone-docker-build[] tag::nlp-requirements[] diff --git a/docs/en/stack/ml/nlp/ml-nlp-text-emb-vector-search-example.asciidoc b/docs/en/stack/ml/nlp/ml-nlp-text-emb-vector-search-example.asciidoc index 191a4eb0c..6c8e63c48 100644 --- a/docs/en/stack/ml/nlp/ml-nlp-text-emb-vector-search-example.asciidoc +++ b/docs/en/stack/ml/nlp/ml-nlp-text-emb-vector-search-example.asciidoc @@ -19,6 +19,11 @@ consists of real questions from the Microsoft Bing search engine and human generated answers for them. The example works with a sample of this data set, uses a model to produce text embeddings, and then runs vector search on it. +You can find +https://github.com/elastic/elasticsearch-labs/blob/main/notebooks/integrations/hugging-face/loading-model-from-hugging-face.ipynb[this example as a Jupyter notebook] +using the Python client in the `elasticsearch-labs` repo. + + [discrete] [[ex-te-vs-requirements]] == Requirements @@ -43,7 +48,7 @@ image: [source,shell] -------------------------------------------------- -docker run -it --rm elastic/eland \ +docker run -it --rm docker.elastic.co/eland/eland \ eland_import_hub_model \ --cloud-id $CLOUD_ID \ -u -p \ @@ -210,9 +215,7 @@ PUT collection-with-embeddings "properties": { "text_embedding.predicted_value": { "type": "dense_vector", - "dims": 384, - "index": true, - "similarity": "cosine" + "dims": 384 }, "text": { "type": "text" diff --git a/docs/en/stack/ml/nlp/ml-nlp.asciidoc b/docs/en/stack/ml/nlp/ml-nlp.asciidoc index 18fe43747..92f161413 100644 --- a/docs/en/stack/ml/nlp/ml-nlp.asciidoc +++ b/docs/en/stack/ml/nlp/ml-nlp.asciidoc @@ -14,6 +14,7 @@ predictions. * <> * <> +* <> * <> * <> * <> diff --git a/docs/en/stack/ml/redirects.asciidoc b/docs/en/stack/ml/redirects.asciidoc index 977f0ba8e..6c63e76ca 100644 --- a/docs/en/stack/ml/redirects.asciidoc +++ b/docs/en/stack/ml/redirects.asciidoc @@ -13,21 +13,6 @@ This page has moved. See <>. This page has moved. See <>. -[role="exclude",id="ml-configuring-pop"] -=== Performing population analysis - -This page has been removed. Refer to <>. - -[role="exclude",id="ml-configuring-populations"] -=== Configuring population analysis - -This page has been removed. Refer to <>. - -[role="exclude",id="ml-configuring-categories"] -=== Detecting anomalous categories of data - -This page has been removed. Refer to <>. - [role="exclude",id="ml-inference-models"] === Trained {ml} models as functions diff --git a/docs/en/stack/ml/setup.asciidoc b/docs/en/stack/ml/setup.asciidoc index 96fe56cae..eb1ad5032 100644 --- a/docs/en/stack/ml/setup.asciidoc +++ b/docs/en/stack/ml/setup.asciidoc @@ -11,17 +11,16 @@ To use the {stack} {ml-features}, you must have: -[%interactive] -- [ ] the {subscriptions}[appropriate subscription] level or the free trial +- the {subscriptions}[appropriate subscription] level or the free trial period activated -- [ ] `xpack.ml.enabled` set to its default value of `true` on every node in the +- `xpack.ml.enabled` set to its default value of `true` on every node in the cluster (refer to {ref}/ml-settings.html[{ml-cap} settings in {es}]) -- [ ] `ml` value defined in the list of `node.roles` on the +- `ml` value defined in the list of `node.roles` on the {ref}/modules-node.html#ml-node[{ml} nodes] -- [ ] {ml} features visible in the {kib} space -- [ ] security privileges assigned to the user that: - * grant use of {ml-features}, and - * grant access to source and destination indices. +- {ml} features visible in the {kib} space +- security privileges assigned to the user that: + * grant use of {ml-features}, and + * grant access to source and destination indices. TIP: The fastest way to get started with {ml-features} is to {ess-trial}[start a free 14-day trial of {ess}] in the cloud. @@ -39,12 +38,15 @@ the two main categories: * *<>*: uses the {ml-features} in {kib} and does not use Dev Tools. It requires either {kib} feature privileges or {es} security privileges and is granted the most permissive combination of both. {kib} feature -privileges are recommended if you control job level visibility via _Spaces_. +privileges are recommended if you control job level visibility via **Spaces**. {ml-cap} features must be visible in the relevant space. Refer to <> for configuration information. -You can configure these privileges under **{stack-manage-app}** > _Security_ in -{kib} or via the respective {es} security APIs. +You can configure these privileges + +- under **Security**. To open Security, find **{stack-manage-app}** in the main menu or +use the {kibana-ref}/kibana-concepts-analysts.html#_finding_your_apps_and_objects[global search field]. +- via the respective {es} security APIs. [discrete] @@ -55,19 +57,17 @@ If you use {ml} APIs, you must have the following cluster and index privileges: For full access: -[%interactive] -* [ ] `machine_learning_admin` built-in role or the equivalent cluster +* `machine_learning_admin` built-in role or the equivalent cluster privileges -* [ ] `read` and `view_index_metadata` on source indices -* [ ] `read`, `manage`, and `index` on destination indices (for +* `read` and `view_index_metadata` on source indices +* `read`, `manage`, and `index` on destination indices (for {dfanalytics-jobs} only) For read-only access: -[%interactive] -* [ ] `machine_learning_user` built-in role or the equivalent cluster privileges -* [ ] `read` index privileges on source indices -* [ ] `read` index privileges on destination indices (for {dfanalytics-jobs} +* `machine_learning_user` built-in role or the equivalent cluster privileges +* `read` index privileges on source indices +* `read` index privileges on destination indices (for {dfanalytics-jobs} only) IMPORTANT: The `machine_learning_admin` and `machine_learning_user` built-in @@ -80,25 +80,33 @@ that contain sensitive information from the source indices to the results. [[kib-security]] === {kib} security +IMPORTANT: Granting `All` or `Read` {kib} feature privilege for {ml-app} will +also grant the role the equivalent feature privileges to certain types of {kib} +saved objects, namely index patterns, dashboards, saved searches, and +visualizations as well as {ml} job, trained model and module saved objects. + + [discrete] [[kib-visibility-spaces]] ==== Feature visibility in Spaces In {kib}, the {ml-features} must be visible in your {kibana-ref}/xpack-spaces.html#spaces-control-feature-visibility[space]. To -control which features are visible in your space, use **{stack-manage-app}** > -_{kib}_ > _Spaces_. +manage which features are visible in your space, go to **{stack-manage-app}** > +**{kib}** > **Spaces** or use the {kibana-ref}/kibana-concepts-analysts.html#_finding_your_apps_and_objects[global search field] +to locate **Spaces** directly. [role="screenshot"] image::spaces.jpg["Manage spaces in {kib}"] In addition to index privileges, source {data-sources} must also exist in the -same space as your {ml} jobs. These can be configured in **{stack-manage-app}** -> _{kib}_ > _{data-sources-caps}_. +same space as your {ml} jobs. You can configure these under **{data-sources-caps}**. To open **{data-sources-caps}**, +find **{stack-manage-app}** > **{kib}** in the main menu, or use the {kibana-ref}/kibana-concepts-analysts.html#_finding_your_apps_and_objects[global search field]. Each {ml} job and trained model can be assigned to all, one, or multiple spaces. -This can be configured in **{stack-manage-app} > Alerts and Insights > Machine Learning**. +This can be configured in **Machine Learning**. To open **Machine Learning**, find **{stack-manage-app} > Alerts and Insights** in the main menu, +or use the {kibana-ref}/kibana-concepts-analysts.html#_finding_your_apps_and_objects[global search field]. You can edit the spaces that a job or model is assigned to by clicking the icons in the **Spaces** column. @@ -112,22 +120,20 @@ image::assign-job-spaces.jpg["Assign machine learning jobs to spaces"] Within a {kib} space, for full access to the {ml-features}, you must have: -[%interactive] -* [ ] `Machine Learning: All` {kib} privileges -* [ ] `Data Views Management: All` {kib} feature privileges -* [ ] `read`, and `view_index_metadata` index privileges on your source indices -* [ ] {data-sources} for your source indices -* [ ] {data-sources}, `read`, `manage`, and `index` index privileges on +* `Machine Learning: All` {kib} privileges +* `Data Views Management: All` {kib} feature privileges +* `read`, and `view_index_metadata` index privileges on your source indices +* {data-sources} for your source indices +* {data-sources}, `read`, `manage`, and `index` index privileges on destination indices (for {dfanalytics-jobs} only) Within a {kib} space, for read-only access to the {ml-features}, you must have: -[%interactive] -* [ ] `Machine Learning: Read` {kib} privileges -* [ ] {data-sources} for your source indices -* [ ] `read` index privilege on your source indices -* [ ] {data-sources} and `read` index privileges on destination indices (for +* `Machine Learning: Read` {kib} privileges +* {data-sources} for your source indices +* `read` index privilege on your source indices +* {data-sources} and `read` index privileges on destination indices (for {dfanalytics-jobs} only) IMPORTANT: A user who has full or read-only access to {ml-features} within @@ -152,12 +158,11 @@ privileges and grant access to `machine_learning_admin` or Within a {kib} space, to upload and import files in the *{data-viz}*, you must have: -[%interactive] -- [ ] `Machine Learning: Read` or `Discover: All` {kib} feature privileges -- [ ] `Data Views Management: All` {kib} feature privileges -- [ ] `ingest_admin` built-in role, or `manage_ingest_pipelines` cluster +- `Machine Learning: Read` or `Discover: All` {kib} feature privileges +- `Data Views Management: All` {kib} feature privileges +- `ingest_admin` built-in role, or `manage_ingest_pipelines` cluster privilege -- [ ] `create`, `create_index`, `manage` and `read` index privileges for +- `create`, `create_index`, `manage` and `read` index privileges for destination indices For more information, see {ref}/security-privileges.html[Security privileges]