Update Release History for 5.7 (#2663)

* docs(release-history.md): Add 5.7.0 to release history

* docs(release-history.md): Add 5.7.1 to release history

.github/CODEOWNERS

@@ -0,0 +1 @@
+* @saadrahim @Rmalavally @amd-aakash @zhang2amd @jlgreathouse @samjwu @MathiasMagnus @LisaDelaney

.github/ISSUE_TEMPLATE/0_issue_report.yml

@@ -0,0 +1,76 @@
+name: Issue Report
+description: File a report for something not working correctly.
+title: "[Issue]: "
+
+body:
+- type: markdown
+  attributes:
+    value: |
+      Thank you for taking the time to fill out this report!
+
+      On a Linux system, you can acquire your OS, CPU, GPU, and ROCm version (for filling out this report) with the following commands:
+      echo "OS:" && cat /etc/os-release | grep -E "^(NAME=|VERSION=)";
+      echo "CPU: " && cat /proc/cpuinfo | grep "model name" | sort --unique;
+      echo "GPU:" && /opt/rocm/bin/rocminfo | grep -E "^\s*(Name|Marketing Name)";
+      echo "ROCm in /opt:" && ls -1 /opt | grep -E "rocm-";
+- type: textarea
+  attributes:
+    label: Problem Description
+    description: Describe the issue you encountered.
+    placeholder: "The steps to reproduce can be included here, or in the dedicated section further below."
+  validations:
+    required: true
+- type: input
+  attributes:
+    label: Operating System
+    description: What is the name and version number of the OS?
+    placeholder: "e.g. Ubuntu 22.04.3 LTS (Jammy Jellyfish)"
+  validations:
+    required: true
+- type: input
+  attributes:
+    label: CPU
+    description: What CPU did you encounter the issue on?
+    placeholder: "e.g. AMD Ryzen 9 5900HX with Radeon Graphics"
+  validations:
+    required: true
+- type: input
+  attributes:
+    label: GPU
+    description: What GPU(s) did you encounter the issue on?
+    placeholder: "e.g. MI200"
+  validations:
+    required: true
+- type: input
+  attributes:
+    label: ROCm Version
+    description: What version(s) of ROCm did you encounter the issue on?
+    placeholder: "e.g. 5.7.0"
+  validations:
+    required: true
+- type: input
+  attributes:
+    label: ROCm Component
+    description: (Optional) If this issue relates to a specific ROCm component, it can be mentioned here.
+    placeholder: "e.g. rocBLAS"
+
+- type: textarea
+  attributes:
+    label: Steps to Reproduce
+    description: (Optional) Detailed steps to reproduce the issue.
+    placeholder: Please also include what you expected to happen, and what actually did, at the failing step(s).
+  validations:
+    required: false
+
+- type: textarea
+  attributes:
+    label: Output of /opt/rocm/bin/rocminfo --support
+    description: The output of rocminfo --support will help to better address the problem.
+    placeholder: |
+      ROCk module is loaded
+      =====================
+      HSA System Attributes
+      =====================
+      [...]
+  validations:
+    required: true

.github/ISSUE_TEMPLATE/1_feature_request.yml

@@ -0,0 +1,32 @@
+name: Feature Suggestion
+description: Suggest an additional functionality, or new way of handling an existing functionality.
+title: "[Feature]: "
+
+body:
+- type: markdown
+  attributes:
+    value: |
+      Thank you for taking the time to make a suggestion!
+
+- type: textarea
+  attributes:
+    label: Suggestion Description
+    description: Describe your suggestion.
+  validations:
+    required: true
+- type: input
+  attributes:
+    label: Operating System
+    description: (Optional) If this is for a specific OS, you can mention it here.
+    placeholder: "e.g. Ubuntu"
+- type: input
+  attributes:
+    label: GPU
+    description: (Optional) If this is for a specific GPU or GPU family, you can mention it here.
+    placeholder: "e.g. MI200"
+- type: input
+  attributes:
+    label: ROCm Component
+    description: (Optional) If this issue relates to a specific ROCm component, it can be mentioned here.
+    placeholder: "e.g. rocBLAS"
+

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,5 @@
+blank_issues_enabled: false
+contact_links:
+  - name: ROCm Community Discussions
+    url: https://github.com/RadeonOpenCompute/ROCm/discussions
+    about: Please ask and answer questions here for anything ROCm.

.github/dependabot.yml

@@ -0,0 +1,13 @@
+# To get started with Dependabot version updates, you'll need to specify which
+# package ecosystems to update and where the package manifests are located.
+# Please see the documentation for all configuration options:
+# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates
+
+version: 2
+updates:
+  - package-ecosystem: "pip" # See documentation for possible values
+    directory: "/docs/sphinx" # Location of package manifests
+    open-pull-requests-limit: 10
+    schedule:
+      interval: "daily"
+    versioning-strategy: increase

.github/workflows/linting.yml

@@ -0,0 +1,20 @@
+name: Linting
+
+on:
+  push:
+    branches: 
+    - develop
+    - main
+    - 'docs/*'
+    - 'roc**'
+  pull_request:
+    branches: 
+    - develop
+    - main
+    - 'docs/*'
+    - 'roc**'
+
+jobs:
+  call-workflow-passing-data:
+    name: Documentation
+    uses: RadeonOpenCompute/rocm-docs-core/.github/workflows/linting.yml@develop

.gitignore

@@ -0,0 +1,19 @@
+.venv
+.vscode
+build
+
+# documentation artifacts
+_build/
+_images/
+_static/
+_templates/
+_toc.yml
+docBin/
+_doxygen/
+_readthedocs/
+
+# avoid duplicating contributing.md due to conf.py
+docs/CHANGELOG.md
+docs/contribute/index.md
+docs/about/release-notes.md
+docs/about/CHANGELOG.md

.markdownlint-cli2.yaml

@@ -0,0 +1,18 @@
+config:
+  default: true
+  MD004:
+    style: asterisk
+  MD013: false
+  MD026:
+    punctuation: '.,;:!'
+  MD029:
+    style: ordered
+  MD033: false
+  MD034: false
+  MD041: false
+  MD051: false
+ignores:
+  - CHANGELOG.md
+  - docs/CHANGELOG.md
+  - "{,docs/}{RELEASE,release}.md"
+  - tools/autotag/templates/**/*.md

.readthedocs.yaml

@@ -0,0 +1,18 @@
+# Read the Docs configuration file
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
+
+version: 2
+
+sphinx:
+   configuration: docs/conf.py
+
+formats: [htmlzip, pdf]
+
+python:
+   install:
+   - requirements: docs/sphinx/requirements.txt
+
+build:
+   os: ubuntu-20.04
+   tools:
+      python: "3.8"

.wordlist.txt

@@ -0,0 +1,558 @@
+ABI
+activations
+addr
+AddressSanitizer
+AlexNet
+alloc
+allocator
+allocators
+ALU
+AMD
+AMDGPU
+amdgpu
+AMDGPUs
+AMDMIGraphX
+AMI
+AOCC
+AOMP
+api
+APIC
+APIs
+Arb
+ASan
+ASIC
+ASICs
+ASm
+atmi
+atomics
+autogenerated
+avx
+awk
+backend
+backends
+benchmarking
+bilinear
+BitCode
+BLAS
+Blit
+blit
+BMC
+buildable
+bursty
+bzip
+cacheable
+CCD
+cd
+CDNA
+CentOS
+centric
+changelog
+chiplet
+CIFAR
+CLI
+CMake
+cmake
+CMakeLists
+CMakePackage
+cmd
+coalescable
+codename
+Codespaces
+comgr
+Commitizen
+CommonMark
+composable
+concretization
+Concretized
+Conda
+config
+conformant
+convolutional
+convolves
+CoRR
+CP
+CPC
+CPF
+CPP
+CPU
+CPUs
+CSC
+CSE
+CSn
+csn
+CSV
+CU
+cuBLAS
+CUDA
+cuFFT
+cuLIB
+cuRAND
+CUs
+cuSOLVER
+cuSPARSE
+dataset
+datasets
+dataspace
+datatype
+datatypes
+dbgapi
+de
+deallocation
+denormalize
+Dependabot
+deserializers
+detections
+dev
+devicelibs
+DGEMM
+disambiguates
+distro
+DL
+DMA
+DNN
+DNNL
+Dockerfile
+DockerHub
+Doxygen
+DPM
+DRI
+DW
+DWORD
+el
+enablement
+endpgm
+env
+epilog
+EPYC
+ESXi
+ethernet
+exascale
+executables
+ffmpeg
+FFT
+FFTs
+FHS
+filesystem
+Filesystem
+Flang
+FMA
+Fortran
+fortran
+FP
+galb
+gcc
+GCD
+GCDs
+GCN
+GDB
+gdb
+GDDR
+GDR
+GDS
+GEMM
+GEMMs
+gfortran
+gfx
+GIM
+github
+Gitpod
+GL
+GLXT
+GMI
+gnupg
+GPG
+GPR
+GPU
+GPUs
+grayscale
+GRBM
+gzip
+Haswell
+HBM
+HCA
+heterogenous
+hipamd
+hipBLAS
+hipblas
+hipBLASLt
+HIPCC
+hipCUB
+hipcub
+HIPExtension
+hipFFT
+hipfft
+hipfort
+HIPIFY
+hipify
+hipLIB
+hipRAND
+hipSOLVER
+hipsolver
+hipSPARSE
+hipsparse
+hipSPARSELt
+hipTensor
+HPC
+HPCG
+HPL
+HSA
+hsa
+hsakmt
+HWE
+ib_core
+ICV
+ImageNet
+IMDB
+inband
+incrementing
+inferencing
+InfiniBand
+inflight
+init
+Inlines
+inlining
+installable
+IntelliSense
+interprocedural
+Intersphinx
+intra
+invariants
+invocating
+Ioffe
+IOMMU
+IOP
+IOPM
+IOV
+ipo
+ISA
+ISV
+ISVs
+JSON
+Jupyter
+kdb
+KFD
+Khronos
+KVM
+LAPACK
+LCLK
+LDS
+libjpeg
+libs
+linearized
+linter
+linux
+llvm
+LLVM
+localscratch
+logits
+lossy
+LSAN
+LTS
+Makefile
+Makefiles
+matchers
+Matplotlib
+Mellanox's
+MEM
+MERCHANTABILITY
+MFMA
+microarchitecture
+MIGraphX
+migraphx
+MIOpen
+miopen
+MIOpenGEMM
+miopengemm
+MIVisionX
+mivisionx
+mkdir
+mlirmiopen
+MMA
+MNIST
+MPI
+MSVC
+mtypes
+Multicore
+Multithreaded
+MVAPICH
+mvffr
+MyEnvironment
+MyST
+namespace
+namespaces
+Nano
+Navi
+NBIO
+NBIOs
+NIC
+NICs
+Noncoherently
+NPS
+NUMA
+NumPy
+numref
+NVCC
+NVPTX
+OAM
+OAMs
+ocl
+OCP
+OEM
+OFED
+OMP
+OMPT
+OMPX
+ONNX
+OpenCL
+opencl
+opencv
+OpenFabrics
+OpenGL
+OpenMP
+openmp
+openssl
+OpenVX
+optimizers
+os
+OSS
+OSU
+Pageable
+pageable
+passthrough
+PCI
+PCIe
+PeerDirect
+perfcounter
+Perfetto
+performant
+perl
+PIL
+PILImage
+PowerShell
+pragma
+pre
+prebuilt
+precompiled
+prefetch
+preprocess
+preprocessing
+preq
+prerequisites
+PRNG
+profiler
+protobuf
+PRs
+pseudorandom
+py
+PyPi
+PyTorch
+Qcycles
+quasirandom
+Radeon
+RadeonOpenCompute
+RCCL
+rccl
+RDC
+rdc
+RDMA
+RDNA
+reformats
+RelWithDebInfo
+repos
+Req
+req
+resampling
+RST
+reStructuredText
+RHEL
+Rickle
+roadmap
+roc
+ROC
+rocAL
+rocALUTION
+rocalution
+rocBLAS
+rocblas
+rocclr
+ROCdbgapi
+rocFFT
+rocfft
+ROCgdb
+ROCk
+rocLIB
+rocm
+ROCm
+ROCmCC
+rocminfo
+ROCmSoftwarePlatform
+ROCmValidationSuite
+rocPRIM
+rocprim
+rocprof
+ROCProfiler
+rocprofiler
+ROCr
+rocr
+rocRAND
+rocrand
+rocSOLVER
+rocsolver
+rocSPARSE
+rocsparse
+roct
+rocThrust
+rocthrust
+ROCTracer
+roctracer
+rocWMMA
+RST
+runtime
+runtimes
+RW
+SALU
+SBIOS
+SCA
+scalability
+SDK
+SDMA
+SDRAM
+SENDMSG
+sendmsg
+SENDMSG
+sendmsg
+SerDes
+serializers
+SGPR
+SGPRs
+SHA
+shader
+Shlens
+sigmoid
+SIGQUIT
+SIMD
+SKU
+SKUs
+skylake
+sL
+SLES
+SMEM
+SMI
+smi
+SMT
+softmax
+Spack
+spack
+SPI
+SQs
+SRAM
+SRAMECC
+src
+stochastically
+strided
+subdirectory
+subexpression
+subfolder
+subfolders
+supercomputing
+SWE
+Szegedy
+tagram
+TCA
+TCC
+TCI
+TCIU
+TCP
+TCR
+TensorBoard
+TensorFlow
+TFLOPS
+tg
+th
+tmp
+ToC
+tokenize
+toolchain
+toolchains
+toolset
+toolsets
+TorchAudio
+TorchScript
+TorchServe
+TorchVision
+torchvision
+tracebacks
+TransferBench
+TrapStatus
+txt
+UAC
+uarch
+ubuntu
+UC
+UCC
+UCX
+UIF
+Uncached
+uncached
+Unhandled
+uninstallation
+unsqueeze
+unstacking
+unswitching
+untrusted
+untuned
+USM
+UTCL
+UTIL
+utils
+VALU
+Vanhoucke
+VBIOS
+vdi
+vectorizable
+vectorization
+vectorize
+vectorized
+vectorizer
+vectorizes
+VGPR
+VGPRs
+vjxb
+vL
+VM
+VMEM
+VMWare
+VRAM
+VSIX
+VSkipped
+Vulkan
+walkthrough
+walkthroughs
+wavefront
+wavefronts
+WGP
+whitespaces
+Wojna
+workgroup
+Workgroups
+workgroups
+writeback
+Writebacks
+writebacks
+wrreq
+WX
+wzo
+Xeon
+XGMI
+Xnack
+XT
+Xteam
+XTX
+xz
+YAML
+yaml
+YML
+YModel
+ysvmadyb
+ZenDNN
+zypper

CONTRIBUTING.md

@@ -0,0 +1,229 @@
+# Contributing to ROCm documentation
+
+AMD values and encourages contributions to our code and documentation. If you choose to
+contribute, we encourage you to be polite and respectful. Improving documentation is a long-term
+process, to which we are dedicated.
+
+If you have issues when trying to contribute, refer to the
+[discussions](https://github.com/RadeonOpenCompute/ROCm/discussions) page in our GitHub
+repository.
+
+## Folder structure and naming convention
+
+Our documentation follows the Pitchfork folder structure. Most documentation files are stored in the
+`/docs` folder. Some special files (such as release, contributing, and changelog) are stored in the root
+(`/`) folder.
+
+All images are stored in the `/docs/data` folder. An image's file path mirrors that of the documentation
+file where it is used.
+
+Our naming structure uses kebab case; for example, `my-file-name.rst`.
+
+## Supported formats and syntax
+
+Our documentation includes both Markdown and RST files. We are gradually transitioning existing
+Markdown to RST in order to more effectively meet our documentation needs. When contributing,
+RST is preferred; if you must use Markdown, use GitHub-flavored Markdown.
+
+We use [Sphinx Design](https://sphinx-design.readthedocs.io/en/latest/index.html) syntax and compile
+our API references using [Doxygen](https://www.doxygen.nl/).
+
+The following table shows some common documentation components and the syntax convention we
+use for each:
+
+<table>
+<tr>
+<th>Component</th>
+<th>RST syntax</th>
+</tr>
+<tr>
+<td>Code blocks</td>
+<td>
+
+```rst
+
+.. code-block:: language-name
+
+  My code block.
+
+
+```
+
+</td>
+</tr>
+<tr>
+<td>Cross-referencing internal files</td>
+<td>
+
+```rst
+
+:doc:`Title <../path/to/file/filename>`
+
+```
+
+</td>
+</tr>
+<tr>
+<td>External links</td>
+<td>
+
+```rst
+
+`link name  <URL>`_
+
+```
+
+</td>
+</tr>
+<tr>
+<tr>
+<td>Headings</td>
+<td>
+
+```rst
+
+******************
+Chapter title (H1)
+******************
+
+Section title (H2)
+===============
+
+Subsection title (H3)
+---------------------
+
+Sub-subsection title (H4)
+^^^^^^^^^^^^^^^^^^^^
+
+
+```
+
+</td>
+</tr>
+<tr>
+<td>Images</td>
+<td>
+
+```rst
+
+.. image:: image1.png
+
+```
+
+</td>
+</tr>
+<tr>
+<td>Internal links</td>
+<td>
+
+```rst
+
+1. Add a tag to the section you want to reference:
+
+.. _my-section-tag: section-1
+
+Section 1
+==========
+
+2. Link to your tag:
+
+As shown in :ref:`section-1`.
+
+```
+
+</td>
+</tr>
+<tr>
+<tr>
+<td>Lists</td>
+<td>
+
+```rst
+
+# Ordered (numbered) list item
+
+* Unordered (bulleted) list item
+
+```
+
+</td>
+</tr>
+<tr>
+<tr>
+<td>Math (block)</td>
+<td>
+
+```rst
+
+.. math::
+
+  A = \begin{pmatrix}
+          0.0 & 1.0 & 1.0 & 3.0 \\
+          4.0 & 5.0 & 6.0 & 7.0 \\
+        \end{pmatrix}
+
+```
+
+</td>
+</tr>
+<tr>
+<td>Math (inline)</td>
+<td>
+
+```rst
+
+:math:`2 \times 2 `
+
+```
+
+</td>
+</tr>
+<tr>
+<td>Notes</td>
+<td>
+
+```rst
+
+.. note::
+
+  My note here.
+
+```
+
+</td>
+</tr>
+<tr>
+<td>Tables</td>
+<td>
+
+```rst
+
+.. csv-table::  Optional title here
+  :widths: 30, 70  #optional column widths
+  :header: "entry1 header", "entry2 header"
+
+   "entry1", "entry2"
+
+```
+
+</td>
+</tr>
+</table>
+
+## Language and style
+
+We use the
+[Google developer documentation style guide](https://developers.google.com/style/highlights) to
+guide our content.
+
+Font size and type, page layout, white space control, and other formatting
+details are controlled via
+[rocm-docs-core](https://github.com/RadeonOpenCompute/rocm-docs-core). If you want to notify us
+of any formatting issues, create a pull request in our
+[rocm-docs-core](https://github.com/RadeonOpenCompute/rocm-docs-core) GitHub repository.
+
+## Building our documentation
+
+<!--  % TODO: Fix the link to be able to work at every files  -->
+To learn how to build our documentation, refer to
+[Building documentation](./building.md).

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 Advanced Micro Devices, Inc. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -1,711 +1,50 @@
-## Are You Ready to ROCK?
-The ROCm Platform brings a rich foundation to advanced computing by seamlessly integrating the CPU and GPU with the goal of solving real-world problems.
-This software enables the high-performance operation of AMD GPUs for computation oriented tasks in the Linux operating system.
-Please refer the ROCm Documentation [here](https://rocm-documentation.readthedocs.io/en/latest/index.html).
+# AMD ROCm™ platform
 
-### Current ROCm Version: 2.7
+ROCm is an open-source stack, composed primarily of open-source software, designed for graphics
+processing unit (GPU) computation. ROCm consists of a collection of drivers, development tools, and
+APIs that enable GPU programming from low-level kernel to end-user applications.
 
-- [New features and enhancements in ROCm 2.7](#new-features-and-enhancements-in-rocm-27)
-- [The latest ROCm platform - ROCm 2.7](#the-latest-rocm-platform-rocm-27)
-- [Hardware Support](#hardware-support)
-  * [Supported GPUs](#supported-gpus)
-  * [Supported CPUs](#supported-cpus)
-  * [Not supported or limited support under ROCm](#not-supported-or-limited-support-under-rocm)
-- [Supported Operating Systems](#supported-operating-systems-new-operating-systems-available)
-  * [ROCm support in upstream Linux kernels](#rocm-support-in-upstream-linux-kernels)
-- [Installing from AMD ROCm repositories](#installing-from-amd-rocm-repositories)
-  * [ROCm Binary Package Structure](#rocm-binary-package-structure)
-  * [Ubuntu Support - installing from a Debian repository](#ubuntu-support-installing-from-a-debian-repository)
-  * [CentOS/RHEL 7 (7.6) Support](#centosrhel-7-76-support)
-- [Known issues / workarounds](#known-issues-workarounds)
-- [Closed source components](#closed-source-components)
-- [Getting ROCm source code](#getting-rocm-source-code)
-  * [Installing repo](#installing-repo)
-  * [Downloading the ROCm source code](#downloading-the-rocm-source-code)
-  * [Building the ROCm source code](#building-the-rocm-source-code)
-- [Deprecation Notice](#deprecation-notice-hcc)
-- [Final notes](#final-notes)
+With ROCm, you can customize your GPU software to meet your specific needs. You can develop,
+collaborate, test, and deploy your applications in a free, open source, integrated, and secure software
+ecosystem. ROCm is particularly well-suited to GPU-accelerated high-performance computing (HPC),
+artificial intelligence (AI), scientific computing, and computer aided design (CAD).
 
-### New features and enhancements in ROCm 2.7
+ROCm is powered by AMD’s
+[Heterogeneous-computing Interface for Portability (HIP)](https://github.com/ROCm-Developer-Tools/HIP),
+an open-source software C++ GPU programming environment and its corresponding runtime. HIP
+allows ROCm developers to create portable applications on different platforms by deploying code on a
+range of platforms, from dedicated gaming GPUs to exascale HPC clusters.
 
+ROCm supports programming models, such as OpenMP and OpenCL, and includes all necessary open
+source software compilers, debuggers, and libraries. ROCm is fully integrated into machine learning
+(ML) frameworks, such as PyTorch and TensorFlow.
 
-#### [rocFFT] Real FFT Functional
-Improved real/complex 1D even-length transforms of unit stride. Performance improvements of up to 4.5x are observed. Large problem sizes should see approximately 2x.
+## ROCm documentation
 
-#### rocRand Enhancements and Optimizations
-- Added support for new datatypes: uchar, ushort, half.
-- Improved performance on "Vega 7nm" chips, such as on the Radeon Instinct MI50
-- mtgp32 uniform double performance changes due generation algorithm standardization. Better quality random numbers now generated with 30% decrease in performance
-- Up to 5% performance improvements for other algorithms
+This repository contains the manifest file for ROCm releases, changelogs, and release information.
 
-#### RAS
-Added support for RAS on Radeon Instinct MI50, including:
-- Memory error detection
-- Memory error detection counter
+The `default.xml` file contains information for all repositories and the associated commit used to build
+the current ROCm release; `default.xml` uses the Manifest Format repository.
 
-#### ROCm-SMI enhancements
-Added ROCm-SMI CLI and LIB support for FW version, compute running processes, utilization rates, utilization counter, link error counter, and unique ID.
+Source code for our documentation is located in the `/docs` folder of most ROCm repositories. The
+`develop` branch of our repositories contains content for the next ROCm release.
 
+The ROCm documentation homepage is [rocm.docs.amd.com](https://rocm.docs.amd.com).
 
-Features and enhancements introduced in previous versions of ROCm can be found in [version_history.md](version_history.md)
+### Building our documentation
 
-### The latest ROCm platform - ROCm 2.7
+For a quick-start build, use the following code. For more options and detail, refer to
+[Building documentation](./contribute/building.md).
 
-The latest supported version of the drivers, tools, libraries and source code for the ROCm platform have been released and are available from the following GitHub repositories:
+```bash
+cd docs
 
-* ROCm Core Components
-  - [ROCk Kernel Driver](https://github.com/RadeonOpenCompute/ROCK-Kernel-Driver/tree/roc-2.7.0)
-  - [ROCr Runtime](https://github.com/RadeonOpenCompute/ROCR-Runtime/tree/roc-2.7.0)
-  - [ROCt Thunk Interface](https://github.com/RadeonOpenCompute/ROCT-Thunk-Interface/tree/roc-2.7.0)
-* ROCm Support Software
-  - [ROCm SMI](https://github.com/RadeonOpenCompute/ROC-smi/tree/roc-2.7.0)
-  - [ROCm cmake](https://github.com/RadeonOpenCompute/rocm-cmake/tree/master-rocm-2.7)
-  - [rocminfo](https://github.com/RadeonOpenCompute/rocminfo/tree/roc-2.7.0)
-  - [ROCm Bandwidth Test](https://github.com/RadeonOpenCompute/rocm_bandwidth_test/tree/roc-2.7.0)
-* ROCm Development Tools
-  - [HCC compiler](https://github.com/RadeonOpenCompute/hcc/tree/roc-hcc-2.7.0)
-  - [HIP](https://github.com/ROCm-Developer-Tools/HIP/tree/roc-2.7.0)
-  - [ROCm Device Libraries](https://github.com/RadeonOpenCompute/ROCm-Device-Libs/tree/roc-hcc-2.7.0)
-  - ROCm OpenCL, which is created from the following components:
-    - [ROCm OpenCL Runtime](http://github.com/RadeonOpenCompute/ROCm-OpenCL-Runtime/tree/roc-2.7.0)
-    - [ROCm OpenCL Driver](http://github.com/RadeonOpenCompute/ROCm-OpenCL-Driver/tree/roc-2.7.0)
-  - The ROCm OpenCL compiler, which is created from the following components:
-      - [ROCm LLVM OCL](http://github.com/RadeonOpenCompute/llvm/tree/roc-ocl-2.7.0)
-      - [ROCm LLVM HCC](http://github.com/RadeonOpenCompute/llvm/tree/roc-hcc-2.7.0)
-      - [ROCm Clang](http://github.com/RadeonOpenCompute/clang/tree/roc-2.7.0)
-      - [ROCm lld OCL](http://github.com/RadeonOpenCompute/lld/tree/roc-ocl-2.7.0)
-      - [ROCm lld HCC](http://github.com/RadeonOpenCompute/lld/tree/roc-hcc-2.7.0)
-      - [ROCm Device Libraries](https://github.com/RadeonOpenCompute/ROCm-Device-Libs/tree/roc-ocl-2.7.0)
-  - [ROCM Clang-OCL Kernel Compiler](https://github.com/RadeonOpenCompute/clang-ocl/tree/roc-2.7.0)
-  - [Asynchronous Task and Memory Interface (ATMI)](https://github.com/RadeonOpenCompute/atmi/tree/rocm_2.7.0)
-  - [ROCr Debug Agent](https://github.com/ROCm-Developer-Tools/rocr_debug_agent/tree/roc-2.7.0)
-  - [ROCm Code Object Manager](https://github.com/RadeonOpenCompute/ROCm-CompilerSupport/tree/roc-2.7.0)
-  - [ROC Profiler](https://github.com/ROCm-Developer-Tools/rocprofiler/tree/roc-2.6.x)
-  - [ROC Tracer](https://github.com/ROCmSoftwarePlatform/roctracer/tree/roc-2.7.x)
-  - [Radeon Compute Profiler](https://github.com/GPUOpen-Tools/RCP/tree/3a49405)
-  - Example Applications:
-    - [HCC Examples](https://github.com/ROCm-Developer-Tools/HCC-Example-Application/tree/ffd65333)
-    - [HIP Examples](https://github.com/ROCm-Developer-Tools/HIP-Examples/tree/roc-2.7.0)
-* ROCm Libraries
-  - [rocBLAS](https://github.com/ROCmSoftwarePlatform/rocBLAS/tree/master-rocm-2.7)
-  - [hipBLAS](https://github.com/ROCmSoftwarePlatform/hipBLAS/tree/master-rocm-2.7)
-  - [rocFFT](https://github.com/ROCmSoftwarePlatform/rocFFT/tree/master-rocm-2.7)
-  - [rocRAND](https://github.com/ROCmSoftwarePlatform/rocRAND/tree/master-rocm-2.7)
-  - [rocSPARSE](https://github.com/ROCmSoftwarePlatform/rocSPARSE/tree/master-rocm-2.7)
-  - [hipSPARSE](https://github.com/ROCmSoftwarePlatform/hipSPARSE/tree/master-rocm-2.7)
-  - [rocALUTION](https://github.com/ROCmSoftwarePlatform/rocALUTION/tree/master-rocm-2.7)
-  - [MIOpenGEMM](https://github.com/ROCmSoftwarePlatform/MIOpenGEMM/tree/9547fb9e)
-  - [MIOpen](https://github.com/ROCmSoftwarePlatform/MIOpen/tree/roc-2.7.0)
-  - [rocThrust](https://github.com/ROCmSoftwarePlatform/rocThrust/tree/master-rocm-2.7)
-  - [ROCm SMI Lib](https://github.com/RadeonOpenCompute/rocm_smi_lib/tree/roc-2.7.0)
-  - [RCCL](https://github.com/ROCmSoftwarePlatform/rccl/tree/master-rocm-2.7)
-  - [MIVisionX](https://github.com/GPUOpen-ProfessionalCompute-Libraries/MIVisionX/tree/1.3.0)
-  - [hipCUB](https://github.com/ROCmSoftwarePlatform/hipCUB/tree/rocm-2.7)
+pip3 install -r sphinx/requirements.txt
 
-### Hardware Support
-ROCm is focused on using AMD GPUs to accelerate computational tasks such as machine learning, engineering workloads, and scientific computing.
-In order to focus our development efforts on these domains of interest, ROCm supports a targeted set of hardware configurations which are detailed further in this section.
-
-#### Supported GPUs
-Because the ROCm Platform has a focus on particular computational domains, we offer official support for a selection of AMD GPUs that are designed to offer good performance and price in these domains.
-
-ROCm officially supports AMD GPUs that use following chips:
-
-  * GFX8 GPUs
-    * "Fiji" chips, such as on the AMD Radeon R9 Fury X and Radeon Instinct MI8
-    * "Polaris 10" chips, such as on the AMD Radeon RX 580 and Radeon Instinct MI6
-  * GFX9 GPUs
-    * "Vega 10" chips, such as on the AMD Radeon RX Vega 64 and Radeon Instinct MI25
-    * "Vega 7nm" chips, such as on the Radeon Instinct MI50, Radeon Instinct MI60 or AMD Radeon VII
-
-ROCm is a collection of software ranging from drivers and runtimes to libraries and developer tools.
-Some of this software may work with more GPUs than the "officially supported" list above, though AMD does not make any official claims of support for these devices on the ROCm software platform.
-The following list of GPUs are enabled in the ROCm software, though full support is not guaranteed:
-
-  * GFX8 GPUs
-    * "Polaris 11" chips, such as on the AMD Radeon RX 570 and Radeon Pro WX 4100
-    * "Polaris 12" chips, such as on the AMD Radeon RX 550 and Radeon RX 540
-  * GFX7 GPUs
-    * "Hawaii" chips, such as the AMD Radeon R9 390X and FirePro W9100
-
-As described in the next section, GFX8 GPUs require PCI Express 3.0 (PCIe 3.0) with support for PCIe atomics. This requires both CPU and motherboard support. GFX9 GPUs require PCIe 3.0 with support for PCIe atomics by default, but they can operate in most cases without this capability.
-
-The integrated GPUs in AMD APUs are not officially supported targets for ROCm.
-As described [below](#limited-support), "Carrizo", "Bristol Ridge", and "Raven Ridge" APUs are enabled in our upstream drivers and the ROCm OpenCL runtime.
-However, they are not enabled in our HCC or HIP runtimes, and may not work due to motherboard or OEM hardware limitations.
-As such, they are not yet officially supported targets for ROCm.
-
-For a more detailed list of hardware support, please see [the following documentation](https://rocm.github.io/hardware.html).
-
-#### Supported CPUs
-As described above, GFX8 GPUs require PCIe 3.0 with PCIe atomics in order to run ROCm.
-In particular, the CPU and every active PCIe point between the CPU and GPU require support for PCIe 3.0 and PCIe atomics.
-The CPU root must indicate PCIe AtomicOp Completion capabilities and any intermediate switch must indicate PCIe AtomicOp Routing capabilities.
-
-Current CPUs which support PCIe Gen3 + PCIe Atomics are:
-
-  * AMD Ryzen CPUs
-  * The CPUs in AMD Ryzen APUs
-  * AMD Ryzen Threadripper CPUs
-  * AMD EPYC CPUs
-  * Intel Xeon E7 v3 or newer CPUs
-  * Intel Xeon E5 v3 or newer CPUs
-  * Intel Xeon E3 v3 or newer CPUs
-  * Intel Core i7 v4, Core i5 v4, Core i3 v4 or newer CPUs (i.e. Haswell family or newer)
-  * Some Ivy Bridge-E systems
-
-Beginning with ROCm 1.8, GFX9 GPUs (such as Vega 10) no longer require PCIe atomics.
-We have similarly opened up more options for number of PCIe lanes.
-GFX9 GPUs can now be run on CPUs without PCIe atomics and on older PCIe generations, such as PCIe 2.0.
-This is not supported on GPUs below GFX9, e.g. GFX8 cards in the Fiji and Polaris families.
-
-If you are using any PCIe switches in your system, please note that PCIe Atomics are only supported on some switches, such as Broadcom PLX.
-When you install your GPUs, make sure you install them in a PCIe 3.0 x16, x8, x4, or x1 slot attached either directly to the CPU's Root I/O controller or via a PCIe switch directly attached to the CPU's Root I/O controller.
-
-In our experience, many issues stem from trying to use consumer motherboards which provide physical x16 connectors that are electrically connected as e.g. PCIe 2.0 x4, PCIe slots connected via the Southbridge PCIe I/O controller, or PCIe slots connected through a PCIe switch that does
-not support PCIe atomics.
-
-If you attempt to run ROCm on a system without proper PCIe atomic support, you may see an error in the kernel log (`dmesg`):
-```
-kfd: skipped device 1002:7300, PCI rejects atomics
+python3 -m sphinx -T -E -b html -d _build/doctrees -D language=en . _build/html
 ```
 
-Experimental support for our Hawaii (GFX7) GPUs (Radeon R9 290, R9 390, FirePro W9100, S9150, S9170)
-does not require or take advantage of PCIe Atomics. However, we still recommend that you use a CPU
-from the list provided above for compatibility purposes.
+## Older ROCm releases
 
-#### Not supported or limited support under ROCm
-##### Limited support
-
-* ROCm 2.7.x should support PCIe 2.0 enabled CPUs such as the AMD Opteron, Phenom, Phenom II, Athlon, Athlon X2, Athlon II and older Intel Xeon and Intel Core Architecture and Pentium CPUs. However, we have done very limited testing on these configurations, since our test farm has been catering to CPUs listed above. This is where we need community support. _If you find problems on such setups, please report these issues_.
-* Thunderbolt 1, 2, and 3 enabled breakout boxes should now be able to work with ROCm. Thunderbolt 1 and 2 are PCIe 2.0 based, and thus are only supported with GPUs that do not require PCIe 3.0 atomics (e.g. Vega 10). However, we have done no testing on this configuration and would need community support due to limited access to this type of equipment.
-* AMD "Carrizo" and "Bristol Ridge" APUs are enabled to run OpenCL, but do not yet support HCC, HIP, or our libraries built on top of these compilers and runtimes.
-  * As of ROCm 2.1, "Carrizo" and "Bristol Ridge" require the use of upstream kernel drivers.
-  * In addition, various "Carrizo" and "Bristol Ridge" platforms may not work due to OEM and ODM choices when it comes to key configurations parameters such as inclusion of the required CRAT tables and IOMMU configuration parameters in the system BIOS.
-  * Before purchasing such a system for ROCm, please verify that the BIOS provides an option for enabling IOMMUv2 and that the system BIOS properly exposes the correct CRAT table. Inquire with your vendor about the latter.
-* AMD "Raven Ridge" APUs are enabled to run OpenCL, but do not yet support HCC, HIP, or our libraries built on top of these compilers and runtimes.
-  * As of ROCm 2.1, "Raven Ridge" requires the use of upstream kernel drivers.
-  * In addition, various "Raven Ridge" platforms may not work due to OEM and ODM choices when it comes to key configurations parameters such as inclusion of the required CRAT tables and IOMMU configuration parameters in the system BIOS.
-  * Before purchasing such a system for ROCm, please verify that the BIOS provides an option for enabling IOMMUv2 and that the system BIOS properly exposes the correct CRAT table. Inquire with your vendor about the latter.
-
-##### Not supported
-
-* "Tonga", "Iceland", "Vega M", and "Vega 12" GPUs are not supported in ROCm 2.7.x
-* We do not support GFX8-class GPUs (Fiji, Polaris, etc.) on CPUs that do not have PCIe 3.0 with PCIe atomics.
-  * As such, we do not support AMD Carrizo and Kaveri APUs as hosts for such GPUs.
-  * Thunderbolt 1 and 2 enabled GPUs are not supported by GFX8 GPUs on ROCm. Thunderbolt 1 & 2 are based on PCIe 2.0.
-
-### Supported Operating Systems - New operating systems available
-
-The ROCm 2.7.x platform supports the following operating systems:
-
- * Ubuntu 16.04.5(Kernel 4.15) and 18.04.2(Kernel 4.18)
- * CentOS 7.6 (Using devtoolset-7 runtime support)
- * RHEL 7.6 (Using devtoolset-7 runtime support)
-
-#### ROCm support in upstream Linux kernels
-
-As of ROCm 1.9.0, the ROCm user-level software is compatible with the AMD drivers in certain upstream Linux kernels.
-As such, users have the option of either using the ROCK kernel driver that are part of AMD's ROCm repositories or using the upstream driver and only installing ROCm user-level utilities from AMD's ROCm repositories.
-
-These releases of the upstream Linux kernel support the following GPUs in ROCm:
- * 4.17: Fiji, Polaris 10, Polaris 11
- * 4.18: Fiji, Polaris 10, Polaris 11, Vega10
- * 4.20: Fiji, Polaris 10, Polaris 11, Vega10, Vega 7nm
-
-The upstream driver may be useful for running ROCm software on systems that are not compatible with the kernel driver available in AMD's repositories.
-For users that have the option of using either AMD's or the upstreamed driver, there are various tradeoffs to take into consideration:
-
-|   | Using AMD's `rock-dkms` package | Using the upstream kernel driver |
-| ---- | ------------------------------------------------------------| ----- |
-| Pros | More GPU features, and they are enabled earlier | Includes the latest Linux kernel features |
-|      | Tested by AMD on supported distributions | May work on other distributions and with custom kernels |
-|      | Supported GPUs enabled regardless of kernel version | |
-|      | Includes the latest GPU firmware | |
-| Cons | May not work on all Linux distributions or versions | Features and hardware support varies depending on kernel version |
-|      | Not currently supported on kernels newer than 4.18 | Limits GPU's usage of system memory to 3/8 of system memory |
-|      |   | IPC and RDMA capabilities are not yet enabled |
-|      |   | Not tested by AMD to the same level as `rock-dkms` package |
-|      |   | Does not include most up-to-date firmware |
-
-
-### Installing from AMD ROCm repositories
-
-AMD hosts both [Debian](http://repo.radeon.com/rocm/apt/debian/) and [RPM](http://repo.radeon.com/rocm/yum/rpm/) repositories for the ROCm 2.7.x packages at this time.
-
-The packages in the Debian repository have been signed to ensure package integrity.
-
-#### ROCm Binary Package Structure
-
-ROCm is a collection of software ranging from drivers and runtimes to libraries and developer tools.
-In AMD's package distributions, these software projects are provided as a separate packages.
-This allows users to install only the packages they need, if they do not wish to install all of ROCm.
-These packages will install most of the ROCm software into `/opt/rocm/` by default.
-
-The packages for each of the major ROCm components are:
-
-* ROCm Core Components
-  - ROCk Kernel Driver: `rock-dkms`
-  - ROCr Runtime: `hsa-rocr-dev`, `hsa-ext-rocr-dev`
-  - ROCt Thunk Interface: `hsakmt-roct`, `hsakmt-roct-dev`
-* ROCm Support Software
-  - ROCm SMI: `rocm-smi`
-  - ROCm cmake: `rocm-cmake`
-  - rocminfo: `rocminfo`
-  - ROCm Bandwidth Test: `rocm_bandwidth_test`
-* ROCm Development Tools
-  - HCC compiler: `hcc`
-  - HIP: `hip_base`, `hip_doc`, `hip_hcc`, `hip_samples`
-  - ROCm Device Libraries: `rocm-device-libs`
-  - ROCm OpenCL: `rocm-opencl`, `rocm-opencl-devel` (on RHEL/CentOS), `rocm-opencl-dev` (on Ubuntu)
-  - ROCM Clang-OCL Kernel Compiler: `rocm-clang-ocl`
-  - Asynchronous Task and Memory Interface (ATMI): `atmi`
-  - ROCr Debug Agent: `rocr_debug_agent`
-  - ROCm Code Object Manager: `comgr`
-  - ROC Profiler: `rocprofiler-dev`
-  - ROC Tracer: `roctracer-dev`
-  - Radeon Compute Profiler: `rocm-profiler`
-* ROCm Libraries
-  - rocALUTION: `rocalution`
-  - rocBLAS: `rocblas`
-  - hipBLAS: `hipblas`
-  - hipCUB: `hipCUB`
-  - rocFFT: `rocfft`
-  - rocRAND: `rocrand`
-  - rocSPARSE: `rocsparse`
-  - hipSPARSE: `hipsparse`
-  - ROCm SMI Lib: `rocm_smi_lib64`
-  - rocThrust: `rocThrust`
-  - MIOpen: `MIOpen-HIP` (for the HIP version), `MIOpen-OpenCL` (for the OpenCL version)
-  - MIOpenGEMM: `miopengemm`
-  - MIVisionX: `mivisionx`
-  - RCCL: `rccl`
-
-To make it easier to install ROCm, the AMD binary repositories provide a number of meta-packages that will automatically install multiple other packages.
-For example, `rocm-dkms` is the primary meta-package that is used to install most of the base technology needed for ROCm to operate.
-It will install the `rock-dkms` kernel driver, and another meta-package (`rocm-dev`) which installs most of the user-land ROCm core components, support software, and development tools.
-
-The `rocm-utils` meta-package will install useful utilities that, while not required for ROCm to operate, may still be beneficial to have.
-Finally, the `rocm-libs` meta-package will install some (but not all) of the libraries that are part of ROCm.
-
-The chain of software installed by these meta-packages is illustrated below
-
-```
-  rocm-dkms
-    |--rock-dkms
-    \--rocm-dev
-       |--hsa-rocr-dev
-       |--hsa-ext-rocr-dev
-       |--hsakmt-roct
-       |--hsakmt-roct-dev
-       |--rocm-cmake
-       |--rocm-device-libs
-       |--hcc
-       |--hip_base
-       |--hip_doc
-       |--hip_hcc
-       |--hip_samples
-       |--rocm-smi
-       |--hsa-amd-aqlprofile
-       |--comgr
-       |--rocr_debug_agent
-       \--rocm-utils
-          |--rocminfo
-          \--rocm-clang-ocl # This will cause OpenCL to be installed
-
-  rocm-libs
-    |--rocalution
-    |--hipblas
-    |--rocblas
-    |--rocfft
-    |--rocrand
-    |--hipsparse
-    \--rocsparse
-```
-
-These meta-packages are not required but may be useful to make it easier to install ROCm on most systems.
-Some users may want to skip certain packages. For instance, a user that wants to use the upstream kernel drivers (rather than those supplied by AMD) may want to skip the `rocm-dkms` and `rock-dkms` packages, and instead directly install `rocm-dev`.
-
-Similarly, a user that only wants to install OpenCL support instead of HCC and HIP may want to skip the `rocm-dkms` and `rocm-dev` packages.
-Instead, they could directly install `rock-dkms`, `rocm-opencl`, and `rocm-opencl-dev` and their dependencies.
-
-#### Ubuntu Support - installing from a Debian repository
-
-The following directions show how to install ROCm on supported Debian-based systems such as Ubuntu 18.04.
-These directions may not work as written on unsupported Debian-based distributions.
-For example, newer versions of Ubuntu may not be compatible with the `rock-dkms` kernel driver.
-As such, users may want to skip the `rocm-dkms` and `rock-dkms` packages, as described [above](#rocm-binary-package-structure), and instead [use the upstream kernel driver](#using-debian-based-rocm-with-upstream-kernel-drivers).
-
-##### First make sure your system is up to date
-
-```shell
-sudo apt update
-sudo apt dist-upgrade
-sudo apt install libnuma-dev
-sudo reboot
-```
-
-##### Add the ROCm apt repository
-
-For Debian-based systems like Ubuntu, configure the Debian ROCm repository as
-follows:
-
-```shell
-wget -qO - http://repo.radeon.com/rocm/apt/debian/rocm.gpg.key | sudo apt-key add -
-echo 'deb [arch=amd64] http://repo.radeon.com/rocm/apt/debian/ xenial main' | sudo tee /etc/apt/sources.list.d/rocm.list
-```
-The gpg key might change, so it may need to be updated when installing a new release.
-If the key signature verification is failed while update, please re-add the key from
-ROCm apt repository. The current rocm.gpg.key is not available in a standard key ring
-distribution, but has the following sha1sum hash:
-
-`e85a40d1a43453fe37d63aa6899bc96e08f2817a  rocm.gpg.key`
-
-##### Install
-
-Next, update the apt repository list and install the `rocm-dkms` meta-package:
-
-```shell
-sudo apt update
-sudo apt install rocm-dkms
-```
-
-##### Next set your permissions
-
-Users will need to be in the `video` group in order to have access to the GPU.
-As such, you should ensure that your user account is a member of the `video` group prior to using ROCm.
-You can find which groups you are a member of with the following command:
-
-```shell
-groups
-```
-
-To add yourself to the video group you will need the sudo password and can use the following command:
-
-```shell
-sudo usermod -a -G video $LOGNAME
-```
-
-You may want to ensure that any future users you add to your system are put into the "video" group by default. To do that, you can run the following commands:
-
-```shell
-echo 'ADD_EXTRA_GROUPS=1' | sudo tee -a /etc/adduser.conf
-echo 'EXTRA_GROUPS=video' | sudo tee -a /etc/adduser.conf
-```
-
-Once complete, reboot your system.
-
-##### Test basic ROCm installation
-
-After rebooting the system run the following commands to verify that the ROCm installation was successful. If you see your GPUs listed by both of these commands, you should be ready to go!
-
-```shell
-/opt/rocm/bin/rocminfo
-/opt/rocm/opencl/bin/x86_64/clinfo
-```
-
-Note that, to make running ROCm programs easier, you may wish to put the ROCm binaries in your PATH.
-
-```shell
-echo 'export PATH=$PATH:/opt/rocm/bin:/opt/rocm/profiler/bin:/opt/rocm/opencl/bin/x86_64' | sudo tee -a /etc/profile.d/rocm.sh
-```
-
-If you have an [install issue](https://rocm.github.io/install_issues.html) please read this FAQ.
-
-##### Performing an OpenCL-only Installation of ROCm
-
-Some users may want to install a subset of the full ROCm installation.
-In particular, if you are trying to install on a system with a limited amount of storage space, or which will only run a small collection of known applications, you may want to install only the packages that are required to run OpenCL applications.
-To do that, you can run the following installation command **instead** of the command to install `rocm-dkms`.
-
-```shell
-sudo apt-get install dkms rock-dkms rocm-opencl-dev
-```
-
-##### How to uninstall from Ubuntu 16.04 or Ubuntu 18.04
-
-To uninstall the ROCm packages installed in the above directions, you can execute;
-
-```shell
-sudo apt autoremove rocm-dkms rocm-dev rocm-utils
-```
-
-##### Installing development packages for cross compilation
-
-It is often useful to develop and test on different systems.
-For example, some development or build systems may not have an AMD GPU installed.
-In this scenario, you may prefer to avoid installing the ROCK kernel driver to your development system.
-
-In this case, install the development subset of packages:
-
-```shell
-sudo apt update
-sudo apt install rocm-dev
-```
->**Note:** To execute ROCm enabled apps you will require a system with the full
->ROCm driver stack installed
-
-##### Using Debian-based ROCm with upstream kernel drivers
-
-As described in [the above section about upstream Linux kernel support](#rocm-support-in-upstream-linux-kernels), users may want to try installing ROCm user-level software without installing AMD's custom ROCK kernel driver.
-Users who do want to use upstream kernels can run the following commands instead of installing `rocm-dkms`
-
-```shell
-sudo apt update
-sudo apt install rocm-dev
-echo 'SUBSYSTEM=="kfd", KERNEL=="kfd", TAG+="uaccess", GROUP="video"' | sudo tee /etc/udev/rules.d/70-kfd.rules
-```
-
-#### CentOS/RHEL 7 (7.6) Support
-
-The following directions show how to install ROCm on supported RPM-based systems such as CentOS 7.6.
-These directions may not work as written on unsupported RPM-based distributions.
-For example, Fedora may work but may not be compatible with the `rock-dkms` kernel driver.
-As such, users may want to skip the `rocm-dkms` and `rock-dkms` packages, as described [above](#rocm-binary-package-structure), and instead [use the upstream kernel driver](#using-rpm-based-rocm-with-upstream-kernel-drivers).
-
-Support for CentOS/RHEL 7 was added in ROCm 1.8, but ROCm requires a special
-runtime environment provided by the RHEL Software Collections and additional
-dkms support packages to properly install and run.
-
-##### Preparing RHEL 7 (7.6) for installation
-
-RHEL is a subscription-based operating system, and you must enable several external
-repositories to enable installation of the devtoolset-7 environment and the DKMS
-support files. These steps are not required for CentOS.
-
-First, the subscription for RHEL must be enabled and attached to a pool id. Please
-see Obtaining an RHEL image and license page for instructions on registering your
-system with the RHEL subscription server and attaching to a pool id.
-
-
-Second, enable the following repositories:
-
-```shell
-sudo subscription-manager repos --enable rhel-server-rhscl-7-rpms
-sudo subscription-manager repos --enable rhel-7-server-optional-rpms
-sudo subscription-manager repos --enable rhel-7-server-extras-rpms
-```
-
-Third, enable additional repositories by downloading and installing the epel-release-latest-7 repository RPM:
-
-```shell
-sudo rpm -ivh https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm
-```
-
-##### Install and setup Devtoolset-7
-
-To setup the Devtoolset-7 environment, follow the instructions on this page:
-
-https://www.softwarecollections.org/en/scls/rhscl/devtoolset-7/
-
-Note that devtoolset-7 is a Software Collections package, and it is not supported by AMD.
-
-##### Prepare CentOS/RHEL (7.6) for DKMS Install
-
-Installing kernel drivers on CentOS/RHEL 7.6 requires dkms tool being installed:
-
-```shell
-sudo yum install -y epel-release
-sudo yum install -y dkms kernel-headers-`uname -r` kernel-devel-`uname -r`
-```
-
-##### Installing ROCm on the system
-
-It is recommended to [remove previous ROCm installations](https://github.com/RadeonOpenCompute/ROCm#how-to-uninstall-rocm-from-centosrhel-76) before installing the latest version to ensure a smooth installation.
-
-At this point ROCm can be installed on the target system. Create a /etc/yum.repos.d/rocm.repo file with the following contents:
-
-```shell
-[ROCm]
-name=ROCm
-baseurl=http://repo.radeon.com/rocm/yum/rpm
-enabled=1
-gpgcheck=0
-```
-
-The repo's URL should point to the location of the repositories repodata database. Install ROCm components using these commands:
-
-```shell
-sudo yum install rocm-dkms
-```
-
-The rock-dkms component should be installed and the `/dev/kfd` device should be available on reboot.
-
-##### Set up permissions
-
-Ensure that your user account is a member of the "video" or "wheel" group prior to using the ROCm driver.
-You can find which groups you are a member of with the following command:
-
-```shell
-groups
-```
-
-To add yourself to the video (or wheel) group you will need the sudo password and can use the
-following command:
-
-```shell
-sudo usermod -a -G video $LOGNAME
-```
-
-You may want to ensure that any future users you add to your system are put into the "video" group by default. To do that, you can run the following commands:
-
-```shell
-echo 'ADD_EXTRA_GROUPS=1' | sudo tee -a /etc/adduser.conf
-echo 'EXTRA_GROUPS=video' | sudo tee -a /etc/adduser.conf
-```
-
-Current release supports CentOS/RHEL 7.6. If users want to update the OS version, they should completely remove ROCm packages before updating to the latest version of the OS, to avoid DKMS related issues.
-
-Once complete, reboot your system.
-
-###### Test basic ROCm installation
-
-After rebooting the system run the following commands to verify that the ROCm installation was successful. If you see your GPUs listed by both of these commands, you should be ready to go!
-
-```shell
-/opt/rocm/bin/rocminfo
-/opt/rocm/opencl/bin/x86_64/clinfo
-```
-
-Note that, to make running ROCm programs easier, you may wish to put the ROCm binaries in your PATH.
-
-```shell
-echo 'export PATH=$PATH:/opt/rocm/bin:/opt/rocm/profiler/bin:/opt/rocm/opencl/bin/x86_64' | sudo tee -a /etc/profile.d/rocm.sh
-```
-
-If you have an [install issue](https://rocm.github.io/install_issues.html) please read this FAQ.
-
-###### Performing an OpenCL-only Installation of ROCm
-
-Some users may want to install a subset of the full ROCm installation.
-In particular, if you are trying to install on a system with a limited amount of storage space, or which will only run a small collection of known applications, you may want to install only the packages that are required to run OpenCL applications.
-To do that, you can run the following installation command **instead** of the command to install `rocm-dkms`.
-
-```shell
-sudo yum install rock-dkms rocm-opencl-devel
-```
-
-##### Compiling applications using HCC, HIP, and other ROCm software
-
-To compile applications or samples, please use gcc-7.2 provided by the devtoolset-7 environment.
-To do this, compile all applications after running this command:
-
-```shell
-scl enable devtoolset-7 bash
-```
-##### How to uninstall ROCm from CentOS/RHEL 7.6
-
-To uninstall the ROCm packages installed by the above directions, you can execute:
-
-```shell
-sudo yum autoremove rocm-dkms rock-dkms
-```
-
-##### Installing development packages for cross compilation
-
-It is often useful to develop and test on different systems.
-For example, some development or build systems may not have an AMD GPU installed.
-In this scenario, you may prefer to avoid installing the ROCK kernel driver to your development system.
-
-In this case, install the development subset of packages:
-
-```shell
-sudo yum install rocm-dev
-```
->**Note:** To execute ROCm enabled apps you will require a system with the full
->ROCm driver stack installed
-
-##### Using ROCm with upstream kernel drivers
-
-As described in [the above section about upstream Linux kernel support](#rocm-support-in-upstream-linux-kernels), use
-rs may want to try installing ROCm user-level software without installing AMD's custom ROCK kernel driver.
-Users who do want to use upstream kernels can run the following commands instead of installing `rocm-dkms`
-
-```shell
-sudo yum install rocm-dev
-echo 'SUBSYSTEM=="kfd", KERNEL=="kfd", TAG+="uaccess", GROUP="video"' | sudo tee /etc/udev/rules.d/70-kfd.rules
-```
-
-### Known issues / workarounds
-
-#### rocprofiler --hiptrace and --hsatrace fails to load roctracer library
-In ROCm 2.7, rocprofiler --hiptrace and --hsatrace fails to load roctracer library.  
-The workaround is to create two links under '/opt/rocm':
-$ ln -s /opt/rocm/roctracer/lib/libroctracer64.so /opt/rocm/rocprofiler/lib/libroctracer64.so
-$ ln -s /opt/rocm/roctracer/tool/libtracer_tool.so /opt/rocm/rocprofiler/tool/libtracer_tool.so
-
-
-#### rocFFT unit tests - memory access fault
-Known failure with some power-of-2 size transforms in 1D real FFTs. This issue has been fixed in master branch of public rocFFT repo: https://github.com/ROCmSoftwarePlatform/rocFFT
-
-### Closed source components
-
-The ROCm platform relies on a few closed source components to provide functionality
-such as HSA image support. These components are only available through the ROCm
-repositories, and they will either be deprecated or become open source components in the
-future. These components are made available in the following packages:
-
-*  hsa-ext-rocr-dev
-
-### Getting ROCm source code
-
-ROCm is built from open source software.
-As such, it is possible to make modifications to the various components of ROCm by downloading the source code, making modifications to it, and rebuilding the components.
-The source code for ROCm components can be cloned from each of the GitHub repositories using git.
-In order to make it easier to download the correct versions of each of these tools, this ROCm repository contains a [repo](https://gerrit.googlesource.com/git-repo/) manifest file, [default.xml](default.xml).
-Interested users can thus use this manifest file to download the source code for all of the ROCm software.
-
-#### Installing repo
-
-Google's repo tool allows you to manage multiple git repositories simultaneously.
-You can install it by executing the following example commands:
-
-```shell
-mkdir -p ~/bin/
-curl https://storage.googleapis.com/git-repo-downloads/repo > ~/bin/repo
-chmod a+x ~/bin/repo
-```
-Note that you can choose a different folder to install repo into if you desire. `~/bin/` is simply used as an example.
-
-#### Downloading the ROCm source code
-
-The following example shows how to use the `repo` binary downloaded above to download all of the ROCm source code.
-If you chose a directory other than `~/bin/` to install `repo`, you should use that directory below.
-
-```shell
-mkdir -p ~/ROCm/
-cd ~/ROCm/
-~/bin/repo init -u https://github.com/RadeonOpenCompute/ROCm.git -b roc-2.7.0
-repo sync
-```
-
-This will cause repo to download all of the open source code associated with this ROCm release.
-You may want to ensure that you have ssh-keys configured on your machine for your GitHub ID.
-
-#### Building the ROCm source code
-
-Each ROCm component repository contains directions for building that component.
-As such, you should go to the repository you are interested in building to find how to build it.
-
-That said, AMD also offers [a project](https://github.com/RadeonOpenCompute/Experimental_ROC) that demonstrates how to download, build, package, and install ROCm software on various distributions.
-The scripts here may be useful for anyone looking to build ROCm components.
-
-#### Deprecation Notice
-
-### HCC
-AMD is deprecating HCC to put more focus on HIP development and on
-other languages supporting heterogeneous compute.  We will no longer
-develop any new feature in HCC. We will stop maintaining HCC after
-its final release, which is planned for the end of 2019.  If your
-application was developed with the hc C++ API, we would encourage you
-to transition it to other languages supported by AMD, such as HIP or
-OpenCL.  HIP and hc languages share the same compiler technology, so
-many hc kernel language features (including inline assembly) are also
-available through the HIP compilation path.
-
-### hipThrust
-hip-thrust has been removed in ROCm2.7.
-
-
-### Final notes
-* OpenCL Runtime and Compiler will be submitted to the Khronos Group for conformance testing prior to its final release.
+For release information for older ROCm releases, refer to
+[`CHANGELOG`](./CHANGELOG.md).

RELEASE.md

@@ -0,0 +1,75 @@
+# Release Notes
+<!-- Do not edit this file! This file is autogenerated with -->
+<!--   tools/autotag/tag_script.py                          -->
+
+<!-- Disable lints since this is an auto-generated file.    -->
+<!-- markdownlint-disable blanks-around-headers             -->
+<!-- markdownlint-disable no-duplicate-header               -->
+<!-- markdownlint-disable no-blanks-blockquote              -->
+<!-- markdownlint-disable ul-indent                         -->
+<!-- markdownlint-disable no-trailing-spaces                -->
+
+<!-- spellcheck-disable -->
+
+Welcome to the release notes for the ROCm platform.
+
+-------------------
+
+## ROCm 5.7.1
+<!-- markdownlint-disable first-line-h1 -->
+<!-- markdownlint-disable no-duplicate-header -->
+
+### What's New in This Release
+
+### ROCm Libraries
+
+#### rocBLAS
+A new functionality rocblas-gemm-tune and an environment variable ROCBLAS_TENSILE_GEMM_OVERRIDE_PATH are added to rocBLAS in the ROCm 5.7.1 release.
+
+*rocblas-gemm-tune* is used to find the best-performing GEMM kernel for each GEMM problem set. It has a command line interface, which mimics the --yaml input used by rocblas-bench. To generate the expected --yaml input, profile logging can be used, by setting the environment variable ROCBLAS_LAYER4.
+
+For more information on rocBLAS logging, see Logging in rocBLAS, in the [API Reference Guide](https://rocm.docs.amd.com/projects/rocBLAS/en/docs-5.7.1/API_Reference_Guide.html#logging-in-rocblas).
+
+An example input file: Expected output (note selected GEMM idx may differ): Where the far right values (solution_index) are the indices of the best-performing kernels for those GEMMs in the rocBLAS kernel library. These indices can be directly used in future GEMM calls. See rocBLAS/samples/example_user_driven_tuning.cpp for sample code of directly using kernels via their indices.
+
+If the output is stored in a file, the results can be used to override default kernel selection with the kernels found, by setting the environment variable ROCBLAS_TENSILE_GEMM_OVERRIDE_PATH, where points to the stored file.
+
+For more details, refer to the [rocBLAS Programmer's Guide.](https://rocm.docs.amd.com/projects/rocBLAS/en/latest/Programmers_Guide.html#rocblas-gemm-tune)
+
+#### HIP 5.7.1 (for ROCm 5.7.1)
+
+ROCm 5.7.1 is a point release with several bug fixes in the HIP runtime.
+
+### Fixed defects
+The *hipPointerGetAttributes* API returns the correct HIP memory type as *hipMemoryTypeManaged* for managed memory.
+
+### Library Changes in ROCM 5.7.1
+
+| Library | Version |
+|---------|---------|
+| hipBLAS | [1.1.0](https://github.com/ROCmSoftwarePlatform/hipBLAS/releases/tag/rocm-5.7.1) |
+| hipCUB | [2.13.1](https://github.com/ROCmSoftwarePlatform/hipCUB/releases/tag/rocm-5.7.1) |
+| hipFFT | [1.0.12](https://github.com/ROCmSoftwarePlatform/hipFFT/releases/tag/rocm-5.7.1) |
+| hipSOLVER | 1.8.1 ⇒ [1.8.2](https://github.com/ROCmSoftwarePlatform/hipSOLVER/releases/tag/rocm-5.7.1) |
+| hipSPARSE | [2.3.8](https://github.com/ROCmSoftwarePlatform/hipSPARSE/releases/tag/rocm-5.7.1) |
+| MIOpen | [2.19.0](https://github.com/ROCmSoftwarePlatform/MIOpen/releases/tag/rocm-5.7.1) |
+| rocALUTION | [2.1.11](https://github.com/ROCmSoftwarePlatform/rocALUTION/releases/tag/rocm-5.7.1) |
+| rocBLAS | [3.1.0](https://github.com/ROCmSoftwarePlatform/rocBLAS/releases/tag/rocm-5.7.1) |
+| rocFFT | [1.0.24](https://github.com/ROCmSoftwarePlatform/rocFFT/releases/tag/rocm-5.7.1) |
+| rocm-cmake | [0.10.0](https://github.com/RadeonOpenCompute/rocm-cmake/releases/tag/rocm-5.7.1) |
+| rocPRIM | [2.13.1](https://github.com/ROCmSoftwarePlatform/rocPRIM/releases/tag/rocm-5.7.1) |
+| rocRAND | [2.10.17](https://github.com/ROCmSoftwarePlatform/rocRAND/releases/tag/rocm-5.7.1) |
+| rocSOLVER | [3.23.0](https://github.com/ROCmSoftwarePlatform/rocSOLVER/releases/tag/rocm-5.7.1) |
+| rocSPARSE | [2.5.4](https://github.com/ROCmSoftwarePlatform/rocSPARSE/releases/tag/rocm-5.7.1) |
+| rocThrust | [2.18.0](https://github.com/ROCmSoftwarePlatform/rocThrust/releases/tag/rocm-5.7.1) |
+| rocWMMA | [1.2.0](https://github.com/ROCmSoftwarePlatform/rocWMMA/releases/tag/rocm-5.7.1) |
+| Tensile | [4.38.0](https://github.com/ROCmSoftwarePlatform/Tensile/releases/tag/rocm-5.7.1) |
+
+#### hipSOLVER 1.8.2
+
+hipSOLVER 1.8.2 for ROCm 5.7.1
+
+##### Fixed
+
+- Fixed conflicts between the hipsolver-dev and -asan packages by excluding
+  hipsolver_module.f90 from the latter

default.xml

@@ -1,74 +1,79 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <manifest>
-
     <remote name="roc-github"
-            fetch="http://github.com/RadeonOpenCompute/" />
+fetch="https://github.com/RadeonOpenCompute/" />
     <remote name="rocm-devtools"
-            fetch="https://github.com/ROCm-Developer-Tools/" />
+fetch="https://github.com/ROCm-Developer-Tools/" />
     <remote name="rocm-swplat"
-            fetch="https://github.com/ROCmSoftwarePlatform/" />
+fetch="https://github.com/ROCmSoftwarePlatform/" />
     <remote name="gpuopen-libs"
-            fetch="https://github.com/GPUOpen-ProfessionalCompute-Libraries/" />
+fetch="https://github.com/GPUOpen-ProfessionalCompute-Libraries/" />
     <remote name="gpuopen-tools"
-            fetch="https://github.com/GPUOpen-Tools/" />
-
-    <default revision="refs/tags/roc-2.7.0"
-             remote="roc-github"
-             sync-c="true"
-             sync-j="4" />
-
+fetch="https://github.com/GPUOpen-Tools/" />
+    <remote name="KhronosGroup"
+fetch="https://github.com/KhronosGroup/" />
+    <default revision="refs/tags/rocm-5.7.1"
+     remote="roc-github"
+     sync-c="true"
+     sync-j="4" />
+<!--list of projects for ROCM-->
     <project name="ROCK-Kernel-Driver" />
     <project name="ROCT-Thunk-Interface" />
     <project name="ROCR-Runtime" />
-    <project name="ROC-smi" />
-    <project name="rocm-cmake" revision="master-rocm-2.7" />
+    <project name="amdsmi" />
+    <project name="rocm_smi_lib" />
+    <project name="rocm-core" />
+    <project name="rocm-cmake" />
     <project name="rocminfo" />
-    <project name="rocprofiler" remote="rocm-devtools" revision="refs/tags/roc-2.6.0"/>
-    <project name="roctracer" remote="rocm-devtools" revision="roc-2.7.x"/>
-    <!-- If you want to get the full OpenCL runtime, there is a separate repo
-         manifest that is more authoritative than the copy in this file. It can
-         be found at the following URL:
-         https://github.com/RadeonOpenCompute/ROCm-OpenCL-Runtime/blob/roc-2.0.0/opencl.xml -->
-    <remote name="KhronosGroup" fetch="https://github.com/KhronosGroup/" />
-    <project name="ROCm-OpenCL-Runtime" />
-    <project path="ROCm-OpenCL-Runtime/compiler/driver" name="ROCm-OpenCL-Driver"/>
-    <project path="ROCm-OpenCL-Runtime/compiler/llvm" name="llvm" revision="refs/tags/roc-ocl-2.7.0" />
-    <project path="ROCm-OpenCL-Runtime/compiler/llvm/tools/clang" name="clang" />
-    <project path="ROCm-OpenCL-Runtime/compiler/llvm/tools/lld" name="lld" revision="refs/tags/roc-ocl-2.7.0" />
-    <project path="ROCm-OpenCL-Runtime/library/amdgcn" name="ROCm-Device-Libs" revision="refs/tags/roc-ocl-2.7.0" />
-    <project path="ROCm-OpenCL-Runtime/api/opencl/khronos/icd" name="OpenCL-ICD-Loader" remote="KhronosGroup" revision="261c1288aadd9dcc4637aca08332f603e6c13715" />
-
+    <project name="rocm_bandwidth_test" />
+    <project name="rocprofiler" remote="rocm-devtools" />
+    <project name="roctracer" remote="rocm-devtools" />
+    <project path="ROCm-OpenCL-Runtime/api/opencl/khronos/icd" name="OpenCL-ICD-Loader" remote="KhronosGroup" revision="6c03f8b58fafd9dd693eaac826749a5cfad515f8" />
     <project name="clang-ocl" />
-    <!-- HCC needs to be recursively synced to get it submodules -->
-    <project name="hcc" sync-s="true" revision="refs/tags/roc-hcc-2.7.0" />
-    <project name="HCC-Example-Application" remote="rocm-devtools" revision="ffd6533305e79eed667badd3c4cdb7879a1281b8" />
+    <project name="rdc" />
+<!--HIP Projects-->
     <project name="HIP" remote="rocm-devtools" />
     <project name="HIP-Examples" remote="rocm-devtools" />
-    <!-- The following projects are all associated with the AMDGPU LLVM compiler -->
-    <project name="llvm" path="llvm_amd-common" revision="refs/tags/roc-ocl-2.7.0" />
-    <project name="lld" path="llvm_amd-common/lld" revision="refs/tags/roc-ocl-2.7.0" />
-    <project name="clang" path="llvm_amd-common/clang" />
-    <project name="ROCm-Device-Libs" revision="refs/tags/roc-ocl-2.7.0" />
-    <project name="atmi" revision="refs/tags/rocm_2.7.0" />
+    <project name="clr" remote="rocm-devtools" />
+    <project name="HIPIFY" remote="rocm-devtools" />
+    <project name="HIPCC" remote="rocm-devtools" />
+<!-- The following projects are all associated with the AMDGPU LLVM compiler -->
+    <project name="llvm-project" />
+    <project name="ROCm-Device-Libs" />
     <project name="ROCm-CompilerSupport" />
+    <project name="half" remote="rocm-swplat" revision="37742ce15b76b44e4b271c1e66d13d2fa7bd003e" />
+<!-- gdb projects -->
+    <project name="ROCgdb" remote="rocm-devtools" />
+    <project name="ROCdbgapi" remote="rocm-devtools" />
     <project name="rocr_debug_agent" remote="rocm-devtools" />
-    <project name="rocm_bandwidth_test" />
-    <project name="RCP" remote="gpuopen-tools" revision="3a49405a1500067c49d181844ec90aea606055bb" />
-
-    <!-- ROCm Libraries -->
-    <project name="rocBLAS" remote="rocm-swplat" revision="refs/tags/rocm-2.7" />
-    <project name="hipBLAS" remote="rocm-swplat" revision="refs/tags/rocm-2.7" />
-    <project name="rocFFT" remote="rocm-swplat" revision="refs/tags/rocm-2.7" />
-    <project name="rocRAND" remote="rocm-swplat" revision="refs/tags/rocm-2.7" />
-    <project name="rocSPARSE" remote="rocm-swplat" revision="refs/tags/rocm-2.7" />
-    <project name="hipSPARSE" remote="rocm-swplat" revision="refs/tags/rocm-2.7" />
-    <project name="rocALUTION" remote="rocm-swplat" revision="refs/tags/rocm-2.7" />
-    <project name="MIOpenGEMM" remote="rocm-swplat" revision="9547fb9e8499a5a9f16da83b1e6b749de82dd9fb" />
-    <project name="MIOpen" remote="rocm-swplat" revision="refs/tags/roc-2.7.0" />
-    <project name="rocm_smi_lib" />
-    <project name="rccl" remote="rocm-swplat" revision="refs/tags/2.7.0" />
-    <project name="MIVisionX" remote="gpuopen-libs" revision="refs/tags/1.3.0" />
-    <project name="rocThrust" remote="rocm-swplat" revision="refs/tags/rocm-2.7" />
-    <project name="hipCUB" remote="rocm-swplat" revision="refs/tags/rocm-2.7" />
-
+<!-- ROCm Libraries -->
+    <project groups="mathlibs" name="rocBLAS" remote="rocm-swplat" />
+    <project groups="mathlibs" name="Tensile" remote="rocm-swplat" />
+    <project groups="mathlibs" name="hipTensor" remote="rocm-swplat" />
+    <project groups="mathlibs" name="hipBLAS" remote="rocm-swplat" />
+    <project groups="mathlibs" name="rocFFT" remote="rocm-swplat" />
+    <project groups="mathlibs" name="hipFFT" remote="rocm-swplat" />
+    <project groups="mathlibs" name="rocRAND" remote="rocm-swplat" />
+    <project groups="mathlibs" name="rocSPARSE" remote="rocm-swplat" />
+    <project groups="mathlibs" name="rocSOLVER" remote="rocm-swplat" />
+    <project groups="mathlibs" name="hipSOLVER" remote="rocm-swplat" />
+    <project groups="mathlibs" name="hipSPARSE" remote="rocm-swplat" />
+    <project groups="mathlibs" name="rocALUTION" remote="rocm-swplat" />
+    <project groups="mathlibs" name="rocThrust" remote="rocm-swplat" />
+    <project groups="mathlibs" name="hipCUB" remote="rocm-swplat" />
+    <project groups="mathlibs" name="rocPRIM" remote="rocm-swplat" />
+    <project groups="mathlibs" name="rocWMMA" remote="rocm-swplat" />
+    <project groups="mathlibs" name="rccl" remote="rocm-swplat" />
+    <project name="rocMLIR" remote="rocm-swplat" />
+    <project name="MIOpen" remote="rocm-swplat" />
+    <project name="composable_kernel" remote="rocm-swplat" />
+    <project name="MIVisionX" remote="gpuopen-libs" />
+    <project name="rpp" remote="gpuopen-libs" />
+    <project name="hipfort" remote="rocm-swplat" />
+    <project name="AMDMIGraphX" remote="rocm-swplat" />
+    <project name="ROCmValidationSuite" remote="rocm-devtools" />
+<!-- Projects for OpenMP-Extras -->
+    <project name="aomp" path="openmp-extras/aomp" remote="rocm-devtools" />
+    <project name="aomp-extras" path="openmp-extras/aomp-extras" remote="rocm-devtools" />
+    <project name="flang" path="openmp-extras/flang" remote="rocm-devtools" />
 </manifest>

docs/about/compatibility/3rd-party-support-matrix.md

@@ -0,0 +1,63 @@
+# Third party support matrix
+
+ROCm™ supports various 3rd party libraries and frameworks. Supported versions
+are tested and known to work. Non-supported versions of 3rd parties may also
+work, but aren't tested.
+
+## Deep learning
+
+ROCm releases support the most recent and two prior releases of PyTorch and
+TensorFlow.
+
+| ROCm  | [PyTorch](https://github.com/pytorch/pytorch/releases/) | [TensorFlow](https://github.com/tensorflow/tensorflow/releases/) |
+|:------|:--------------------------:|:--------------------:|
+| 5.0.2 | 1.8,  1.9,  1.10           | 2.6, 2.7, 2.8        |
+| 5.1.3 | 1.9,  1.10, 1.11           | 2.7, 2.8, 2.9        |
+| 5.2.x | 1.10, 1.11, 1.12           | 2.8, 2.9, 2.9        |
+| 5.3.x | 1.10.1, 1.11, 1.12.1, 1.13 | 2.8, 2.9, 2.10       |
+| 5.4.x | 1.10.1, 1.11, 1.12.1, 1.13 | 2.8, 2.9, 2.10, 2.11 |
+| 5.5.x | 1.10.1, 1.11, 1.12.1, 1.13 | 2.10, 2.11, 2.13     |
+| 5.6.x | 1.12.1, 1.13, 2.0          | 2.12, 2.13           |
+| 5.7.x | 1.12.1, 1.13, 2.0          | 2.12, 2.13           |
+
+(communication-libraries)=
+
+## Communication libraries
+
+ROCm supports [OpenUCX](https://openucx.org/), an open-source,
+production-grade communication framework for data-centric and high performance
+applications.
+
+UCX version | ROCm 5.4 and older | ROCm 5.5 and newer |
+|:----------|:------------------:|:------------------:|
+| -1.14.0   | COMPATIBLE         | INCOMPATIBLE       |
+|  1.14.1+  | COMPATIBLE         | COMPATIBLE         |
+
+The Unified Collective Communication ([UCC](https://github.com/openucx/ucc)) library also has
+support for ROCm devices.
+
+UCC version | ROCm 5.5 and older | ROCm 5.6 and newer |
+|:----------|:------------------:|:------------------:|
+| -1.1.0    | COMPATIBLE         | INCOMPATIBLE       |
+|  1.2.0+   | COMPATIBLE         | COMPATIBLE         |
+
+## Algorithm libraries
+
+ROCm releases provide algorithm libraries with interfaces compatible with
+contemporary CUDA / NVIDIA HPC SDK alternatives.
+
+* Thrust → rocThrust
+* CUB → hipCUB
+
+| ROCm  | Thrust / CUB | HPC SDK |
+|:------|:------------:|:-------:|
+| 5.0.2 | 1.14         | 21.9       |
+| 5.1.3 | 1.15         | 22.1       |
+| 5.2.x | 1.15         | 22.2, 22.3 |
+| 5.3.x | 1.16         | 22.7       |
+| 5.4.x | 1.16         | 22.9       |
+| 5.5.x | 1.17         | 22.9       |
+| 5.6.x | 1.17.2       | 22.9       |
+| 5.7.x | 1.17.2       | 22.9       |
+
+For the latest documentation of these libraries, refer to [API libraries](../../reference/library-index.md).

docs/about/compatibility/docker-image-support-matrix.rst

@@ -0,0 +1,130 @@
+******************************************************************
+Docker image support matrix
+******************************************************************
+
+AMD validates and publishes `PyTorch <https://hub.docker.com/r/rocm/pytorch>`_ and
+`TensorFlow <https://hub.docker.com/r/rocm/tensorflow>`_ containers on dockerhub. The following
+tags, and associated inventories, are validated with ROCm 5.7.
+
+.. tab-set::
+
+    .. tab-item:: PyTorch
+
+        .. tab-set::
+
+            .. tab-item:: Ubuntu 22.04
+
+                Tag: `rocm/pytorch:rocm5.7_ubuntu22.04_py3.10_pytorch_2.0.1 <https://hub.docker.com/layers/rocm/pytorch/rocm5.7_ubuntu22.04_py3.10_pytorch_2.0.1/images/sha256-21df283b1712f3d73884b9bc4733919374344ceacb694e8fbc2c50bdd3e767ee>`_
+
+                * Inventory:
+
+                    * `ROCm 5.7 <https://repo.radeon.com/rocm/apt/5.7/>`_
+                    * `Python 3.10 <https://www.python.org/downloads/release/python-31013/>`_
+                    * `Torch 2.0.1 <https://github.com/ROCmSoftwarePlatform/pytorch/tree/release/2.0>`_
+                    * `Apex 0.1 <https://github.com/ROCmSoftwarePlatform/apex/tree/v0.1>`_
+                    * `Torchvision 0.15.0 <https://github.com/pytorch/vision/tree/release/0.15>`_
+                    * `Tensorboard 2.14.0 <https://github.com/tensorflow/tensorboard/tree/2.14>`_
+                    * `MAGMA <https://bitbucket.org/icl/magma/src/master/>`_
+                    * `UCX 1.10.0 <https://github.com/openucx/ucx/tree/v1.10.0>`_
+                    * `OMPI 4.0.3 <https://github.com/open-mpi/ompi/tree/v4.0.3>`_
+                    * `OFED 5.4.3 <https://content.mellanox.com/ofed/MLNX_OFED-5.3-1.0.5.0/MLNX_OFED_LINUX-5.3-1.0.5.0-ubuntu20.04-x86_64.tgz>`_
+
+            .. tab-item:: Ubuntu 20.04
+
+                Tag: `rocm/pytorch:rocm5.7_ubuntu20.04_py3.9_pytorch_staging <https://hub.docker.com/layers/rocm/pytorch/rocm5.7_ubuntu20.04_py3.9_pytorch_2.0.1/images/sha256-4dd86046e5f777f53ae40a75ecfc76a5e819f01f3b2d40eacbb2db95c2f971d4)>`_
+
+                * Inventory:
+
+                    * `ROCm 5.7 <https://repo.radeon.com/rocm/apt/5.7/>`_
+                    * `Python 3.9 <https://www.python.org/downloads/release/python-3918/>`_
+                    * `Torch 2.1.0 <https://github.com/ROCmSoftwarePlatform/pytorch/tree/rocm5.7_internal_testing>`_
+                    * `Apex 0.1 <https://github.com/ROCmSoftwarePlatform/apex/tree/v0.1>`_
+                    * `Torchvision 0.16.0 <https://github.com/pytorch/vision/tree/release/0.16>`_
+                    * `Tensorboard 2.14.0 <https://github.com/tensorflow/tensorboard/tree/2.14>`_
+                    * `MAGMA <https://bitbucket.org/icl/magma/src/master/>`_
+                    * `UCX 1.10.0 <https://github.com/openucx/ucx/tree/v1.10.0>`_
+                    * `OMPI 4.0.3 <https://github.com/open-mpi/ompi/tree/v4.0.3>`_
+                    * `OFED 5.4.3 <https://content.mellanox.com/ofed/MLNX_OFED-5.3-1.0.5.0/MLNX_OFED_LINUX-5.3-1.0.5.0-ubuntu20.04-x86_64.tgz>`_
+
+
+                Tag: `Ubuntu rocm/pytorch:rocm5.7_ubuntu20.04_py3.9_pytorch_1.12.1 <https://hub.docker.com/layers/rocm/pytorch/rocm5.7_ubuntu20.04_py3.9_pytorch_1.12.1/images/sha256-e67db9373c045a7b6defd43cc3d067e7d49fd5d380f3f8582d2fb219c1756e1f>`_
+
+                * Inventory:
+
+                    * `ROCm 5.7 <https://repo.radeon.com/rocm/apt/5.7/>`_
+                    * `Python 3.9 <https://www.python.org/downloads/release/python-3918/>`_
+                    * `Torch 1.12.1 <https://github.com/ROCmSoftwarePlatform/pytorch/tree/release/1.12>`_
+                    * `Apex 0.1 <https://github.com/ROCmSoftwarePlatform/apex/tree/v0.1>`_
+                    * `Torchvision 0.13.1 <https://github.com/pytorch/vision/tree/v0.13.1>`_
+                    * `Tensorboard 2.14.0 <https://github.com/tensorflow/tensorboard/tree/2.14>`_
+                    * `MAGMA <https://bitbucket.org/icl/magma/src/master/>`_
+                    * `UCX 1.10.0 <https://github.com/openucx/ucx/tree/v1.10.0>`_
+                    * `OMPI 4.0.3 <https://github.com/open-mpi/ompi/tree/v4.0.3>`_
+                    * `OFED 5.4.3 <https://content.mellanox.com/ofed/MLNX_OFED-5.3-1.0.5.0/MLNX_OFED_LINUX-5.3-1.0.5.0-ubuntu20.04-x86_64.tgz>`_
+
+                Tag: `Ubuntu rocm/pytorch:rocm5.7_ubuntu20.04_py3.9_pytorch_1.13.1 <https://hub.docker.com/layers/rocm/pytorch/rocm5.7_ubuntu20.04_py3.9_pytorch_1.13.1/images/sha256-ed99d159026093d2aaf5c48c1e4b0911508773430377051372733f75c340a4c1>`_
+
+                * Inventory:
+
+                    * `ROCm 5.7 <https://repo.radeon.com/rocm/apt/5.7/>`_
+                    * `Python 3.9 <https://www.python.org/downloads/release/python-3918/>`_
+                    * `Torch 1.12.1 <https://github.com/ROCmSoftwarePlatform/pytorch/tree/release/1.13>`_
+                    * `Apex 0.1 <https://github.com/ROCmSoftwarePlatform/apex/tree/v0.1>`_
+                    * `Torchvision 0.14.0 <https://github.com/pytorch/vision/tree/v0.14.0>`_
+                    * `Tensorboard 2.12.0 <https://github.com/tensorflow/tensorboard/tree/2.12.0>`_
+                    * `MAGMA <https://bitbucket.org/icl/magma/src/master/>`_
+                    * `UCX 1.10.0 <https://github.com/openucx/ucx/tree/v1.10.0>`_
+                    * `OMPI 4.0.3 <https://github.com/open-mpi/ompi/tree/v4.0.3>`_
+                    * `OFED 5.4.3 <https://content.mellanox.com/ofed/MLNX_OFED-5.3-1.0.5.0/MLNX_OFED_LINUX-5.3-1.0.5.0-ubuntu20.04-x86_64.tgz>`_
+
+                Tag: `Ubuntu rocm/pytorch:rocm5.7_ubuntu20.04_py3.9_pytorch_2.0.1 <https://hub.docker.com/layers/rocm/pytorch/rocm5.7_ubuntu20.04_py3.9_pytorch_2.0.1/images/sha256-4dd86046e5f777f53ae40a75ecfc76a5e819f01f3b2d40eacbb2db95c2f971d4>`_
+
+                * Inventory:
+
+                    * `ROCm 5.7 <https://repo.radeon.com/rocm/apt/5.7/>`_
+                    * `Python 3.9 <https://www.python.org/downloads/release/python-3918/>`_
+                    * `Torch 2.0.1 <https://github.com/ROCmSoftwarePlatform/pytorch/tree/release/2.0>`_
+                    * `Apex 0.1 <https://github.com/ROCmSoftwarePlatform/apex/tree/v0.1>`_
+                    * `Torchvision 0.15.2 <https://github.com/pytorch/vision/tree/release/0.15>`_
+                    * `Tensorboard 2.14.0 <https://github.com/tensorflow/tensorboard/tree/2.14>`_
+                    * `MAGMA <https://bitbucket.org/icl/magma/src/master/>`_
+                    * `UCX 1.10.0 <https://github.com/openucx/ucx/tree/v1.10.0>`_
+                    * `OMPI 4.0.3 <https://github.com/open-mpi/ompi/tree/v4.0.3>`_
+                    * `OFED 5.4.3 <https://content.mellanox.com/ofed/MLNX_OFED-5.3-1.0.5.0/MLNX_OFED_LINUX-5.3-1.0.5.0-ubuntu20.04-x86_64.tgz>`_
+
+            .. tab-item:: CentOS 7
+
+                Tag: `rocm/pytorch:rocm5.7_centos7_py3.9_pytorch_staging <https://hub.docker.com/layers/rocm/pytorch/rocm5.7_centos7_py3.9_pytorch_staging/images/sha256-92240cdf0b4aa7afa76fc78be995caa19ee9c54b5c9f1683bdcac28cedb58d2b>`_
+
+                * Inventory:
+
+                * `ROCm 5.7 <https://repo.radeon.com/rocm/yum/5.7/>`_
+                * `Python 3.9 <https://www.python.org/downloads/release/python-3918/>`_
+                * `Torch 2.1.0 <https://github.com/ROCmSoftwarePlatform/pytorch/tree/rocm5.7_internal_testing>`_
+                * `Apex 0.1 <https://github.com/ROCmSoftwarePlatform/apex/tree/v0.1>`_
+                * `Torchvision 0.16.0 <https://github.com/pytorch/vision/tree/release/0.16>`_
+                * `MAGMA <https://bitbucket.org/icl/magma/src/master/>`_
+
+    .. tab-item:: TensorFlow
+
+        .. tab-set::
+
+            .. tab-item:: Ubuntu 20.04
+
+                Tag: `rocm5.7-tf2.12-dev <https://hub.docker.com/layers/rocm/tensorflow/rocm5.7-tf2.12-dev/images/sha256-e0ac4d49122702e5167175acaeb98a79b9500f585d5e74df18facf6b52ce3e59>`_
+
+                * Inventory:
+
+                    * `ROCm 5.7 <https://repo.radeon.com/rocm/apt/5.7/>`_
+                    * `Python 3.9 <https://www.python.org/downloads/release/python-3918/>`_
+                    * `tensorflow-rocm 2.12.1 <https://pypi.org/project/tensorflow-rocm/2.12.1.570/>`_
+                    * `Tensorboard 2.12.3 <https://github.com/tensorflow/tensorboard/tree/2.12>`_
+
+                Tag: `rocm5.7-tf2.13-dev <https://hub.docker.com/layers/rocm/tensorflow/rocm5.7-tf2.13-dev/images/sha256-6f995539eebc062aac2b53db40e2b545192d8b032d0deada8c24c6651a7ac332>`_
+
+                * Inventory:
+
+                    * `ROCm 5.7 <https://repo.radeon.com/rocm/apt/5.7/>`_
+                    * `Python 3.9 <https://www.python.org/downloads/release/python-3918/>`_
+                    * `tensorflow-rocm 2.13.0 <https://pypi.org/project/tensorflow-rocm/2.13.0.570/>`_
+                    * `Tensorboard 2.13.0 <https://github.com/tensorflow/tensorboard/tree/2.13>`_

docs/about/compatibility/linux-support.md

@@ -0,0 +1,116 @@
+# GPU and OS support (Linux)
+
+(linux-support)=
+
+## Supported Linux distributions
+
+AMD ROCm™ Platform supports the following Linux distributions.
+
+::::{tab-set}
+
+:::{tab-item} Supported
+
+| Distribution | Processor Architectures | Validated Kernel | Support |
+| :----------- | :---------------------: | :--------------: | ------: |
+| RHEL 9.2       | x86-64 | 5.14 (5.14.0-284.11.1.el9_2.x86_64)        | ✅ |
+| RHEL 9.1       | x86-64 | 5.14.0-284.11.1.el9_2.x86_64             | ✅ |
+| RHEL 8.8       | x86-64 | 4.18.0-477.el8.x86_64        | ✅ |
+| RHEL 8.7       | x86-64 | 4.18.0-425.10.1.el8_7.x86_64              | ✅ |
+| SLES 15 SP5    | x86-64 |  5.14.21-150500.53-default       | ✅ |
+| SLES 15 SP4    | x86-64 | 5.14.21-150400.24.63-default               | ✅ |
+| Ubuntu 22.04.2 | x86-64 | 5.19.0-45-generic | ✅ |
+| Ubuntu 20.04.5 | x86-64 | 5.15.0-75-generic          | ✅ |
+
+:::{versionadded} 5.6
+
+* RHEL 8.8 and 9.2 support is added.
+* SLES 15 SP5 support is added
+
+:::
+
+:::{tab-item} Unsupported
+
+| Distribution | Processor Architectures | Validated Kernel | Support |
+| :----------- | :---------------------: | :--------------: | ------: |
+| RHEL 9.0       | x86-64 | 5.14               | ❌ |
+| RHEL 8.6       | x86-64 | 5.14               | ❌ |
+| SLES 15 SP3    | x86-64 | 5.3                | ❌ |
+| Ubuntu 22.04.0 | x86-64 | 5.15 LTS, 5.17 OEM | ❌ |
+| Ubuntu 20.04.4 | x86-64 | 5.13 HWE, 5.13 OEM | ❌ |
+| Ubuntu 22.04.1 | x86-64 | 5.15 LTS           | ❌ |
+
+:::
+
+::::
+
+✅: **Supported** - AMD performs full testing of all ROCm components on distro
+  GA image.
+❌: **Unsupported** - AMD no longer performs builds and testing on these
+  previously supported distro GA images.
+
+## Virtualization support
+
+ROCm supports virtualization for select GPUs only as shown below.
+
+| Hypervisor     | Version  | GPU   | Validated Guest OS (validated kernel)                                            |
+|----------------|----------|-------|----------------------------------------------------------------------------------|
+| VMWare         | ESXi 8   | MI250 | Ubuntu 20.04 (`5.15.0-56-generic`)                                               |
+| VMWare         | ESXi 8   | MI210 | Ubuntu 20.04 (`5.15.0-56-generic`), SLES 15 SP4 (`5.14.21-150400.24.18-default`) |
+| VMWare         | ESXi 7   | MI210 | Ubuntu 20.04 (`5.15.0-56-generic`), SLES 15 SP4 (`5.14.21-150400.24.18-default`) |
+
+## Linux-supported GPUs
+
+The table below shows supported GPUs for Instinct™, Radeon Pro™ and Radeon™
+GPUs. Please click the tabs below to switch between GPU product lines. If a GPU
+is not listed on this table, the GPU is not officially supported by AMD.
+
+:::::{tab-set}
+
+::::{tab-item} AMD Instinct™
+:sync: instinct
+
+| Product Name | Architecture | [LLVM Target](https://www.llvm.org/docs/AMDGPUUsage.html#processors) |Support |
+|:------------:|:------------:|:--------------------------------------------------------------------:|:-------:|
+| AMD Instinct™ MI250X | CDNA2  | gfx90a | ✅ |
+| AMD Instinct™ MI250  | CDNA2  | gfx90a | ✅ |
+| AMD Instinct™ MI210  | CDNA2  | gfx90a | ✅ |
+| AMD Instinct™ MI100  | CDNA   | gfx908 | ✅ |
+| AMD Instinct™ MI50   | GCN5.1 | gfx906 | ✅ |
+| AMD Instinct™ MI25   | GCN5.0 | gfx900 | ❌ |
+
+::::
+
+::::{tab-item} Radeon Pro™
+:sync: radeonpro
+
+| Name | Architecture |[LLVM Target](https://www.llvm.org/docs/AMDGPUUsage.html#processors) | Support|
+|:----:|:------------:|:--------------------------------------------------------------------:|:-------:|
+| AMD Radeon™ Pro W7900   | RDNA3  | gfx1100 | ✅ (Ubuntu 22.04 only)|
+| AMD Radeon™ Pro W6800   | RDNA2  | gfx1030 | ✅ |
+| AMD Radeon™ Pro V620    | RDNA2  | gfx1030 | ✅ |
+| AMD Radeon™ Pro VII     | GCN5.1 | gfx906  | ✅ |
+::::
+
+::::{tab-item} Radeon™
+:sync: radeonpro
+
+| Name | Architecture    |[LLVM Target](https://www.llvm.org/docs/AMDGPUUsage.html#processors) | Support|
+|:----:|:---------------:|:--------------------------------------------------------------------:|:-------:|
+| AMD Radeon™ RX 7900 XTX | RDNA3 | gfx1100  | ✅ (Ubuntu 22.04 only)|
+| AMD Radeon™ VII        | GCN5.1 | gfx906  | ✅ |
+
+::::
+:::::
+
+### Support status
+
+✅: **Supported** - AMD enables these GPUs in our software distributions for
+  the corresponding ROCm product.
+⚠️: **Deprecated** - Support will be removed in a future release.
+❌: **Unsupported** - This configuration is not enabled in our software
+  distributions.
+
+## CPU support
+
+ROCm requires CPUs that support PCIe™ atomics. Modern CPUs after the release of
+1st generation AMD Zen CPU and Intel™ Haswell support PCIe atomics.

docs/about/compatibility/openmp.md

@@ -0,0 +1,474 @@
+# OpenMP support in ROCm
+
+## Introduction
+
+The ROCm™ installation includes an LLVM-based implementation that fully supports
+the OpenMP 4.5 standard and a subset of OpenMP 5.0, 5.1, and 5.2 standards.
+Fortran, C/C++ compilers, and corresponding runtime libraries are included.
+Along with host APIs, the OpenMP compilers support offloading code and data onto
+GPU devices. This document briefly describes the installation location of the
+OpenMP toolchain, example usage of device offloading, and usage of `rocprof`
+with OpenMP applications. The GPUs supported are the same as those supported by
+this ROCm release. See the list of supported GPUs for [Linux](../../about/compatibility/linux-support.md) and [Windows](../../about/compatibility/windows-support.md).
+
+The ROCm OpenMP compiler is implemented using LLVM compiler technology.
+The following image illustrates the internal steps taken to translate a user’s application into an executable that can offload computation to the AMDGPU. The compilation is a two-pass process. Pass 1 compiles the application to generate the CPU code and Pass 2 links the CPU code to the AMDGPU device code.
+
+![OpenMP toolchain](../../data/reference/openmp/openmp-toolchain.svg "OpenMP toolchain")
+
+### Installation
+
+The OpenMP toolchain is automatically installed as part of the standard ROCm
+installation and is available under `/opt/rocm-{version}/llvm`. The
+sub-directories are:
+
+* bin: Compilers (`flang` and `clang`) and other binaries.
+* examples: The usage section below shows how to compile and run these programs.
+* include: Header files.
+* lib: Libraries including those required for target offload.
+* lib-debug: Debug versions of the above libraries.
+
+## OpenMP: usage
+
+The example programs can be compiled and run by pointing the environment
+variable `ROCM_PATH` to the ROCm install directory.
+
+**Example:**
+
+```bash
+export ROCM_PATH=/opt/rocm-{version}
+cd $ROCM_PATH/share/openmp-extras/examples/openmp/veccopy
+sudo make run
+```
+
+```{note}
+`sudo` is required since we are building inside the `/opt` directory.
+Alternatively, copy the files to your home directory first.
+```
+
+The above invocation of Make compiles and runs the program. Note the options
+that are required for target offload from an OpenMP program:
+
+```bash
+-fopenmp --offload-arch=<gpu-arch>
+```
+
+```{note}
+The compiler also accepts the alternative offloading notation:
+
+```bash
+-fopenmp -fopenmp-targets=amdgcn-amd-amdhsa -Xopenmp-target=amdgcn-amd-amdhsa -march=<gpu-arch>
+```
+
+Obtain the value of `gpu-arch` by running the following command:
+
+```bash
+% /opt/rocm-{version}/bin/rocminfo | grep gfx
+```
+
+[//]: # (dated link below, needs updating)
+
+See the complete list of compiler command-line references
+[here](https://github.com/RadeonOpenCompute/llvm-project/blob/amd-stg-open/clang/docs/CommandGuide/clang.rst).
+
+### Using `rocprof` with OpenMP
+
+The following steps describe a typical workflow for using `rocprof` with OpenMP
+code compiled with AOMP:
+
+1. Run `rocprof` with the program command line:
+
+    ```bash
+    % rocprof <application> <args>
+    ```
+
+    This produces a `results.csv` file in the user’s current directory that
+    shows basic stats such as kernel names, grid size, number of registers used,
+    etc. The user can choose to specify the preferred output file name using the
+    o option.
+
+2. Add options for a detailed result:
+
+   ```bash
+   --stats: % rocprof --stats <application> <args>
+   ```
+
+   The stats option produces timestamps for the kernels. Look into the output
+   CSV file for the field, `DurationNs`, which is useful in getting an
+   understanding of the critical kernels in the code.
+
+   Apart from `--stats`, the option `--timestamp` on produces a timestamp for
+   the kernels.
+
+3. After learning about the required kernels, the user can take a detailed look
+   at each one of them. `rocprof` has support for hardware counters: a set of
+   basic and a set of derived ones. See the complete list of counters using
+   options --list-basic and --list-derived. `rocprof` accepts either a text or
+   an XML file as an input.
+
+For more details on `rocprof`, refer to the {doc}`ROCProfilerV1 User Manual <rocprofiler:rocprofv1>`.
+
+### Using tracing options
+
+**Prerequisite:** When using the `--sys-trace` option, compile the OpenMP
+program with:
+
+```bash
+    -Wl,-rpath,/opt/rocm-{version}/lib -lamdhip64
+```
+
+The following tracing options are widely used to generate useful information:
+
+* **`--hsa-trace`**: This option is used to get a JSON output file with the HSA
+  API execution traces and a flat profile in a CSV file.
+
+* **`--sys-trace`**: This allows programmers to trace both HIP and HSA calls.
+  Since this option results in loading ``libamdhip64.so``, follow the
+  prerequisite as mentioned above.
+
+A CSV and a JSON file are produced by the above trace options. The CSV file
+presents the data in a tabular format, and the JSON file can be visualized using
+Google Chrome at chrome://tracing/ or [Perfetto](https://perfetto.dev/).
+Navigate to Chrome or Perfetto and load the JSON file to see the timeline of the
+HSA calls.
+
+For more details on tracing, refer to the {doc}`ROCProfilerV1 User Manual <rocprofiler:rocprofv1>`.
+
+### Environment variables
+
+:::{table}
+:widths: auto
+| Environment Variable        | Purpose                  |
+| --------------------------- | ---------------------------- |
+| `OMP_NUM_TEAMS`             | To set the number of teams for kernel launch, which is otherwise chosen by the implementation by default. You can set this number (subject to implementation limits) for performance tuning. |
+| `LIBOMPTARGET_KERNEL_TRACE` | To print useful statistics for device operations. Setting it to 1 and running the program emits the name of every kernel launched, the number of teams and threads used, and the corresponding register usage. Setting it to 2 additionally emits timing information for kernel launches and data transfer operations between the host and the device. |
+| `LIBOMPTARGET_INFO`         | To print informational messages from the device runtime as the program executes. Setting it to a value of 1 or higher, prints fine-grain information and setting it to -1 prints complete information. |
+| `LIBOMPTARGET_DEBUG`        | To get detailed debugging information about data transfer operations and kernel launch when using a debug version of the device library. Set this environment variable to 1 to get the detailed information from the library. |
+| `GPU_MAX_HW_QUEUES`         | To set the number of HSA queues in the OpenMP runtime. The HSA queues are created on demand up to the maximum value as supplied here. The queue creation starts with a single initialized queue to avoid unnecessary allocation of resources. The provided value is capped if it exceeds the recommended, device-specific value. |
+| `LIBOMPTARGET_AMDGPU_MAX_ASYNC_COPY_BYTES` | To set the threshold size up to which data transfers are initiated asynchronously. The default threshold size is 1*1024*1024 bytes (1MB). |
+| `OMPX_FORCE_SYNC_REGIONS` | To force the runtime to execute all operations synchronously, i.e., wait for an operation to complete immediately. This affects data transfers and kernel execution. While it is mainly designed for debugging, it may have a minor positive effect on performance in certain situations. |
+:::
+
+## OpenMP: features
+
+The OpenMP programming model is greatly enhanced with the following new features
+implemented in the past releases.
+
+(openmp_usm)=
+
+### Asynchronous behavior in OpenMP target regions
+
+* Controlling Asynchronous Behavior
+
+The OpenMP offloading runtime executes in an asynchronous fashion by default, allowing multiple data transfers to start concurrently. However, if the data to be transferred becomes larger than the default threshold of 1MB, the runtime falls back to a synchronous data transfer. The buffers that have been locked already are always executed asynchronously.
+You can overrule this default behavior by setting `LIBOMPTARGET_AMDGPU_MAX_ASYNC_COPY_BYTES` and `OMPX_FORCE_SYNC_REGIONS`. See the [Environment Variables](#environment-variables) table for details.
+
+* Multithreaded Offloading on the Same Device
+
+The `libomptarget` plugin for GPU offloading allows creation of separate configurable HSA queues per chiplet, which enables two or more threads to concurrently offload to the same device.
+
+* Parallel Memory Copy Invocations
+
+Implicit asynchronous execution of single target region enables parallel memory copy invocations.
+
+### Unified shared memory
+
+Unified Shared Memory (USM) provides a pointer-based approach to memory
+management. To implement USM, fulfill the following system requirements along
+with Xnack capability.
+
+#### Prerequisites
+
+* Linux Kernel versions above 5.14
+* Latest KFD driver packaged in ROCm stack
+* Xnack, as USM support can only be tested with applications compiled with Xnack
+  capability
+
+#### Xnack capability
+
+When enabled, Xnack capability allows GPU threads to access CPU (system) memory,
+allocated with OS-allocators, such as `malloc`, `new`, and `mmap`. Xnack must be
+enabled both at compile- and run-time. To enable Xnack support at compile-time,
+use:
+
+```bash
+--offload-arch=gfx908:xnack+
+```
+
+Or use another functionally equivalent option Xnack-any:
+
+```bash
+--offload-arch=gfx908
+```
+
+To enable Xnack functionality at runtime on a per-application basis,
+use environment variable:
+
+```bash
+HSA_XNACK=1
+```
+
+When Xnack support is not needed:
+
+* Build the applications to maximize resource utilization using:
+
+```bash
+--offload-arch=gfx908:xnack-
+```
+
+* At runtime, set the `HSA_XNACK` environment variable to 0.
+
+#### Unified shared memory pragma
+
+This OpenMP pragma is available on MI200 through `xnack+` support.
+
+```bash
+omp requires unified_shared_memory
+```
+
+As stated in the OpenMP specifications, this pragma makes the map clause on
+target constructs optional. By default, on MI200, all memory allocated on the
+host is fine grain. Using the map clause on a target clause is allowed, which
+transforms the access semantics of the associated memory to coarse grain.
+
+```bash
+A simple program demonstrating the use of this feature is:
+$ cat parallel_for.cpp
+#include <stdlib.h>
+#include <stdio.h>
+
+#define N 64
+#pragma omp requires unified_shared_memory
+int main() {
+  int n = N;
+  int *a = new int[n];
+  int *b = new int[n];
+
+  for(int i = 0; i < n; i++)
+    b[i] = i;
+
+  #pragma omp target parallel for map(to:b[:n])
+  for(int i = 0; i < n; i++)
+    a[i] = b[i];
+
+  for(int i = 0; i < n; i++)
+    if(a[i] != i)
+      printf("error at %d: expected %d, got %d\n", i, i+1, a[i]);
+
+  return 0;
+}
+$ clang++ -O2 -target x86_64-pc-linux-gnu -fopenmp --offload-arch=gfx90a:xnack+ parallel_for.cpp
+$ HSA_XNACK=1 ./a.out
+```
+
+In the above code example, pointer “a” is not mapped in the target region, while
+pointer “b” is. Both are valid pointers on the GPU device and passed by-value to
+the kernel implementing the target region. This means the pointer values on the
+host and the device are the same.
+
+The difference between the memory pages pointed to by these two variables is
+that the pages pointed by “a” are in fine-grain memory, while the pages pointed
+to by “b” are in coarse-grain memory during and after the execution of the
+target region. This is accomplished in the OpenMP runtime library with calls to
+the ROCr runtime to set the pages pointed by “b” as coarse grain.
+
+### OMPT target support
+
+The OpenMP runtime in ROCm implements a subset of the OMPT device APIs, as
+described in the OpenMP specification document. These APIs allow first-party
+tools to examine the profile and kernel traces that execute on a device. A tool
+can register callbacks for data transfer and kernel dispatch entry points or use
+APIs to start and stop tracing for device-related activities such as data
+transfer and kernel dispatch timings and associated metadata. If device tracing
+is enabled, trace records for device activities are collected during program
+execution and returned to the tool using the APIs described in the
+specification.
+
+The following example demonstrates how a tool uses the supported OMPT target
+APIs. The `README` in `/opt/rocm/llvm/examples/tools/ompt` outlines the steps to
+be followed, and the provided example can be run as shown below:
+
+```bash
+cd $ROCM_PATH/share/openmp-extras/examples/tools/ompt/veccopy-ompt-target-tracing
+sudo make run
+```
+
+The file `veccopy-ompt-target-tracing.c` simulates how a tool initiates device
+activity tracing. The file `callbacks.h` shows the callbacks registered and
+implemented by the tool.
+
+### Floating point atomic operations
+
+The MI200-series GPUs support the generation of hardware floating-point atomics
+using the OpenMP atomic pragma. The support includes single- and
+double-precision floating-point atomic operations. The programmer must ensure
+that the memory subjected to the atomic operation is in coarse-grain memory by
+mapping it explicitly with the help of map clauses when not implicitly mapped by
+the compiler as per the [OpenMP
+specifications](https://www.openmp.org/specifications/). This makes these
+hardware floating-point atomic instructions “fast,” as they are faster than
+using a default compare-and-swap loop scheme, but at the same time “unsafe,” as
+they are not supported on fine-grain memory. The operation in
+`unified_shared_memory` mode also requires programmers to map the memory
+explicitly when not implicitly mapped by the compiler.
+
+To request fast floating-point atomic instructions at the file level, use
+compiler flag `-munsafe-fp-atomics` or a hint clause on a specific pragma:
+
+```bash
+double a = 0.0;
+#pragma omp atomic hint(AMD_fast_fp_atomics)
+a = a + 1.0;
+```
+
+```{note}
+`AMD_unsafe_fp_atomics` is an alias for `AMD_fast_fp_atomics`, and
+`AMD_safe_fp_atomics` is implemented with a compare-and-swap loop.
+```
+
+To disable the generation of fast floating-point atomic instructions at the file
+level, build using the option `-msafe-fp-atomics` or use a hint clause on a
+specific pragma:
+
+```bash
+double a = 0.0;
+#pragma omp atomic hint(AMD_safe_fp_atomics)
+a = a + 1.0;
+```
+
+The hint clause value always has a precedence over the compiler flag, which
+allows programmers to create atomic constructs with a different behavior than
+the rest of the file.
+
+See the example below, where the user builds the program using
+`-msafe-fp-atomics` to select a file-wide “safe atomic” compilation. However,
+the fast atomics hint clause over variable “a” takes precedence and operates on
+“a” using a fast/unsafe floating-point atomic, while the variable “b” in the
+absence of a hint clause is operated upon using safe floating-point atomics as
+per the compiler flag.
+
+```bash
+double a = 0.0;.
+#pragma omp atomic hint(AMD_fast_fp_atomics)
+a = a + 1.0;
+
+double b = 0.0;
+#pragma omp atomic
+b = b + 1.0;
+```
+
+### AddressSanitizer tool
+
+AddressSanitizer (ASan) is a memory error detector tool utilized by applications to
+detect various errors ranging from spatial issues such as out-of-bound access to
+temporal issues such as use-after-free. The AOMP compiler supports ASan for AMD
+GPUs with applications written in both HIP and OpenMP.
+
+**Features supported on host platform (Target x86_64):**
+
+* Use-after-free
+* Buffer overflows
+* Heap buffer overflow
+* Stack buffer overflow
+* Global buffer overflow
+* Use-after-return
+* Use-after-scope
+* Initialization order bugs
+
+**Features supported on AMDGPU platform (`amdgcn-amd-amdhsa`):**
+
+* Heap buffer overflow
+* Global buffer overflow
+
+**Software (kernel/OS) requirements:** Unified Shared Memory support with Xnack
+capability. See the section on [Unified Shared Memory](#unified-shared-memory)
+for prerequisites and details on Xnack.
+
+**Example:**
+
+* Heap buffer overflow
+
+```bash
+void  main() {
+.......  // Some program statements
+.......  // Some program statements
+#pragma omp target map(to : A[0:N], B[0:N]) map(from: C[0:N])
+{
+#pragma omp parallel for
+    for(int i =0 ; i < N; i++){
+    C[i+10] = A[i] + B[i];
+  }   // end of for loop
+}
+.......   // Some program statements
+}// end of main
+```
+
+See the complete sample code for heap buffer overflow
+[here](https://github.com/ROCm-Developer-Tools/aomp/blob/aomp-dev/examples/tools/asan/heap_buffer_overflow/openmp/vecadd-HBO.cpp).
+
+* Global buffer overflow
+
+```bash
+#pragma omp declare target
+   int A[N],B[N],C[N];
+#pragma omp end declare target
+void main(){
+......  // some program statements
+......  // some program statements
+#pragma omp target data map(to:A[0:N],B[0:N]) map(from: C[0:N])
+{
+#pragma omp target update to(A,B)
+#pragma omp target parallel for
+for(int i=0; i<N; i++){
+    C[i]=A[i*100]+B[i+22];
+} // end of for loop
+#pragma omp target update from(C)
+}
+........  // some program statements
+} // end of main
+```
+
+See the complete sample code for global buffer overflow
+[here](https://github.com/ROCm-Developer-Tools/aomp/blob/aomp-dev/examples/tools/asan/global_buffer_overflow/openmp/vecadd-GBO.cpp).
+
+### Clang compiler option for kernel optimization
+
+You can use the clang compiler option `-fopenmp-target-fast` for kernel optimization if certain constraints implied by its component options are satisfied. `-fopenmp-target-fast` enables the following options:
+
+* `-fopenmp-target-ignore-env-vars`: It enables code generation of specialized kernels including no-loop and Cross-team reductions.
+
+* `-fopenmp-assume-no-thread-state`: It enables the compiler to assume that no thread in a parallel region modifies an Internal Control Variable (`ICV`), thus potentially reducing the device runtime code execution.
+
+* `-fopenmp-assume-no-nested-parallelism`: It enables the compiler to assume that no thread in a parallel region encounters a parallel region, thus potentially reducing the device runtime code execution.
+
+* `-O3` if no `-O*` is specified by the user.
+
+### Specialized kernels
+
+Clang will attempt to generate specialized kernels based on compiler options and OpenMP constructs. The following specialized kernels are supported:
+
+* No-loop
+* Big-jump-loop
+* Cross-team reductions
+
+To enable the generation of specialized kernels, follow these guidelines:
+
+* Do not specify teams, threads, and schedule-related environment variables. The `num_teams` clause in an OpenMP target construct acts as an override and prevents the generation of the no-loop kernel. If the specification of `num_teams` clause is a user requirement then clang tries to generate the big-jump-loop kernel instead of the no-loop kernel.
+
+* Assert the absence of the teams, threads, and schedule-related environment variables by adding the command-line option `-fopenmp-target-ignore-env-vars`.
+
+* To automatically enable the specialized kernel generation, use `-Ofast` or `-fopenmp-target-fast` for compilation.
+
+* To disable specialized kernel generation, use `-fno-openmp-target-ignore-env-vars`.
+
+#### No-loop kernel generation
+
+The no-loop kernel generation feature optimizes the compiler performance by generating a specialized kernel for certain OpenMP target constructs such as target teams distribute parallel for. The specialized kernel generation feature assumes every thread executes a single iteration of the user loop, which leads the runtime to launch a total number of GPU threads equal to or greater than the iteration space size of the target region loop. This allows the compiler to generate code for the loop body without an enclosing loop, resulting in reduced control-flow complexity and potentially better performance.
+
+#### Big-jump-loop kernel generation
+
+A no-loop kernel is not generated if the OpenMP teams construct uses a `num_teams` clause. Instead, the compiler attempts to generate a different specialized kernel called the big-jump-loop kernel. The compiler launches the kernel with a grid size determined by the number of teams specified by the OpenMP `num_teams` clause and the `blocksize` chosen either by the compiler or specified by the corresponding OpenMP clause.
+
+#### Cross-team optimized reduction kernel generation
+
+If the OpenMP construct has a reduction clause, the compiler attempts to generate optimized code by utilizing efficient cross-team communication. New APIs for cross-team reduction are implemented in the device runtime and are automatically generated by clang.

docs/about/compatibility/user-kernel-space-compat-matrix.md

@@ -0,0 +1,24 @@
+# User/kernel-space support matrix
+
+ROCm™ provides forward and backward compatibility between the Kernel Fusion
+Driver (KFD) and its user space software for +/- 2 releases. This table shows
+the compatibility combinations that are currently supported.
+
+| KFD   | Tested user space versions |
+|:------|:--------------------------:|
+| 5.0.2 | 5.1.0, 5.2.0               |
+| 5.1.0 | 5.0.2                      |
+| 5.1.3 | 5.2.0, 5.3.0               |
+| 5.2.0 | 5.0.2, 5.1.3               |
+| 5.2.3 | 5.3.0, 5.4.0               |
+| 5.3.0 | 5.1.3, 5.2.3               |
+| 5.3.3 | 5.4.0, 5.5.0               |
+| 5.4.0 | 5.2.3, 5.3.3               |
+| 5.4.3 | 5.5.0, 5.6.0               |
+| 5.4.4 | 5.5.0                      |
+| 5.5.0 | 5.3.3, 5.4.3               |
+| 5.5.1 | 5.6.0, 5.7.0               |
+| 5.6.0 | 5.4.3, 5.5.1               |
+| 5.6.1 | 5.7.0                      |
+| 5.7.0 | 5.5.0, 5.6.1               |
+| 5.7.1 | 5.5.0, 5.6.1               |

docs/about/compatibility/windows-support.md

@@ -0,0 +1,80 @@
+# GPU and OS support (Windows)
+
+(windows-support)=
+
+## Supported SKUs
+
+AMD HIP SDK supports the following Windows variants.
+
+| Distribution        |Processor Architectures| Validated update   |
+|---------------------|-----------------------|--------------------|
+| Windows 10          | x86-64                | 22H2 (GA)          |
+| Windows 11          | x86-64                | 22H2 (GA)          |
+| Windows Server 2022 | x86-64                |                    |
+
+## Windows-supported GPUs
+
+The table below shows supported GPUs for Radeon Pro™ and Radeon™ GPUs. Please
+click the tabs below to switch between GPU product lines. If a GPU is not listed
+on this table, the GPU is not officially supported by AMD.
+
+::::{tab-set}
+
+:::{tab-item} Radeon Pro™
+:sync: radeonpro
+
+| Name | Architecture |[LLVM Target](https://www.llvm.org/docs/AMDGPUUsage.html#processors) | Runtime | HIP SDK |
+|:----:|:------------:|:--------------------------------------------------------------------:|:-------:|:----------------:|
+| AMD Radeon Pro™ W7900   | RDNA3  | gfx1100 | ✅ | ✅ |
+| AMD Radeon Pro™ W7800   | RDNA3  | gfx1100 | ✅ | ✅ |
+| AMD Radeon Pro™ W6800   | RDNA2  | gfx1030 | ✅ | ✅ |
+| AMD Radeon Pro™ W6600   | RDNA2  | gfx1032 | ✅ | ❌ |
+| AMD Radeon Pro™ W5500   | RDNA1  | gfx1012 | ❌ | ❌ |
+| AMD Radeon Pro™ VII     | GCN5.1 | gfx906  | ❌ | ❌ |
+
+:::
+
+:::{tab-item} Radeon™
+:sync: radeon
+
+| Name | Architecture | [LLVM Target](https://www.llvm.org/docs/AMDGPUUsage.html#processors) | Runtime | HIP SDK |
+|:----:|:------------:|:--------------------------------------------------------------------:|:-------:|:----------------:|
+| AMD Radeon™ RX 7900 XTX | RDNA3  | gfx1100 | ✅ | ✅ |
+| AMD Radeon™ RX 7900 XT  | RDNA3  | gfx1100 | ✅ | ✅ |
+| AMD Radeon™ RX 7600     | RDNA3  | gfx1102 | ✅ | ✅ |
+| AMD Radeon™ RX 6950 XT  | RDNA2  | gfx1030 | ✅ | ✅ |
+| AMD Radeon™ RX 6900 XT  | RDNA2  | gfx1030 | ✅ | ✅ |
+| AMD Radeon™ RX 6800 XT  | RDNA2  | gfx1030 | ✅ | ✅ |
+| AMD Radeon™ RX 6800     | RDNA2  | gfx1030 | ✅ | ✅ |
+| AMD Radeon™ RX 6750 XT  | RDNA2  | gfx1031 | ✅ | ❌ |
+| AMD Radeon™ RX 6700 XT  | RDNA2  | gfx1031 | ✅ | ❌ |
+| AMD Radeon™ RX 6700     | RDNA2  | gfx1031 | ✅ | ❌ |
+| AMD Radeon™ RX 6650 XT  | RDNA2  | gfx1032 | ✅ | ❌ |
+| AMD Radeon™ RX 6600 XT  | RDNA2  | gfx1032 | ✅ | ❌ |
+| AMD Radeon™ RX 6600     | RDNA2  | gfx1032 | ✅ | ❌ |
+
+:::
+
+::::
+
+### Component support
+
+ROCm components are described in [What is ROCm?](../../what-is-rocm.md) Support
+on Windows is provided with two levels on enablement.
+
+* **Runtime**: Runtime enables the use of the HIP and OpenCL runtimes only.
+* **HIP SDK**: Runtime plus additional components are listed in [Libraries](../../reference/library-index.md).
+ Note that some math libraries are Linux exclusive.
+
+### Support status
+
+✅: **Supported** - AMD enables these GPUs in our software distributions for
+  the corresponding ROCm product.
+⚠️: **Deprecated** - Support will be removed in a future release.
+❌: **Unsupported** - This configuration is not enabled in our software
+  distributions.
+
+## CPU support
+
+ROCm requires CPUs that support PCIe™ atomics. Modern CPUs after the release of
+1st generation AMD Zen CPU and Intel™ Haswell support PCIe atomics.

docs/about/license.md

@@ -0,0 +1,9 @@
+# License
+
+> Note: This license applies to the [ROCm repository](https://github.com/RadeonOpenCompute/ROCm) that primarily contains documentation. For other licensing information, refer to the [Licensing Terms page](./licensing).
+
+```{include} ../../LICENSE
+```
+
+```{include} ./licensing.md
+```

docs/about/licensing.md

@@ -0,0 +1,127 @@
+# ROCm licensing terms
+
+ROCm™ is released by Advanced Micro Devices, Inc. and is licensed per component separately.
+The following table is a list of ROCm components with links to their respective license
+terms. These components may include third party components subject to
+additional licenses. Please review individual repositories for more information.
+
+The table shows ROCm components, the name of license, and link to the license terms.
+The table is ordered to follow the ROCm manifest file.
+
+<!-- spellcheck-disable -->
+| Component | License |
+|:---------------------|:-------------------------|
+| [AMDMIGraphX](https://github.com/ROCmSoftwarePlatform/AMDMIGraphX/) | [MIT](https://github.com/ROCmSoftwarePlatform/AMDMIGraphX/blob/develop/LICENSE) |
+| [HIPCC](https://github.com/ROCm-Developer-Tools/HIPCC/blob/develop/LICENSE.txt) | [MIT](https://github.com/ROCm-Developer-Tools/HIPCC/blob/develop/LICENSE.txt) |
+| [HIPIFY](https://github.com/ROCm-Developer-Tools/HIPIFY/) | [MIT](https://github.com/ROCm-Developer-Tools/HIPIFY/blob/amd-staging/LICENSE.txt) |
+| [HIP](https://github.com/ROCm-Developer-Tools/HIP/) | [MIT](https://github.com/ROCm-Developer-Tools/HIP/blob/develop/LICENSE.txt) |
+| [MIOpenGEMM](https://github.com/ROCmSoftwarePlatform/MIOpenGEMM/) | [MIT](https://github.com/ROCmSoftwarePlatform/MIOpenGEMM/blob/master/LICENSE.txt) |
+| [MIOpen](https://github.com/ROCmSoftwarePlatform/MIOpen/) | [MIT](https://github.com/ROCmSoftwarePlatform/MIOpen/blob/master/LICENSE.txt) |
+| [MIVisionX](https://github.com/GPUOpen-ProfessionalCompute-Libraries/MIVisionX/) | [MIT](https://github.com/GPUOpen-ProfessionalCompute-Libraries/MIVisionX/blob/master/LICENSE.txt) |
+| [RCP](https://github.com/GPUOpen-Tools/radeon_compute_profiler/) | [MIT](https://github.com/GPUOpen-Tools/radeon_compute_profiler/blob/master/LICENSE) |
+| [ROCK-Kernel-Driver](https://github.com/RadeonOpenCompute/ROCK-Kernel-Driver/) | [GPL 2.0 WITH Linux-syscall-note](https://github.com/RadeonOpenCompute/ROCK-Kernel-Driver/blob/master/COPYING) |
+| [ROCR-Runtime](https://github.com/RadeonOpenCompute/ROCR-Runtime/) | [The University of Illinois/NCSA](https://github.com/RadeonOpenCompute/ROCR-Runtime/blob/master/LICENSE.txt) |
+| [ROCT-Thunk-Interface](https://github.com/RadeonOpenCompute/ROCT-Thunk-Interface/) | [MIT](https://github.com/RadeonOpenCompute/ROCT-Thunk-Interface/blob/master/LICENSE.md) |
+| [ROCclr](https://github.com/ROCm-Developer-Tools/ROCclr/) | [MIT](https://github.com/ROCm-Developer-Tools/ROCclr/blob/develop/LICENSE.txt) |
+| [ROCdbgapi](https://github.com/ROCm-Developer-Tools/ROCdbgapi/) | [MIT](https://github.com/ROCm-Developer-Tools/ROCdbgapi/blob/amd-master/LICENSE.txt) |
+| [ROCgdb](https://github.com/ROCm-Developer-Tools/ROCgdb/) | [GNU General Public License v2.0](https://github.com/ROCm-Developer-Tools/ROCgdb/blob/amd-master/COPYING) |
+| [ROCm-CompilerSupport](https://github.com/RadeonOpenCompute/ROCm-CompilerSupport/) | [The University of Illinois/NCSA](https://github.com/RadeonOpenCompute/ROCm-CompilerSupport/blob/amd-stg-open/LICENSE.txt) |
+| [ROCm-Device-Libs](https://github.com/RadeonOpenCompute/ROCm-Device-Libs/) | [The University of Illinois/NCSA](https://github.com/RadeonOpenCompute/ROCm-Device-Libs/blob/amd-stg-open/LICENSE.TXT) |
+| [ROCm-OpenCL-Runtime/api/opencl/khronos/icd](https://github.com/KhronosGroup/OpenCL-ICD-Loader/) | [Apache 2.0](https://github.com/KhronosGroup/OpenCL-ICD-Loader/blob/main/LICENSE) |
+| [ROCm-OpenCL-Runtime](https://github.com/RadeonOpenCompute/ROCm-OpenCL-Runtime/) | [MIT](https://github.com/RadeonOpenCompute/ROCm-OpenCL-Runtime/blob/develop/LICENSE.txt) |
+| [ROCmValidationSuite](https://github.com/ROCm-Developer-Tools/ROCmValidationSuite/) | [MIT](https://github.com/ROCm-Developer-Tools/ROCmValidationSuite/blob/master/LICENSE) |
+| [Tensile](https://github.com/ROCmSoftwarePlatform/Tensile/) | [MIT](https://github.com/ROCmSoftwarePlatform/Tensile/blob/develop/LICENSE.md) |
+| [aomp-extras](https://github.com/ROCm-Developer-Tools/aomp-extras/) | [MIT](https://github.com/ROCm-Developer-Tools/aomp-extras/blob/aomp-dev/LICENSE) |
+| [aomp](https://github.com/ROCm-Developer-Tools/aomp/) | [Apache 2.0](https://github.com/ROCm-Developer-Tools/aomp/blob/aomp-dev/LICENSE) |
+| [atmi](https://github.com/RadeonOpenCompute/atmi/) | [MIT](https://github.com/RadeonOpenCompute/atmi/blob/master/LICENSE.txt) |
+| [clang-ocl](https://github.com/RadeonOpenCompute/clang-ocl/) | [MIT](https://github.com/RadeonOpenCompute/clang-ocl/blob/master/LICENSE) |
+| [flang](https://github.com/ROCm-Developer-Tools/flang/) | [Apache 2.0](https://github.com/ROCm-Developer-Tools/flang/blob/master/LICENSE.txt) |
+| [half](https://github.com/ROCmSoftwarePlatform/half/) | [MIT](https://github.com/ROCmSoftwarePlatform/half/blob/master/LICENSE.txt) |
+| [hipBLAS](https://github.com/ROCmSoftwarePlatform/hipBLAS/) | [MIT](https://github.com/ROCmSoftwarePlatform/hipBLAS/blob/develop/LICENSE.md) |
+| [hipCUB](https://github.com/ROCmSoftwarePlatform/hipCUB/) | [Custom](https://github.com/ROCmSoftwarePlatform/hipCUB/blob/develop/LICENSE.txt) |
+| [hipFFT](https://github.com/ROCmSoftwarePlatform/hipFFT/) | [MIT](https://github.com/ROCmSoftwarePlatform/hipFFT/blob/develop/LICENSE.md) |
+| [hipSOLVER](https://github.com/ROCmSoftwarePlatform/hipSOLVER/) | [MIT](https://github.com/ROCmSoftwarePlatform/hipSOLVER/blob/develop/LICENSE.md) |
+| [hipSPARSELt](https://github.com/ROCmSoftwarePlatform/hipSPARSELt/) | [MIT](https://github.com/ROCmSoftwarePlatform/hipSPARSELt/blob/develop/LICENSE.md) |
+| [hipSPARSE](https://github.com/ROCmSoftwarePlatform/hipSPARSE/) | [MIT](https://github.com/ROCmSoftwarePlatform/hipSPARSE/blob/develop/LICENSE.md) |
+| [hipTensor](https://github.com/ROCmSoftwarePlatform/hipTensor) | [MIT](https://github.com/ROCmSoftwarePlatform/hipTensor/blob/develop/LICENSE) |
+| [hipamd](https://github.com/ROCm-Developer-Tools/hipamd/) | [MIT](https://github.com/ROCm-Developer-Tools/hipamd/blob/develop/LICENSE.txt) |
+| [hipfort](https://github.com/ROCmSoftwarePlatform/hipfort/) | [MIT](https://github.com/ROCmSoftwarePlatform/hipfort/blob/master/LICENSE) |
+| [llvm-project](https://github.com/ROCm-Developer-Tools/llvm-project/) | [Apache](https://github.com/ROCm-Developer-Tools/llvm-project/blob/main/LICENSE.TXT) |
+| [rccl](https://github.com/ROCmSoftwarePlatform/rccl/) | [Custom](https://github.com/ROCmSoftwarePlatform/rccl/blob/develop/LICENSE.txt) |
+| [rdc](https://github.com/RadeonOpenCompute/rdc/) | [MIT](https://github.com/RadeonOpenCompute/rdc/blob/master/LICENSE) |
+| [rocALUTION](https://github.com/ROCmSoftwarePlatform/rocALUTION/) | [MIT](https://github.com/ROCmSoftwarePlatform/rocALUTION/blob/develop/LICENSE.md) |
+| [rocBLAS](https://github.com/ROCmSoftwarePlatform/rocBLAS/) | [MIT](https://github.com/ROCmSoftwarePlatform/rocBLAS/blob/develop/LICENSE.md) |
+| [rocFFT](https://github.com/ROCmSoftwarePlatform/rocFFT/) | [MIT](https://github.com/ROCmSoftwarePlatform/rocFFT/blob/develop/LICENSE.md) |
+| [rocPRIM](https://github.com/ROCmSoftwarePlatform/rocPRIM/) | [MIT](https://github.com/ROCmSoftwarePlatform/rocPRIM/blob/develop/LICENSE.txt) |
+| [rocRAND](https://github.com/ROCmSoftwarePlatform/rocRAND/) | [MIT](https://github.com/ROCmSoftwarePlatform/rocRAND/blob/develop/LICENSE.txt) |
+| [rocSOLVER](https://github.com/ROCmSoftwarePlatform/rocSOLVER/) | [BSD-2-Clause](https://github.com/ROCmSoftwarePlatform/rocSOLVER/blob/develop/LICENSE.md) |
+| [rocSPARSE](https://github.com/ROCmSoftwarePlatform/rocSPARSE/) | [MIT](https://github.com/ROCmSoftwarePlatform/rocSPARSE/blob/develop/LICENSE.md) |
+| [rocThrust](https://github.com/ROCmSoftwarePlatform/rocThrust/) | [Apache 2.0](https://github.com/ROCmSoftwarePlatform/rocThrust/blob/develop/LICENSE) |
+| [rocWMMA](https://github.com/ROCmSoftwarePlatform/rocWMMA/) | [MIT](https://github.com/ROCmSoftwarePlatform/rocWMMA/blob/develop/LICENSE.md) |
+| [rocm-cmake](https://github.com/RadeonOpenCompute/rocm-cmake/) | [MIT](https://github.com/RadeonOpenCompute/rocm-cmake/blob/develop/LICENSE) |
+| [rocm_bandwidth_test](https://github.com/RadeonOpenCompute/rocm_bandwidth_test/) | [The University of Illinois/NCSA](https://github.com/RadeonOpenCompute/rocm_bandwidth_test/blob/master/LICENSE.txt) |
+| [rocm_smi_lib](https://github.com/RadeonOpenCompute/rocm_smi_lib/) | [The University of Illinois/NCSA](https://github.com/RadeonOpenCompute/rocm_smi_lib/blob/master/License.txt) |
+| [rocminfo](https://github.com/RadeonOpenCompute/rocminfo/) | [The University of Illinois/NCSA](https://github.com/RadeonOpenCompute/rocminfo/blob/master/License.txt) |
+| [rocprofiler](https://github.com/ROCm-Developer-Tools/rocprofiler/) | [MIT](https://github.com/ROCm-Developer-Tools/rocprofiler/blob/amd-master/LICENSE) |
+| [rocr_debug_agent](https://github.com/ROCm-Developer-Tools/rocr_debug_agent/) | [The University of Illinois/NCSA](https://github.com/ROCm-Developer-Tools/rocr_debug_agent/blob/master/LICENSE.txt) |
+| [roctracer](https://github.com/ROCm-Developer-Tools/roctracer/) | [MIT](https://github.com/ROCm-Developer-Tools/roctracer/blob/amd-master/LICENSE) |
+| rocm-llvm-alt | [AMD Proprietary License](https://www.amd.com/en/support/amd-software-eula)
+
+Open sourced ROCm components are released via public GitHub
+repositories, packages on https://repo.radeon.com and other distribution channels.
+Proprietary products are only available on https://repo.radeon.com. Currently, only
+one component of ROCm, rocm-llvm-alt is governed by a proprietary license.
+Proprietary components are organized in a proprietary subdirectory in the package
+repositories to distinguish from open sourced packages.
+
+The additional terms and conditions below apply to your use of ROCm technical
+documentation.
+
+©2023 Advanced Micro Devices, Inc. All rights reserved.
+
+The information presented in this document is for informational purposes only
+and may contain technical inaccuracies, omissions, and typographical errors. The
+information contained herein is subject to change and may be rendered inaccurate
+for many reasons, including but not limited to product and roadmap changes,
+component and motherboard version changes, new model and/or product releases,
+product differences between differing manufacturers, software changes, BIOS
+flashes, firmware upgrades, or the like. Any computer system has risks of
+security vulnerabilities that cannot be completely prevented or mitigated. AMD
+assumes no obligation to update or otherwise correct or revise this information.
+However, AMD reserves the right to revise this information and to make changes
+from time to time to the content hereof without obligation of AMD to notify any
+person of such revisions or changes.
+
+THIS INFORMATION IS PROVIDED “AS IS.” AMD MAKES NO REPRESENTATIONS OR WARRANTIES
+WITH RESPECT TO THE CONTENTS HEREOF AND ASSUMES NO RESPONSIBILITY FOR ANY
+INACCURACIES, ERRORS, OR OMISSIONS THAT MAY APPEAR IN THIS INFORMATION. AMD
+SPECIFICALLY DISCLAIMS ANY IMPLIED WARRANTIES OF NON-INFRINGEMENT,
+MERCHANTABILITY, OR FITNESS FOR ANY PARTICULAR PURPOSE. IN NO EVENT WILL AMD BE
+LIABLE TO ANY PERSON FOR ANY RELIANCE, DIRECT, INDIRECT, SPECIAL, OR OTHER
+CONSEQUENTIAL DAMAGES ARISING FROM THE USE OF ANY INFORMATION CONTAINED HEREIN,
+EVEN IF AMD IS EXPRESSLY ADVISED OF THE POSSIBILITY OF SUCH DAMAGES.
+
+AMD, the AMD Arrow logo, ROCm, and combinations thereof are trademarks of
+Advanced Micro Devices, Inc. Other product names used in this publication are
+for identification purposes only and may be trademarks of their respective
+companies.
+
+## Package licensing
+
+```{attention}
+AQL Profiler and AOCC CPU optimization are both provided in binary form, each
+subject to the license agreement enclosed in the directory for the binary and is
+available here: `/opt/rocm/share/doc/rocm-llvm-alt/EULA`. By using, installing,
+copying or distributing AQL Profiler and/or AOCC CPU Optimizations, you agree to
+the terms and conditions of this license agreement. If you do not agree to the
+terms of this agreement, do not install, copy or use the AQL Profiler and/or the
+AOCC CPU Optimizations.
+```
+
+For the rest of the ROCm packages, you can find the licensing information at the
+following location: `/opt/rocm/share/doc/<component-name>/`
+
+For example, you can fetch the licensing information of the `_amd_comgr_`
+component (Code Object Manager) from the `amd_comgr` folder. A file named
+`LICENSE.txt` contains the license details at:
+`/opt/rocm-5.4.3/share/doc/amd_comgr/LICENSE.txt`

docs/about/release-history.md

@@ -0,0 +1,25 @@
+# ROCm release history
+
+| Version | Release Date |
+| ------- | ------------ |
+| [5.7.1](https://rocm.docs.amd.com/en/docs-5.7.1/) | Oct 13, 2023 |
+| [5.7.0](https://rocm.docs.amd.com/en/docs-5.7.0/) | Sep 15, 2023 |
+| [5.6.0](https://rocm.docs.amd.com/en/docs-5.6.0/) | Jun 28, 2023 |
+| [5.5.1](https://rocm.docs.amd.com/en/docs-5.5.1/) | May 24, 2023 |
+| [5.5.0](https://rocm.docs.amd.com/en/docs-5.5.0/) | May 1, 2023 |
+| [5.4.3](https://rocm.docs.amd.com/en/docs-5.4.3/) | Feb 7, 2023 |
+| [5.4.2](https://rocm.docs.amd.com/en/docs-5.4.2/) | Jan 13, 2023 |
+| [5.4.1](https://rocm.docs.amd.com/en/docs-5.4.1/) | Dec 15, 2022 |
+| [5.4.0](https://rocm.docs.amd.com/en/docs-5.4.0/) | Nov 30, 2022 |
+| [5.3.3](https://rocm.docs.amd.com/en/docs-5.3.3/) | Nov 17, 2022 |
+| [5.3.2](https://rocm.docs.amd.com/en/docs-5.3.2/) | Nov 9, 2022 |
+| [5.3.0](https://rocm.docs.amd.com/en/docs-5.3.0/) | Oct 4, 2022 |
+| [5.2.3](https://rocm.docs.amd.com/en/docs-5.2.3/) | Aug 18, 2022 |
+| [5.2.1](https://rocm.docs.amd.com/en/docs-5.2.1/) | Jul 21, 2022 |
+| [5.2.0](https://rocm.docs.amd.com/en/docs-5.2.0/) | Jun 28, 2022 |
+| [5.1.3](https://rocm.docs.amd.com/en/docs-5.1.3/) | May 20, 2022 |
+| [5.1.1](https://rocm.docs.amd.com/en/docs-5.1.1/) | Apr 8, 2022 |
+| [5.1.0](https://rocm.docs.amd.com/en/docs-5.1.0/) | Mar 30, 2022 |
+| [5.0.2](https://rocm.docs.amd.com/en/docs-5.0.2/) | Mar 4, 2022 |
+| [5.0.1](https://rocm.docs.amd.com/en/docs-5.0.1/) | Feb 16, 2022 |
+| [5.0.0](https://rocm.docs.amd.com/en/docs-5.0.0/) | Feb 9, 2022 |

docs/about/whats-new/whats-new.md

@@ -0,0 +1,93 @@
+# What's new in ROCm?
+
+ROCm is now supported on Windows.
+
+## Windows support
+
+Starting with ROCm 5.5, the HIP SDK brings a subset of ROCm to developers on Windows.
+The collection of features enabled on Windows is referred to as the HIP SDK.
+These features allow developers to use the HIP runtime, HIP math libraries
+and HIP Primitive libraries. The following table shows the differences
+between Windows and Linux releases.
+
+|Component|Linux|Windows|
+|---------|-----|-------|
+|Driver|Radeon Software for Linux |AMD Software Pro Edition|
+|Compiler|`hipcc`/`amdclang++`|`hipcc`/`clang++`|
+|Debugger|`rocgdb`|no debugger available|
+|Profiler|`rocprof`|[Radeon GPU Profiler](https://gpuopen.com/rgp/)|
+|Porting Tools|HIPIFY|Coming Soon|
+|Runtime|HIP (Open Sourced)|HIP (closed source)|
+|Math Libraries|Supported|Supported|
+|Primitives Libraries|Supported|Supported|
+|Communication Libraries|Supported|Not Available|
+|AI Libraries|MIOpen, MIGraphX|Not Available|
+|System Management|`rocm-smi-lib`, RDC, `rocminfo`|`amdsmi`, `hipInfo`|
+|AI Frameworks|PyTorch, TensorFlow, etc.|Not Available|
+|CMake HIP Language|Enabled|Unsupported|
+|Visual Studio| Not applicable| Plugin Available|
+|HIP Ray Tracing| Supported|Supported|
+
+AMD is continuing to invest in Windows support and AMD plans to release enhanced
+features in subsequent revisions.
+
+```{note}
+The 5.5 Windows Installer collectively groups the Math and Primitives
+libraries.
+```
+
+```{note}
+GPU support on Windows and Linux may differ. You must refer to
+Windows and Linux GPU support tables separately.
+```
+
+```{note}
+HIP Ray Tracing is not distributed via ROCm in Linux.
+```
+
+## ROCm release versioning
+
+Linux OS releases set the canonical version numbers for ROCm. Windows will
+follow Linux version numbers as Windows releases are based on Linux ROCm
+releases. However, not all Linux ROCm releases will have a corresponding Windows
+release. The following table shows the ROCm releases on Windows and Linux. Releases
+with both Windows and Linux are referred to as a joint release. Releases with
+only Linux support are referred to as a skipped release from the Windows
+perspective.
+
+|Release version|Linux|Windows|
+|---------------|-----|-------|
+|5.5|✅|✅|
+|5.6|✅|❌|
+
+ROCm Linux releases are versioned with following the Major.Minor.Patch
+version number system. Windows releases will only be versioned with Major.Minor.
+
+In general, Windows releases will trail Linux releases. Software developers that
+wish to support both Linux and Windows using a single ROCm version should
+refrain from upgrading ROCm unless there is a joint release.
+
+## Windows documentation implications
+
+The ROCm documentation website contains both Windows and Linux documentation.
+Just below each article title, a convenient article information section states
+whether the page applies to Linux only, Windows only or both OSes. To find the
+exact Windows documentation for a release of the HIP SDK, please view the ROCm documentation with the same
+Major.Minor version number while ignoring the Patch version. The Patch version
+only matters for Linux releases.  For convenience,
+Windows documentation will continue to be included in the overall ROCm
+documentation for the skipped Windows releases.
+
+Windows release notes will contain only information pertinent to Windows.
+The software developer must read all the previous ROCm release notes (including)
+skipped ROCm versions on Windows for information on all the changes present in
+the Windows release.
+
+## Windows builds from source
+
+Not all source code required to build Windows from source is available under a
+permissive open source license. Build instructions on Windows is only provided
+for projects that can be built from source on Windows using a toolchain that
+has closed source build prerequisites. The ROCm manifest file is not valid for
+Windows. AMD does not release a manifest or tag our components in Windows.
+Users may use corresponding Linux tags to build on Windows.

docs/conceptual/More-about-how-ROCm-uses-PCIe-Atomics.rst

@@ -0,0 +1,145 @@
+===========================
+How ROCm uses PCIe atomics
+===========================
+
+
+ROCm PCIe feature and overview of BAR memory
+======================================================================
+
+
+ROCm is an extension of HSA platform architecture, so it shares the queueing model, memory model, signaling and synchronization protocols. Platform atomics are integral to perform queuing and signaling memory operations where there may be multiple-writers across CPU and GPU agents.
+
+The full list of HSA system architecture platform requirements are here: `HSA Sys Arch Features <http://hsafoundation.com/wp-content/uploads/2021/02/HSA-SysArch-1.2.pdf>`_.
+
+The ROCm Platform uses the new PCI Express 3.0 (PCIe 3.0) features for Atomic Read-Modify-Write Transactions which extends inter-processor synchronization mechanisms to IO to support the defined set of HSA capabilities needed for queuing and signaling memory operations.
+
+The new PCIe AtomicOps operate as completers for ``CAS`` (Compare and Swap), ``FetchADD``, ``SWAP`` atomics. The AtomicsOps are initiated by the
+I/O device which support 32-bit, 64-bit and 128-bit operand which target address have to be naturally aligned to operation sizes.
+
+For ROCm the Platform atomics are used in ROCm in the following ways:
+
+   * Update HSA queue’s read_dispatch_id: 64 bit atomic add used by the command processor on the GPU agent to update the packet ID it 	  processed.
+   * Update HSA queue’s write_dispatch_id: 64 bit atomic add used by the CPU and GPU agent to support multi-writer queue insertions.
+   * Update HSA Signals – 64bit atomic ops are used for CPU & GPU synchronization.
+
+The PCIe 3.0 AtomicOp feature allows atomic transactions to be requested by, routed through and completed by PCIe components. Routing and completion does not require software support. Component support for each is detectable via the DEVCAP2 register. Upstream bridges need to have AtomicOp routing enabled or the Atomic Operations will fail even though PCIe endpoint and PCIe I/O devices has the capability to Atomics Operations.
+
+To do AtomicOp routing capability between two or more Root Ports, each associated Root Port must indicate that capability via the AtomicOp routing supported bit in the Device Capabilities 2 register.
+
+If your system has a PCIe Express Switch it needs to support AtomicsOp routing. AtomicOp requests are permitted only if a component’s ``DEVCTL2.ATOMICOP_REQUESTER_ENABLE`` field is set. These requests can only be serviced if the upstream components support AtomicOp completion and/or routing to a component which does. AtomicOp Routing Support=1 Routing is supported, AtomicOp Routing Support=0 routing is not supported.
+
+An atomic operation is a non-posted transaction supporting 32-bit and 64-bit address formats, there must be a response for Completion containing the result of the operation. Errors associated with the operation (uncorrectable error accessing the target location or carrying out the Atomic operation) are signaled to the requester by setting the Completion Status field in the completion descriptor, they are set to to Completer Abort (CA) or Unsupported Request (UR).
+
+To understand more about how PCIe atomic operations work, see `PCIe atomics <https://pcisig.com/specifications/pciexpress/specifications/ECN_Atomic_Ops_080417.pdf>`_
+
+`Linux Kernel Patch to pci_enable_atomic_request <https://patchwork.kernel.org/project/linux-pci/patch/1443110390-4080-1-git-send-email-jay@jcornwall.me/>`_
+
+There are also a number of papers which talk about these new capabilities:
+
+  * `Atomic Read Modify Write Primitives by Intel <https://www.intel.es/content/dam/doc/white-paper/atomic-read-modify-write-primitives-i-o-devices-paper.pdf>`_
+  * `PCI express 3 Accelerator White paper by Intel <https://www.intel.sg/content/dam/doc/white-paper/pci-express3-accelerator-white-paper.pdf>`_
+  * `Intel PCIe Generation 3 Hotchips Paper <https://www.hotchips.org/wp-content/uploads/hc_archives/hc21/1_sun/HC21.23.1.SystemInterconnectTutorial-Epub/HC21.23.131.Ajanovic-Intel-PCIeGen3.pdf>`_
+  * `PCIe Generation 4 Base Specification includes Atomics Operation <https://astralvx.com/storage/2020/11/PCI_Express_Base_4.0_Rev0.3_February19-2014.pdf>`_
+
+Other I/O devices with PCIe atomics support
+
+   * `Mellanox ConnectX-5 InfiniBand Card <http://www.mellanox.com/related-docs/prod_adapter_cards/PB_ConnectX-5_VPI_Card.pdf>`_
+   * `Cray Aries Interconnect <http://www.hoti.org/hoti20/slides/Bob_Alverson.pdf>`_
+   * `Xilinx PCIe Ultrascale White paper <https://docs.xilinx.com/v/u/8OZSA2V1b1LLU2rRCDVGQw>`_
+   * `Xilinx 7 Series Devices <https://docs.xilinx.com/v/u/1nfXeFNnGpA0ywyykvWHWQ>`_
+
+Future bus technology with richer I/O atomics operation Support
+
+  * GenZ
+
+New PCIe Endpoints with support beyond AMD Ryzen and EPYC CPU; Intel Haswell or newer CPU’s with PCIe Generation 3.0 support.
+
+  * `Mellanox Bluefield SOC <https://docs.nvidia.com/networking/display/BlueFieldSWv25111213/BlueField+Software+Overview>`_
+  * `Cavium Thunder X2 <https://en.wikichip.org/wiki/cavium/thunderx2>`_
+
+In ROCm, we also take advantage of PCIe ID based ordering technology for P2P when the GPU originates two writes to two different targets:
+
+  | 1. write to another GPU memory,
+
+  | 2. then write to system memory to indicate transfer complete.
+
+They are routed off to different ends of the computer but we want to make sure the write to system memory to indicate transfer complete occurs AFTER P2P write to GPU has complete.
+
+BAR memory overview
+***************************************************************************************************
+On a Xeon E5 based system in the BIOS we can turn on above 4GB PCIe addressing, if so he need to set MMIO Base address ( MMIOH Base) and Range ( MMIO High Size) in the BIOS.
+
+In SuperMicro system in the system bios you need to see the following
+
+   * Advanced->PCIe/PCI/PnP configuration-> Above 4G Decoding = Enabled
+
+   * Advanced->PCIe/PCI/PnP Configuration->MMIOH Base = 512G
+
+   * Advanced->PCIe/PCI/PnP Configuration->MMIO High Size = 256G
+
+When we support Large Bar Capability there is a Large Bar Vbios which also disable the IO bar.
+
+For GFX9 and Vega10 which have Physical Address up 44 bit and 48 bit Virtual address.
+
+   * BAR0-1 registers: 64bit, prefetchable, GPU memory. 8GB or 16GB depending on Vega10 SKU. Must be placed < 2^44 to support P2P  	access from other Vega10.
+   * BAR2-3 registers: 64bit, prefetchable, Doorbell. Must be placed < 2^44 to support P2P access from other Vega10.
+   * BAR4 register: Optional, not a boot device.
+   * BAR5 register: 32bit, non-prefetchable, MMIO. Must be placed < 4GB.
+
+Here is how our base address register (BAR) works on GFX 8 GPU’s with 40 bit Physical Address Limit ::
+
+  11:00.0 Display controller: Advanced Micro Devices, Inc. [AMD/ATI] Fiji [Radeon R9 FURY / NANO Series] (rev c1)
+
+  Subsystem: Advanced Micro Devices, Inc. [AMD/ATI] Device 0b35
+
+  Flags: bus master, fast devsel, latency 0, IRQ 119
+
+  Memory at bf40000000 (64-bit, prefetchable) [size=256M]
+
+  Memory at bf50000000 (64-bit, prefetchable) [size=2M]
+
+  I/O ports at 3000 [size=256]
+
+  Memory at c7400000 (32-bit, non-prefetchable) [size=256K]
+
+  Expansion ROM at c7440000 [disabled] [size=128K]
+
+Legend:
+
+1 : GPU Frame Buffer BAR – In this example it happens to be 256M, but typically this will be size of the GPU memory (typically 4GB+). This BAR has to be placed < 2^40 to allow peer-to-peer access from other GFX8 AMD GPUs. For GFX9 (Vega GPU) the BAR has to be placed < 2^44 to allow peer-to-peer access from other GFX9 AMD GPUs.
+
+2 : Doorbell BAR – The size of the BAR is typically will be < 10MB (currently fixed at 2MB) for this generation GPUs. This BAR has to be placed < 2^40 to allow peer-to-peer access from other current generation AMD GPUs.
+
+3 : IO BAR - This is for legacy VGA and boot device support, but since this the GPUs in this project are not VGA devices (headless), this is not a concern even if the SBIOS does not setup.
+
+4 : MMIO BAR – This is required for the AMD Driver SW to access the configuration registers. Since the reminder of the BAR available is only 1 DWORD (32bit), this is placed < 4GB. This is fixed at 256KB.
+
+5 : Expansion ROM – This is required for the AMD Driver SW to access the GPU’s video-bios. This is currently fixed at 128KB.
+
+Excerpts from 'Overview of Changes to PCI Express 3.0'
+================================================================
+By Mike Jackson, Senior Staff Architect, MindShare, Inc.
+***************************************************************************************************
+Atomic operations – goal:
+***************************************************************************************************
+Support SMP-type operations across a PCIe network to allow for things like offloading tasks between CPU cores and accelerators like a GPU. The spec says this enables advanced synchronization mechanisms that are particularly useful with multiple producers or consumers that need to be synchronized in a non-blocking fashion. Three new atomic non-posted requests were added, plus the corresponding completion (the address must be naturally aligned with the operand size or the TLP is malformed):
+
+  * Fetch and Add – uses one operand as the “add” value. Reads the target location, adds the operand, and then writes the result back 	  to the original location.
+
+  * Unconditional Swap – uses one operand as the “swap” value. Reads the target location and then writes the swap value to it.
+
+  * Compare and Swap – uses 2 operands: first data is compare value, second is swap value. Reads the target location, checks it     	against the compare value and, if equal, writes the swap value to the target location.
+
+  * AtomicOpCompletion – new completion to give the result so far atomic request and indicate that the atomicity of the transaction 	has been maintained.
+
+Since atomic operations are not locked they don't have the performance downsides of the PCI locked protocol. Compared to locked cycles, they provide “lower latency, higher scalability, advanced synchronization algorithms, and dramatically lower impact on other PCIe traffic.” The lock mechanism can still be used across a bridge to PCI or PCI-X to achieve the desired operation.
+
+Atomic operations can go from device to device, device to host, or host to device. Each completer indicates whether it supports this capability and guarantees atomic access if it does. The ability to route atomic operations is also indicated in the registers for a given port.
+
+ID-based ordering – goal:
+***************************************************************************************************
+Improve performance by avoiding stalls caused by ordering rules. For example, posted writes are never normally allowed to pass each other in a queue, but if they are requested by different functions, we can have some confidence that the requests are not dependent on each other. The previously reserved Attribute bit [2] is now combined with the RO bit to indicate ID ordering with or without relaxed ordering.
+
+This only has meaning for memory requests, and is reserved for Configuration or IO requests. Completers are not required to copy this bit into a completion, and only use the bit if their enable bit is set for this operation.
+
+To read more on PCIe Gen 3 new options https://www.mindshare.com/files/resources/PCIe%203-0.pdf

docs/conceptual/ai-migraphx-optimization.md

@@ -0,0 +1,326 @@
+# Inference optimization with MIGraphX
+
+The following sections cover inferencing and introduces [MIGraphX](https://rocm.docs.amd.com/projects/AMDMIGraphX/en/latest/).
+
+## Inference
+
+The inference is where capabilities learned during deep-learning training are put to work. It refers to using a fully trained neural network to make conclusions (predictions) on unseen data that the model has never interacted with before. Deep-learning inferencing is achieved by feeding new data, such as new images, to the network, giving the Deep Neural Network a chance to classify the image.
+
+Taking our previous example of MNIST, the DNN can be fed new images of handwritten digit images, allowing the neural network to classify digits. A fully trained DNN should make accurate predictions about what an image represents, and inference cannot happen without training.
+
+## MIGraphX introduction
+
+MIGraphX is a graph compiler focused on accelerating the machine-learning inference that can target AMD GPUs and CPUs. MIGraphX accelerates the machine-learning models by leveraging several graph-level transformations and optimizations. These optimizations include:
+
+* Operator fusion
+* Arithmetic simplifications
+* Dead-code elimination
+* Common subexpression elimination (CSE)
+* Constant propagation
+
+After doing all these transformations, MIGraphX emits code for the AMD GPU by calling to MIOpen or rocBLAS or creating HIP kernels for a particular operator. MIGraphX can also target CPUs using DNNL or ZenDNN libraries.
+
+MIGraphX provides easy-to-use APIs in C++ and Python to import machine models in ONNX or TensorFlow. Users can compile, save, load, and run these models using the MIGraphX C++ and Python APIs. Internally, MIGraphX parses ONNX or TensorFlow models into internal graph representation where each operator in the model gets mapped to an operator within MIGraphX. Each of these operators defines various attributes such as:
+
+* Number of arguments
+* Type of arguments
+* Shape of arguments
+
+After optimization passes, all these operators get mapped to different kernels on GPUs or CPUs.
+
+After importing a model into MIGraphX, the model is represented as `migraphx::program`. `migraphx::program` is made up of `migraphx::module`. The program can consist of several modules, but it always has one main_module. Modules are made up of `migraphx::instruction_ref`. Instructions contain the `migraphx::op` and arguments to the operator.  
+
+## Installing MIGraphX
+
+There are three options to get started with MIGraphX installation. MIGraphX depends on ROCm libraries; assume that the machine has ROCm installed.
+
+### Option 1: installing binaries
+
+To install MIGraphX on Debian-based systems like Ubuntu, use the following command:
+
+```bash
+sudo apt update && sudo apt install -y migraphx
+```
+
+The header files and libraries are installed under `/opt/rocm-\<version\>`, where \<version\> is the ROCm version.
+
+### Option 2: building from source
+
+There are two ways to build the MIGraphX sources.
+
+* [Use the ROCm build tool](https://github.com/ROCmSoftwarePlatform/AMDMIGraphX#use-the-rocm-build-tool-rbuild) - This approach uses `[rbuild](https://github.com/RadeonOpenCompute/rbuild)` to install the prerequisites and build the libraries with just one command.
+
+  or
+
+* [Use CMake](https://github.com/ROCmSoftwarePlatform/AMDMIGraphX#use-cmake-to-build-migraphx) - This approach uses a script to install the prerequisites, then uses CMake to build the source.
+
+For detailed steps on building from source and installing dependencies, refer to the following `README` file:
+
+[https://github.com/ROCmSoftwarePlatform/AMDMIGraphX#building-from-source](https://github.com/ROCmSoftwarePlatform/AMDMIGraphX#building-from-source)
+
+### Option 3: use docker
+
+To use Docker, follow these steps:
+
+1. The easiest way to set up the development environment is to use Docker. To build Docker from scratch, first clone the MIGraphX repository by running:
+
+    ```bash
+    git clone --recursive https://github.com/ROCmSoftwarePlatform/AMDMIGraphX
+    ```
+
+2. The repository contains a Dockerfile from which you can build a Docker image as:
+
+    ```bash
+    docker build -t migraphx .
+    ```
+
+3. Then to enter the development environment, use Docker run:
+
+    ```bash
+    docker run --device='/dev/kfd' --device='/dev/dri' -v=`pwd`:/code/AMDMIGraphX -w /code/AMDMIGraphX --group-add video -it migraphx
+    ```
+
+The Docker image contains all the prerequisites required for the installation, so users can go to the folder `/code/AMDMIGraphX` and follow the steps mentioned in [Option 2: Building from Source](#option-2-building-from-source).
+
+## MIGraphX example
+
+MIGraphX provides both C++ and Python APIs. The following sections show examples of both using the Inception v3 model. To walk through the examples, fetch the Inception v3 ONNX model by running the following:
+
+```py
+import torch
+import torchvision.models as models
+inception = models.inception_v3(pretrained=True)
+torch.onnx.export(inception,torch.randn(1,3,299,299), "inceptioni1.onnx")
+```
+
+This will create `inceptioni1.onnx`, which can be imported in MIGraphX using C++ or Python API.
+
+### MIGraphX Python API
+
+Follow these steps:
+
+1. To import the MIGraphX module in Python script, set `PYTHONPATH` to the MIGraphX libraries installation. If binaries are installed using steps mentioned in [Option 1: Installing Binaries](#option-1-installing-binaries), perform the following action:
+
+    ```bash
+    export PYTHONPATH=$PYTHONPATH:/opt/rocm/
+    ```
+
+2. The following script shows the usage of Python API to import the ONNX model, compile it, and run inference on it. Set `LD_LIBRARY_PATH` to `/opt/rocm/` if required.
+
+    ```py
+    # import migraphx and numpy
+    import migraphx
+    import numpy as np
+    # import and parse inception model
+    model = migraphx.parse_onnx("inceptioni1.onnx")
+    # compile model for the GPU target
+    model.compile(migraphx.get_target("gpu"))
+    # optionally print compiled model
+    model.print()
+    # create random input image
+    input_image = np.random.rand(1, 3, 299, 299).astype('float32')
+    # feed image to model, 'x.1` is the input param name
+    results = model.run({'x.1': input_image})
+    # get the results back
+    result_np = np.array(results[0])
+    # print the inferred class of the input image
+    print(np.argmax(result_np))
+    ```
+
+    Find additional examples of Python API in the `/examples` directory of the MIGraphX repository.
+
+## MIGraphX C++ API
+
+Follow these steps:
+
+1. The following is a minimalist example that shows the usage of MIGraphX C++ API to load ONNX file, compile it for the GPU, and run inference on it. To use MIGraphX C++ API, you only need to load the `migraphx.hpp` file. This example runs inference on the Inception v3 model.
+
+    ```c++
+    #include <vector>
+    #include <string>
+    #include <algorithm>
+    #include <ctime>
+    #include <random>
+    #include <migraphx/migraphx.hpp>
+
+    int main(int argc, char** argv)
+    {
+        migraphx::program prog;
+        migraphx::onnx_options onnx_opts;
+        // import and parse onnx file into migraphx::program
+        prog = parse_onnx("inceptioni1.onnx", onnx_opts);
+        // print imported model
+        prog.print();
+        migraphx::target targ = migraphx::target("gpu");
+        migraphx::compile_options comp_opts;
+        comp_opts.set_offload_copy();
+        // compile for the GPU
+        prog.compile(targ, comp_opts);
+        // print the compiled program
+        prog.print();
+        // randomly generate input image
+        // of shape (1, 3, 299, 299)
+        std::srand(unsigned(std::time(nullptr)));
+        std::vector<float> input_image(1*299*299*3);
+        std::generate(input_image.begin(), input_image.end(), std::rand);
+        // users need to provide data for the input
+        // parameters in order to run inference
+        // you can query into migraph program for the parameters
+        migraphx::program_parameters prog_params;
+        auto param_shapes = prog.get_parameter_shapes();
+        auto input        = param_shapes.names().front();
+        // create argument for the parameter
+        prog_params.add(input, migraphx::argument(param_shapes[input], input_image.data()));
+        // run inference
+        auto outputs = prog.eval(prog_params);
+        // read back the output
+        float* results = reinterpret_cast<float*>(outputs[0].data());
+        float* max     = std::max_element(results, results + 1000);
+        int answer = max - results;
+        std::cout << "answer: " << answer << std::endl;
+    }
+    ```
+
+2. To compile this program, you can use CMake and you only need to link the `migraphx::c` library to use MIGraphX's C++ API. The following is the `CMakeLists.txt` file that can build the earlier example:
+
+    ```cmake
+    cmake_minimum_required(VERSION 3.5)
+    project (CAI)
+
+    set (CMAKE_CXX_STANDARD 14)
+    set (EXAMPLE inception_inference)
+
+    list (APPEND CMAKE_PREFIX_PATH /opt/rocm/hip /opt/rocm)
+    find_package (migraphx)
+
+    message("source file: " ${EXAMPLE}.cpp " ---> bin: " ${EXAMPLE})
+    add_executable(${EXAMPLE} ${EXAMPLE}.cpp)
+
+    target_link_libraries(${EXAMPLE} migraphx::c)
+    ```
+
+3. To build the executable file, run the following from the directory containing the `inception_inference.cpp` file:
+
+    ```bash
+    mkdir build
+    cd build
+    cmake ..
+    make -j$(nproc)
+    ./inception_inference
+    ```
+
+```{note}
+    Set `LD_LIBRARY_PATH` to `/opt/rocm/lib` if required during the build. Additional examples can be found in the MIGraphX repository under the `/examples/` directory.
+```
+
+## Tuning MIGraphX
+
+MIGraphX uses MIOpen kernels to target AMD GPU. For the model compiled with MIGraphX, tune MIOpen to pick the best possible kernel implementation. The MIOpen tuning results in a significant performance boost. Tuning can be done by setting the environment variable `MIOPEN_FIND_ENFORCE=3`.
+
+```{note}
+    The tuning process can take a long time to finish.
+```
+
+**Example:** The average inference time of the inception model example shown previously over 100 iterations using untuned kernels is 0.01383ms. After tuning, it reduces to 0.00459ms, which is a 3x improvement. This result is from ROCm v4.5 on a MI100 GPU.
+
+```{note}
+    The results may vary depending on the system configurations.
+```
+
+For reference, the following code snippet shows inference runs for only the first 10 iterations for both tuned and untuned kernels:
+
+```console
+### UNTUNED ###
+iterator : 0
+Inference complete
+Inference time: 0.063ms
+iterator : 1
+Inference complete
+Inference time: 0.008ms
+iterator : 2
+Inference complete
+Inference time: 0.007ms
+iterator : 3
+Inference complete
+Inference time: 0.007ms
+iterator : 4
+Inference complete
+Inference time: 0.007ms
+iterator : 5
+Inference complete
+Inference time: 0.008ms
+iterator : 6
+Inference complete
+Inference time: 0.007ms
+iterator : 7
+Inference complete
+Inference time: 0.028ms
+iterator : 8
+Inference complete
+Inference time: 0.029ms
+iterator : 9
+Inference complete
+Inference time: 0.029ms
+
+### TUNED ###
+iterator : 0
+Inference complete
+Inference time: 0.063ms
+iterator : 1
+Inference complete
+Inference time: 0.004ms
+iterator : 2
+Inference complete
+Inference time: 0.004ms
+iterator : 3
+Inference complete
+Inference time: 0.004ms
+iterator : 4
+Inference complete
+Inference time: 0.004ms
+iterator : 5
+Inference complete
+Inference time: 0.004ms
+iterator : 6
+Inference complete
+Inference time: 0.004ms
+iterator : 7
+Inference complete
+Inference time: 0.004ms
+iterator : 8
+Inference complete
+Inference time: 0.004ms
+iterator : 9
+Inference complete
+Inference time: 0.004ms
+```
+
+### YModel
+
+The best inference performance through MIGraphX is conditioned upon having tuned kernel configurations stored in a `/home` local User Database (DB). If a user were to move their model to a different server or allow a different user to use it, they would have to run through the MIOpen tuning process again to populate the next User DB with the best kernel configurations and corresponding solvers.
+
+Tuning is time consuming, and if the users have not performed tuning, they would see discrepancies between expected or claimed inference performance and actual inference performance. This has led to repetitive and time-consuming tuning tasks for each user.
+
+MIGraphX introduces a feature, known as YModel, that stores the kernel config parameters found during tuning into a `.mxr` file. This ensures the same level of expected performance, even when a model is copied to a different user/system.
+
+The YModel feature is available starting from ROCm 5.4.1 and UIF 1.1.
+
+#### YModel example
+
+Through the `migraphx-driver` functionality, you can generate `.mxr` files with tuning information stored inside it by passing additional `--binary --output model.mxr` to `migraphx-driver` along with the rest of the necessary flags.
+
+For example, to generate `.mxr` file from the ONNX model, use the following:
+
+```bash
+./path/to/migraphx-driver compile --onnx resnet50.onnx --enable-offload-copy --binary --output resnet50.mxr
+```
+
+To run generated `.mxr` files through `migraphx-driver`, use the following:
+
+```bash
+./path/to/migraphx-driver run --migraphx resnet50.mxr --enable-offload-copy
+```
+
+Alternatively, you can use the MIGraphX C++ or Python API to generate `.mxr` files.
+
+![Generating an MXR file](../data/conceptual/image018.png "Generating an MXR file")

docs/conceptual/cmake-packages.rst

@@ -0,0 +1,383 @@
+***********
+Using CMake
+***********
+
+Most components in ROCm support CMake. Projects depending on header-only or
+library components typically require CMake 3.5 or higher whereas those wanting
+to make use of CMake's HIP language support will require CMake 3.21 or higher.
+
+Finding dependencies
+====================
+
+.. note::
+   For a complete
+   reference on how to deal with dependencies in CMake, refer to the CMake docs
+   on `find_package
+   <https://cmake.org/cmake/help/latest/command/find_package.html>`_ and the
+   `Using Dependencies Guide
+   <https://cmake.org/cmake/help/latest/guide/using-dependencies/index.html>`_
+   to get an overview of CMake's related facilities.
+
+In short, CMake supports finding dependencies in two ways:
+
+*  In Module mode, it consults a file ``Find<PackageName>.cmake`` which tries to
+   find the component in typical install locations and layouts. CMake ships a
+   few dozen such scripts, but users and projects may ship them as well.
+*  In Config mode, it locates a file named ``<packagename>-config.cmake`` or
+   ``<PackageName>Config.cmake`` which describes the installed component in all
+   regards needed to consume it.
+
+ROCm predominantly relies on Config mode, one notable exception being the Module
+driving the compilation of HIP programs on Nvidia runtimes. As such, when
+dependencies are not found in standard system locations, one either has to
+instruct CMake to search for package config files in additional folders using
+the ``CMAKE_PREFIX_PATH`` variable (a semi-colon separated list of file system
+paths), or using ``<PackageName>_ROOT`` variable on a project-specific basis.
+
+There are nearly a dozen ways to set these variables. One may be more convenient
+over the other depending on your workflow. Conceptually the simplest is adding
+it to your CMake configuration command on the command line via
+``-D CMAKE_PREFIX_PATH=....`` . AMD packaged ROCm installs can typically be
+added to the config file search paths such as:
+
+-  Windows: ``-D CMAKE_PREFIX_PATH=${env:HIP_PATH}``
+
+-  Linux: ``-D CMAKE_PREFIX_PATH=/opt/rocm``
+
+ROCm provides the respective *config-file* packages, and this enables
+``find_package`` to be used directly. ROCm does not require any Find module as
+the *config-file* packages are shipped with the upstream projects, such as
+rocPRIM and other ROCm libraries.
+
+For a complete guide on where and how ROCm may be installed on a system, refer
+to the installation guides for `Linux <../install/linux/install.html>`_ and
+`Windows <../install/windows/install.html>`_.
+
+Using HIP in CMake
+==================
+
+ROCm components providing a C/C++ interface support consumption via any
+C/C++ toolchain that CMake knows how to drive. ROCm also supports CMake's HIP
+language features, allowing users to program using the HIP single-source
+programming model. When a program (or translation-unit) uses the HIP API without
+compiling any GPU device code, HIP can be treated in CMake as a simple C/C++
+library.
+
+Using the HIP single-source programming model
+---------------------------------------------
+
+Source code written in the HIP dialect of C++ typically uses the `.hip`
+extension. When the HIP CMake language is enabled, it will automatically
+associate such source files with the HIP toolchain being used.
+
+::
+
+    cmake_minimum_required(VERSION 3.21) # HIP language support requires 3.21
+    cmake_policy(VERSION 3.21.3...3.27)
+    project(MyProj LANGUAGES HIP)
+    add_executable(MyApp Main.hip)
+
+Should you have existing CUDA code that is from the source compatible subset of
+HIP, you can tell CMake that despite their `.cu` extension, they're HIP sources.
+Do note that this mostly facilitates compiling kernel code-only source files,
+as host-side CUDA API won't compile in this fashion.
+
+::
+
+    add_library(MyLib MyLib.cu)
+    set_source_files_properties(MyLib.cu PROPERTIES LANGUAGE HIP)
+
+CMake itself only hosts part of the HIP language support, such as defining
+HIP-specific properties, etc. while the other half ships with the HIP
+implementation, such as ROCm. CMake will search for a file
+`hip-lang-config.cmake` describing how the the properties defined by CMake
+translate to toolchain invocations. If one installs ROCm using non-standard
+methods or layouts and CMake can't locate this file or detect parts of the SDK,
+there's a catch-all, last resort variable consulted locating this file,
+``-D CMAKE_HIP_COMPILER_ROCM_ROOT:PATH=`` which should be set the root of the
+ROCm installation.
+
+If the user doesn't provide a semi-colon delimited list of device architectures
+via ``CMAKE_HIP_ARCHITECTURES``, CMake will select some sensible default. It is
+advised though that if a user knows what devices they wish to target, then set
+this variable explicitly.
+
+Consuming ROCm C/C++ libraries
+------------------------------
+
+Libraries such as rocBLAS, rocFFT, MIOpen, etc. behave as C/C++ libraries.
+Illustrated in the example below is a C++ application using MIOpen from CMake.
+It calls ``find_package(miopen)``, which provides the ``MIOpen`` imported
+target. This can be linked with ``target_link_libraries``
+
+::
+
+    cmake_minimum_required(VERSION 3.5) # find_package(miopen) requires 3.5
+    cmake_policy(VERSION 3.5...3.27)
+    project(MyProj LANGUAGES CXX)
+    find_package(miopen)
+    add_library(MyLib ...)
+    target_link_libraries(MyLib PUBLIC MIOpen)
+
+.. note::
+    Most libraries are designed as host-only API, so using a GPU device
+    compiler is not necessary for downstream projects unless they use GPU device
+    code.
+
+Consuming the HIP API in C++ code
+---------------------------------
+
+Use the HIP API without compiling the GPU device code. As there is no GPU code,
+any C or C++ compiler can be used. The ``find_package(hip)`` provides the
+``hip::host`` imported target to use HIP in this context.
+
+::
+
+    cmake_minimum_required(VERSION 3.5) # find_package(hip) requires 3.5
+    cmake_policy(VERSION 3.5...3.27)
+    project(MyProj LANGUAGES CXX)
+    find_package(hip REQUIRED)
+    add_executable(MyApp ...)
+    target_link_libraries(MyApp PRIVATE hip::host)
+
+Compiling device code in C++ language mode
+------------------------------------------
+
+.. attention::
+    The workflow detailed here is considered legacy and is shown for
+    understanding's sake. It pre-dates the existence of HIP language support in
+    CMake. If source code has HIP device code in it, it is a HIP source file
+    and should be compiled as such. Only resort to the method below if your
+    HIP-enabled CMake codepath can't mandate CMake version 3.21.
+
+If code uses the HIP API and compiles GPU device code, it requires using a
+device compiler. The compiler for CMake can be set using either the
+``CMAKE_C_COMPILER`` and ``CMAKE_CXX_COMPILER`` variable or using the ``CC``
+and ``CXX`` environment variables. This can be set when configuring CMake or
+put into a CMake toolchain file. The device compiler must be set to a
+compiler that supports AMD GPU targets, which is usually Clang.
+
+The ``find_package(hip)`` provides the ``hip::device`` imported target to add
+all the flags necessary for device compilation.
+
+::
+
+    cmake_minimum_required(VERSION 3.8) # cxx_std_11 requires 3.8
+    cmake_policy(VERSION 3.8...3.27)
+    project(MyProj LANGUAGES CXX)
+    find_package(hip REQUIRED)
+    add_library(MyLib ...)
+    target_link_libraries(MyLib PRIVATE hip::device)
+    target_compile_features(MyLib PRIVATE cxx_std_11)
+
+.. note::
+    Compiling for the GPU device requires at least C++11.
+
+This project can then be configured with for eg.
+
+-  Windows: ``cmake -D CMAKE_CXX_COMPILER:PATH=${env:HIP_PATH}\bin\clang++.exe``
+
+-  Linux: ``cmake -D CMAKE_CXX_COMPILER:PATH=/opt/rocm/bin/amdclang++``
+
+Which use the device compiler provided from the binary packages of
+`ROCm HIP SDK <https://www.amd.com/en/developer/rocm-hub.html>`_ and
+`repo.radeon.com <https://repo.radeon.com>`_ respectively.
+
+When using the CXX language support to compile HIP device code, selecting the
+target GPU architectures is done via setting the ``GPU_TARGETS`` variable.
+``CMAKE_HIP_ARCHITECTURES`` only exists when the HIP language is enabled. By
+default, this is set to some subset of the currently supported architectures of
+AMD ROCm. It can be set to eg. ``-D GPU_TARGETS="gfx1032;gfx1035"``.
+
+ROCm CMake packages
+-------------------
+
++-----------+----------+--------------------------------------------------------+
+| Component | Package  | Targets                                                |
++===========+==========+========================================================+
+| HIP       | hip      | ``hip::host``, ``hip::device``                         |
++-----------+----------+--------------------------------------------------------+
+| rocPRIM   | rocprim  | ``roc::rocprim``                                       |
++-----------+----------+--------------------------------------------------------+
+| rocThrust | rocthrust| ``roc::rocthrust``                                     |
++-----------+----------+--------------------------------------------------------+
+| hipCUB    | hipcub   | ``hip::hipcub``                                        |
++-----------+----------+--------------------------------------------------------+
+| rocRAND   | rocrand  | ``roc::rocrand``                                       |
++-----------+----------+--------------------------------------------------------+
+| rocBLAS   | rocblas  | ``roc::rocblas``                                       |
++-----------+----------+--------------------------------------------------------+
+| rocSOLVER | rocsolver| ``roc::rocsolver``                                     |
++-----------+----------+--------------------------------------------------------+
+| hipBLAS   | hipblas  | ``roc::hipblas``                                       |
++-----------+----------+--------------------------------------------------------+
+| rocFFT    | rocfft   | ``roc::rocfft``                                        |
++-----------+----------+--------------------------------------------------------+
+| hipFFT    | hipfft   | ``hip::hipfft``                                        |
++-----------+----------+--------------------------------------------------------+
+| rocSPARSE | rocsparse| ``roc::rocsparse``                                     |
++-----------+----------+--------------------------------------------------------+
+| hipSPARSE | hipsparse| ``roc::hipsparse``                                     |
++-----------+----------+--------------------------------------------------------+
+| rocALUTION|rocalution| ``roc::rocalution``                                    |
++-----------+----------+--------------------------------------------------------+
+| RCCL      | rccl     | ``rccl``                                               |
++-----------+----------+--------------------------------------------------------+
+| MIOpen    | miopen   | ``MIOpen``                                             |
++-----------+----------+--------------------------------------------------------+
+| MIGraphX  | migraphx | ``migraphx::migraphx``, ``migraphx::migraphx_c``,      |
+|           |          | ``migraphx::migraphx_cpu``, ``migraphx::migraphx_gpu``,|
+|           |          | ``migraphx::migraphx_onnx``, ``migraphx::migraphx_tf`` |
++-----------+----------+--------------------------------------------------------+
+
+Using CMake presets
+===================
+
+CMake command lines depending on how specific users like to be when compiling
+code can grow to unwieldy lengths. This is the primary reason why projects tend
+to bake script snippets into their build definitions controlling compiler
+warning levels, changing CMake defaults (``CMAKE_BUILD_TYPE`` or
+``BUILD_SHARED_LIBS`` just to name a few) and all sorts anti-patterns, all in
+the name of convenience.
+
+Load on the command-line interface (CLI) starts immediately by selecting a
+toolchain, the set of utilities used to compile programs. To ease some of the
+toolchain related pains, CMake does consult the ``CC`` and ``CXX`` environmental
+variables when setting a default ``CMAKE_C[XX]_COMPILER`` respectively, but that
+is just the tip of the iceberg. There's a fair number of variables related to
+just the toolchain itself (typically supplied using
+`toolchain files <https://cmake.org/cmake/help/latest/manual/cmake-toolchains.7.html>`_
+), and then we still haven't talked about user preference or project-specific
+options.
+
+IDEs supporting CMake (Visual Studio, Visual Studio Code, CLion, etc.) all came
+up with their own way to register command-line fragments of different purpose in
+a setup'n'forget fashion for quick assembly using graphical front-ends. This is
+all nice, but configurations aren't portable, nor can they be reused in
+Continuous Intergration (CI) pipelines. CMake has condensed existing practice
+into a portable JSON format that works in all IDEs and can be invoked from any
+command line. This is
+`CMake Presets <https://cmake.org/cmake/help/latest/manual/cmake-presets.7.html>`_
+.
+
+There are two types of preset files: one supplied by the project, called
+``CMakePresets.json`` which is meant to be committed to version control,
+typically used to drive CI; and one meant for the user to provide, called
+``CMakeUserPresets.json``, typically used to house user preference and adapting
+the build to the user's environment. These JSON files are allowed to include
+other JSON files and the user presets always implicitly includes the non-user
+variant.
+
+Using HIP with presets
+----------------------
+
+Following is an example ``CMakeUserPresets.json`` file which actually compiles
+the `amd/rocm-examples <https://github.com/amd/rocm-examples>`_ suite of sample
+applications on a typical ROCm installation:
+
+::
+
+    {
+      "version": 3,
+      "cmakeMinimumRequired": {
+        "major": 3,
+        "minor": 21,
+        "patch": 0
+      },
+      "configurePresets": [
+        {
+          "name": "layout",
+          "hidden": true,
+          "binaryDir": "${sourceDir}/build/${presetName}",
+          "installDir": "${sourceDir}/install/${presetName}"
+        },
+        {
+          "name": "generator-ninja-multi-config",
+          "hidden": true,
+          "generator": "Ninja Multi-Config"
+        },
+        {
+          "name": "toolchain-makefiles-c/c++-amdclang",
+          "hidden": true,
+          "cacheVariables": {
+            "CMAKE_C_COMPILER": "/opt/rocm/bin/amdclang",
+            "CMAKE_CXX_COMPILER": "/opt/rocm/bin/amdclang++",
+            "CMAKE_HIP_COMPILER": "/opt/rocm/bin/amdclang++"
+          }
+        },
+        {
+          "name": "clang-strict-iso-high-warn",
+          "hidden": true,
+          "cacheVariables": {
+            "CMAKE_C_FLAGS": "-Wall -Wextra -pedantic",
+            "CMAKE_CXX_FLAGS": "-Wall -Wextra -pedantic",
+            "CMAKE_HIP_FLAGS": "-Wall -Wextra -pedantic"
+          }
+        },
+        {
+          "name": "ninja-mc-rocm",
+          "displayName": "Ninja Multi-Config ROCm",
+          "inherits": [
+            "layout",
+            "generator-ninja-multi-config",
+            "toolchain-makefiles-c/c++-amdclang",
+            "clang-strict-iso-high-warn"
+          ]
+        }
+      ],
+      "buildPresets": [
+        {
+          "name": "ninja-mc-rocm-debug",
+          "displayName": "Debug",
+          "configuration": "Debug",
+          "configurePreset": "ninja-mc-rocm"
+        },
+        {
+          "name": "ninja-mc-rocm-release",
+          "displayName": "Release",
+          "configuration": "Release",
+          "configurePreset": "ninja-mc-rocm"
+        },
+        {
+          "name": "ninja-mc-rocm-debug-verbose",
+          "displayName": "Debug (verbose)",
+          "configuration": "Debug",
+          "configurePreset": "ninja-mc-rocm",
+          "verbose": true
+        },
+        {
+          "name": "ninja-mc-rocm-release-verbose",
+          "displayName": "Release (verbose)",
+          "configuration": "Release",
+          "configurePreset": "ninja-mc-rocm",
+          "verbose": true
+        }
+      ],
+      "testPresets": [
+        {
+          "name": "ninja-mc-rocm-debug",
+          "displayName": "Debug",
+          "configuration": "Debug",
+          "configurePreset": "ninja-mc-rocm",
+          "execution": {
+            "jobs": 0
+          }
+        },
+        {
+          "name": "ninja-mc-rocm-release",
+          "displayName": "Release",
+          "configuration": "Release",
+          "configurePreset": "ninja-mc-rocm",
+          "execution": {
+            "jobs": 0
+          }
+        }
+      ]
+    }
+
+.. note::
+    Getting presets to work reliably on Windows requires some CMake improvements
+    and/or support from compiler vendors. (Refer to
+    `Add support to the Visual Studio generators <https://gitlab.kitware.com/cmake/cmake/-/issues/24245>`_
+    and `Sourcing environment scripts <https://gitlab.kitware.com/cmake/cmake/-/issues/21619>`_
+    .)

docs/conceptual/compiler-disambiguation.md

@@ -0,0 +1,15 @@
+# ROCm compilers disambiguation
+
+ROCm ships multiple compilers of varying origins and purposes. This article
+disambiguates compiler naming used throughout the documentation.
+
+## Compiler terms
+
+| Term | Description |
+| - | - |
+| `amdclang++` | Clang/LLVM-based compiler that is part of `rocm-llvm` package. The source code is available at <a href="https://github.com/RadeonOpenCompute/llvm-project" target="_blank">https://github.com/RadeonOpenCompute/llvm-project</a>. |
+| AOCC | Closed-source clang-based compiler that includes additional CPU optimizations. Offered as part of ROCm via the `rocm-llvm-alt` package. See for details, <a href="https://developer.amd.com/amd-aocc/" target="_blank">https://developer.amd.com/amd-aocc/</a>. |
+| HIP-Clang | Informal term for the `amdclang++` compiler |
+| HIPIFY | Tools including `hipify-clang` and `hipify-perl`, used to automatically translate CUDA source code into portable HIP C++. The source code is available at <a href="https://github.com/ROCm-Developer-Tools/HIPIFY" target="_blank">https://github.com/ROCm-Developer-Tools/HIPIFY</a> |
+| `hipcc` | HIP compiler driver. A utility that invokes `clang` or `nvcc` depending on the target and passes the appropriate include and library options for the target compiler and HIP infrastructure. The source code is available at <a href="https://github.com/ROCm-Developer-Tools/HIPCC" target="_blank">https://github.com/ROCm-Developer-Tools/HIPCC</a>. |
+| ROCmCC | Clang/LLVM-based compiler. ROCmCC in itself is not a binary but refers to the overall compiler. |

docs/conceptual/file-reorg.md

@@ -0,0 +1,165 @@
+# ROCm Linux Filesystem Hierarchy Standard reorganization
+
+## Introduction
+
+The ROCm platform has adopted the Linux Filesystem Hierarchy Standard (FHS) [https://refspecs.linuxfoundation.org/FHS_3.0/fhs/index.html](https://refspecs.linuxfoundation.org/FHS_3.0/fhs/index.html) in order to to ensure ROCm is consistent with standard open source conventions. The following sections specify how current and future releases of ROCm adhere to FHS, how the previous ROCm file system is supported, and how improved versioning specifications are applied to ROCm.
+
+## Adopting the FHS
+
+In order to standardize ROCm directory structure and directory content layout ROCm has adopted the [FHS](https://refspecs.linuxfoundation.org/FHS_3.0/fhs/index.html), adhering to open source conventions for Linux-based distribution. FHS ensures internal consistency within the ROCm stack, as well as external consistency with other systems and distributions. The ROCm proposed file structure is outlined below:
+
+```none
+/opt/rocm-<ver>
+    | -- bin
+         | -- all public binaries
+    | -- lib
+         | -- lib<soname>.so->lib<soname>.so.major->lib<soname>.so.major.minor.patch
+              (public libaries to link with applications)
+         | -- <component>
+              | -- architecture dependent libraries and binaries used internally by components
+         | -- cmake
+              | -- <component>
+                   | --<component>-config.cmake
+    | -- libexec
+         | -- <component>
+              | -- non ISA/architecture independent executables used internally by components
+    | -- include
+         | -- <component>
+              | -- public header files
+    | -- share
+         | -- html
+              | -- <component>
+                   | -- html documentation
+         | -- info
+              | -- <component>
+                   | -- info files
+         | -- man
+              | -- <component>
+                   | -- man pages
+         | -- doc
+              | -- <component>
+                   | -- license files
+         | -- <component>
+              | -- samples
+              | -- architecture independent misc files
+```
+
+## Changes from earlier ROCm versions
+
+The following table provides a brief overview of the new ROCm FHS layout, compared to the layout of earlier ROCm versions. Note that /opt/ is used to denote the default rocm-installation-path and should be replaced in case of a non-standard installation location of the ROCm distribution.
+
+```none
+ ______________________________________________________
+|  New ROCm Layout            |  Previous ROCm Layout  |
+|_____________________________|________________________|
+| /opt/rocm-<ver>             | /opt/rocm-<ver>        |
+|     | -- bin                |     | -- bin           |
+|     | -- lib                |     | -- lib           |
+|          | -- cmake         |     | -- include       |
+|     | -- libexec            |     | -- <component_1> |
+|     | -- include            |          | -- bin      |
+|          | -- <component_1> |          | -- cmake    |
+|     | -- share              |          | -- doc      |
+|          | -- html          |          | -- lib      |
+|          | -- info          |          | -- include  |
+|          | -- man           |          | -- samples  |
+|          | -- doc           |     | -- <component_n> |
+|          | -- <component_1> |          | -- bin      |
+|               | -- samples  |          | -- cmake    |
+|               | -- ..       |          | -- doc      |
+|          | -- <component_n> |          | -- lib      |
+|               | -- samples  |          | -- include  |
+|               | -- ..       |          | -- samples  |
+|______________________________________________________|
+```
+
+## ROCm FHS reorganization: backward compatibility
+
+The FHS file organization for ROCm was first introduced in the release of ROCm 5.2 . Backward compatibility was implemented to make sure users could still run their ROCm applications while transitioning to the new FHS. ROCm has moved header files and libraries to their new locations as indicated in the above structure, and included symbolic-links and wrapper header files in their old location for backward compatibility. The following sections detail ROCm backward compatibility implementation for wrapper header files, executable files, library files and CMake config files.
+
+### Wrapper header files
+
+Wrapper header files are placed in the old location (
+`/opt/rocm-<ver>/<component>/include`) with a warning message to include files
+from the new location (`/opt/rocm-<ver>/include`) as shown in the example below.
+
+```cpp
+#pragma message "This file is deprecated. Use file from include path /opt/rocm-ver/include/ and prefix with hip."
+#include <hip/hip_runtime.h>
+```
+
+* Starting at ROCm 5.2 release, the deprecation for backward compatibility wrapper header files is: `#pragma` message announcing `#warning`.
+* Starting from ROCm 6.0 (tentatively) backward compatibility for wrapper header files will be removed, and the `#pragma` message will be announcing `#error`.
+
+### Executable files
+
+Executable files are available in the `/opt/rocm-<ver>/bin` folder. For backward
+compatibility, the old library location (`/opt/rocm-<ver>/<component>/bin`) has a
+soft link to the library at the new location. Soft links will be removed in a
+future release, tentatively ROCm v6.0.
+
+```bash
+$ ls -l /opt/rocm/hip/bin/
+lrwxrwxrwx 1 root root   24 Jan 1 23:32 hipcc -> ../../bin/hipcc
+```
+
+### Library files
+
+Library files are available in the `/opt/rocm-<ver>/lib` folder. For backward
+compatibility, the old library location (`/opt/rocm-<ver>/<component>/lib`) has a
+soft link to the library at the new location. Soft links will be removed in a
+future release, tentatively ROCm v6.0.
+
+```shell
+$ ls -l /opt/rocm/hip/lib/
+drwxr-xr-x 4 root root 4096 Jan 1 10:45 cmake
+lrwxrwxrwx 1 root root   24 Jan 1 23:32 libamdhip64.so -> ../../lib/libamdhip64.so
+```
+
+### CMake config files
+
+All CMake configuration files are available in the
+`/opt/rocm-<ver>/lib/cmake/<component>` folder. For backward compatibility, the
+old CMake locations (`/opt/rocm-<ver>/<component>/lib/cmake`) consist of a soft
+link to the new CMake config. Soft links will be removed in a future release,
+tentatively ROCm v6.0.
+
+```shell
+$ ls -l /opt/rocm/hip/lib/cmake/hip/
+lrwxrwxrwx 1 root root 42 Jan 1 23:32 hip-config.cmake -> ../../../../lib/cmake/hip/hip-config.cmake
+```
+
+## Changes required in applications using ROCm
+
+Applications using ROCm are advised to use the new file paths. As the old files
+will be deprecated in a future release. Applications have to make sure to include
+correct header file and use correct search paths.
+
+1. `#include<header_file.h>` needs to be changed to
+   `#include <component/header_file.h>`
+
+   For example: `#include <hip.h>` needs to change
+   to `#include <hip/hip.h>`
+
+2. Any variable in CMake or Makefiles pointing to component folder needs to
+   changed.
+
+   For example: `VAR1=/opt/rocm/hip` needs to be changed to `VAR1=/opt/rocm`
+   `VAR2=/opt/rocm/hsa` needs to be changed to `VAR2=/opt/rocm`
+
+3. Any reference to `/opt/rocm/<component>/bin` or `/opt/rocm/<component>/lib`
+   needs to be changed to `/opt/rocm/bin` and `/opt/rocm/lib/`, respectively.
+
+## Changes in versioning specifications
+
+In order to better manage ROCm dependencies specification and allow smoother releases of ROCm while avoiding dependency conflicts, the ROCm platform shall adhere to the following scheme when numbering and incrementing ROCm files versions:
+
+rocm-\<ver\>, where \<ver\> = \<x.y.z\>
+
+x.y.z denote: MAJOR.MINOR.PATCH
+
+z: PATCH - increment z when implementing backward compatible bug fixes.
+
+y: MINOR - increment y when implementing minor changes that add functionality but are still backward compatible.
+
+x: MAJOR - increment x when implementing major changes that are not backward compatible.

docs/conceptual/gpu-arch.md

@@ -0,0 +1,51 @@
+# GPU architecture documentation
+
+:::::{grid} 1 1 2 2
+:gutter: 1
+
+:::{grid-item-card}
+**AMD Instinct MI200 series**
+
+Review hardware aspects of the AMD Instinct™ MI200 series of GPU
+accelerators and the CDNA™ 2 architecture.
+
+* [AMD Instinct™ MI250 microarchitecture](./gpu-arch/mi250.md)
+* [AMD Instinct MI200/CDNA2 ISA](https://www.amd.com/system/files/TechDocs/instinct-mi200-cdna2-instruction-set-architecture.pdf)
+* [White paper](https://www.amd.com/system/files/documents/amd-cdna2-white-paper.pdf)
+* [Performance counters](./gpu-arch/mi200-performance-counters.md)
+
+:::
+
+:::{grid-item-card}
+**AMD Instinct MI100**
+
+Review hardware aspects of the AMD Instinct™ MI100
+accelerators and the CDNA™ 1 architecture that is the foundation of these GPUs.
+
+* [AMD Instinct™ MI100 microarchitecture](./gpu-arch/mi100.md)
+* [AMD Instinct MI100/CDNA1 ISA](https://www.amd.com/system/files/TechDocs/instinct-mi100-cdna1-shader-instruction-set-architecture%C2%A0.pdf)
+* [White paper](https://www.amd.com/system/files/documents/amd-cdna-whitepaper.pdf)
+
+:::
+
+:::{grid-item-card}
+**RDNA**
+
+* [AMD RDNA3 ISA](https://www.amd.com/system/files/TechDocs/rdna3-shader-instruction-set-architecture-feb-2023_0.pdf)
+* [AMD RDNA2 ISA](https://www.amd.com/system/files/TechDocs/rdna2-shader-instruction-set-architecture.pdf)
+* [AMD RDNA ISA](https://www.amd.com/system/files/TechDocs/rdna-shader-instruction-set-architecture.pdf)
+* [AMD RDNA Architecture White Paper](https://www.amd.com/system/files/documents/rdna-whitepaper.pdf)
+
+:::
+
+:::{grid-item-card}
+**Older architectures**
+
+* [AMD Instinct MI50/Vega 7nm ISA](https://www.amd.com/system/files/TechDocs/vega-7nm-shader-instruction-set-architecture.pdf)
+* [AMD Instinct MI25/Vega ISA](https://www.amd.com/system/files/TechDocs/vega-shader-instruction-set-architecture.pdf)
+* [AMD GCN3 ISA](https://www.amd.com/system/files/TechDocs/gcn3-instruction-set-architecture.pdf)
+* [AMD Vega Architecture White Paper](https://en.wikichip.org/w/images/a/a1/vega-whitepaper.pdf)
+
+:::
+
+:::::

docs/conceptual/gpu-arch/mi100.md

@@ -0,0 +1,88 @@
+# AMD Instinct™ MI100 microarchitecture
+
+The following image shows the node-level architecture of a system that
+comprises two AMD EPYC™ processors and (up to) eight AMD Instinct™ accelerators.
+The two EPYC processors are connected to each other with the AMD Infinity™
+fabric which provides a high-bandwidth (up to 18 GT/sec) and coherent links such
+that each processor can access the available node memory as a single
+shared-memory domain in a non-uniform memory architecture (NUMA) fashion. In a
+2P, or dual-socket, configuration, three AMD Infinity™ fabric links are
+available to connect the processors plus one PCIe Gen 4 x16 link per processor
+can attach additional I/O devices such as the host adapters for the network
+fabric.
+
+![Structure of a single GCD in the AMD Instinct MI100 accelerator](../../data/conceptual/gpu-arch/image004.png "Node-level system architecture with two AMD EPYC™ processors and eight AMD Instinct™ accelerators.")
+
+In a typical node configuration, each processor can host up to four AMD
+Instinct™ accelerators that are attached using PCIe Gen 4 links at 16 GT/sec,
+which corresponds to a peak bidirectional link bandwidth of 32 GB/sec. Each hive
+of four accelerators can participate in a fully connected, coherent AMD
+Instinct™ fabric that connects the four accelerators using 23 GT/sec AMD
+Infinity fabric links that run at a higher frequency than the inter-processor
+links. This inter-GPU link can be established in certified server systems if the
+GPUs are mounted in neighboring PCIe slots by installing the AMD Infinity
+Fabric™ bridge for the AMD Instinct™ accelerators.
+
+## Microarchitecture
+
+The microarchitecture of the AMD Instinct accelerators is based on the AMD CDNA
+architecture, which targets compute applications such as high-performance
+computing (HPC) and AI & machine learning (ML) that run on everything from
+individual servers to the world's largest exascale supercomputers. The overall
+system architecture is designed for extreme scalability and compute performance.
+
+![Structure of the AMD Instinct accelerator (MI100 generation)](../../data/conceptual/gpu-arch/image005.png "Structure of the AMD Instinct accelerator (MI100 generation)")
+
+The above image shows the AMD Instinct accelerator with its PCIe Gen 4 x16
+link (16 GT/sec, at the bottom) that connects the GPU to (one of) the host
+processor(s). It also shows the three AMD Infinity Fabric ports that provide
+high-speed links (23 GT/sec, also at the bottom) to the other GPUs of the local
+hive.
+
+On the left and right of the floor plan, the High Bandwidth Memory (HBM)
+attaches via the GPU memory controller.  The MI100 generation of the AMD
+Instinct accelerator offers four stacks of HBM generation 2 (HBM2) for a total
+of 32GB with a 4,096bit-wide memory interface. The peak memory bandwidth of the
+attached HBM2 is 1.228 TB/sec at a memory clock frequency of 1.2 GHz.
+
+The execution units of the GPU are depicted in the above image as Compute
+Units (CU). There are a total 120 compute units that are physically organized
+into eight Shader Engines (SE) with fifteen compute units per shader engine.
+Each compute unit is further sub-divided into four SIMD units that process SIMD
+instructions of 16 data elements per instruction. This enables the CU to process
+64 data elements (a so-called 'wavefront') at a peak clock frequency of 1.5 GHz.
+Therefore, the theoretical maximum FP64 peak performance is 11.5 TFLOPS
+(`4 [SIMD units] x 16 [elements per instruction] x 120 [CU] x 1.5 [GHz]`).
+
+![Block diagram of an MI100 compute unit with detailed SIMD view of the AMD CDNA architecture](../../data/conceptual/gpu-arch/image006.png "An MI100 compute unit with detailed SIMD view of the AMD CDNA architecture")
+
+The preceding image shows the block diagram of a single CU of an AMD Instinct™
+MI100 accelerator and summarizes how instructions flow through the execution
+engines. The CU fetches the instructions via a 32KB instruction cache and moves
+them forward to execution via a dispatcher. The CU can handle up to ten
+wavefronts at a time and feed their instructions into the execution unit. The
+execution unit contains 256 vector general-purpose registers (VGPR) and 800
+scalar general-purpose registers (SGPR). The VGPR and SGPR are dynamically
+allocated to the executing wavefronts. A wavefront can access a maximum of 102
+scalar registers. Excess scalar-register usage will cause register spilling and
+thus may affect execution performance.
+
+A wavefront can occupy any number of VGPRs from 0 to 256, directly affecting
+occupancy; that is, the number of concurrently active wavefronts in the CU. For
+instance, with 119 VGPRs used, only two wavefronts can be active in the CU at
+the same time. With the instruction latency of four cycles per SIMD instruction,
+the occupancy should be as high as possible such that the compute unit can
+improve execution efficiency by scheduling instructions from multiple
+wavefronts.
+
+:::{table} Peak-performance capabilities of MI100 for different data types.
+:name: mi100-perf
+| Computation and Data Type | FLOPS/CLOCK/CU | Peak TFLOPS |
+| :------------------------ | :------------: | ----------: |
+| Vector FP64               | 64             | 11.5        |
+| Matrix FP32               | 256            | 46.1        |
+| Vector FP32               | 128            | 23.1        |
+| Matrix FP16               | 1024           | 184.6       |
+| Matrix BF16               | 512            | 92.3        |
+
+:::

docs/conceptual/gpu-arch/mi200-performance-counters.md

@@ -0,0 +1,455 @@
+# MI200 performance counters and metrics
+<!-- markdownlint-disable no-duplicate-header -->
+
+This document lists and describes the hardware performance counters and the derived metrics available on the AMD Instinct™ MI200 GPU. All hardware performance monitors, and the derived performance metrics are accessible via AMD ROCm™ Profiler tool.
+
+## MI200 performance counters list
+
+```{note}
+Preliminary validation of all MI200 performance counters is in progress. Those with “[*]” appended to the names require further evaluation.
+```
+
+### GRBM
+
+#### GRBM counters
+
+| Hardware Counter   | Unit   | Definition |
+|--------------------|--------| ------------------------------------------------------|
+| `grbm_count`       | Cycles | Free-running GPU clock |
+| `grbm_gui_active`  | Cycles | GPU active cycles |
+| `grbm_cp_busy`     | Cycles | Any of the command processor (CPC/CPF) blocks are busy. |
+| `grbm_spi_busy`    | Cycles | Any of the shader processor input (SPI) are busy in the shader engine(s). |
+| `grbm_ta_busy`     | Cycles | Any of the texture addressing unit are busy in the shader engine(s). |
+| `grbm_tc_busy`     | Cycles | Any of the texture cache blocks (TCP/TCI/TCA/TCC) are busy. |
+| `grbm_cpc_busy`    | Cycles | The command processor - compute (CPC) is busy. |
+| `grbm_cpf_busy`    | Cycles | The command processor - fetcher (CPF) is busy. |
+| `grbm_utcl2_busy`  | Cycles | The unified translation cache - level 2 (UTCL2) block is busy. |
+| `grbm_ea_busy`     | Cycles | The efficiency arbiter (EA) block is busy. |
+
+### Command processor
+
+The command processor counters are further classified into fetcher and compute.
+
+#### CPF
+
+##### CPF counters
+
+| Hardware Counter                     | Unit   | Definition                                                   |
+|--------------------------------------|--------|--------------------------------------------------------------|
+| `cpf_cmp_utcl1_stall_on_translation` | Cycles | One of the compute UTCL1s is stalled waiting on translation. |
+| `cpf_cpf_stat_idle[∗]`               | Cycles | CPF idle                                                   |
+| `cpf_cpf_stat_stall`                 | Cycles | CPF stall                                                  |
+| `cpf_cpf_tciu_busy`                  | Cycles | CPF TCIU interface busy                                    |
+| `cpf_cpf_tciu_idle`                  | Cycles | CPF TCIU interface idle                                    |
+| `cpf_cpf_tciu_stall[∗]`              | Cycles | CPF TCIU interface is stalled waiting on free tags.        |
+
+#### CPC
+
+##### CPC counters
+
+| Hardware Counter                 | Unit   | Definition                                          |
+| ---------------------------------| -------| --------------------------------------------------- |
+| `cpc_me1_busy_for_packet_decode` | Cycles | CPC ME1 busy decoding packets                       |
+| `cpc_utcl1_stall_on_translation` | Cycles | One of the UTCL1s is stalled waiting on translation |
+| `cpc_cpc_stat_busy`              | Cycles | CPC busy                                            |
+| `cpc_cpc_stat_idle`              | Cycles | CPC idle                                            |
+| `cpc_cpc_stat_stall`             | Cycles | CPC stalled                                         |
+| `cpc_cpc_tciu_busy`              | Cycles | CPC TCIU interface busy                             |
+| `cpc_cpc_tciu_idle`              | Cycles | CPC TCIU interface idle                             |
+| `cpc_cpc_utcl2iu_busy`           | Cycles | CPC UTCL2 interface busy                            |
+| `cpc_cpc_utcl2iu_idle`           | Cycles | CPC UTCL2 interface idle                            |
+| `cpc_cpc_utcl2iu_stall[∗]`       | Cycles | CPC UTCL2 interface stalled waiting                 |
+| `cpc_me1_dci0_spi_busy`          | Cycles | CPC ME1 Processor busy                              |
+
+### SPI
+
+#### SPI counters
+
+| Hardware Counter             | Unit        | Definition                                                   |
+| :----------------------------| :-----------| -----------------------------------------------------------: |
+| `spi_csn_busy`                 | Cycles      | Number of clocks with outstanding waves                      |
+| `spi_csn_window_valid`         | Cycles      | Clock count enabled by perfcounter_start event               |
+| `spi_csn_num_threadgroups`     | Workgroups  | Total number of dispatched workgroups                        |
+| `spi_csn_wave`                 | Wavefronts  | Total number of dispatched wavefronts                        |
+| `spi_ra_req_no_alloc`          | Cycles      | Arb cycles with requests but no allocation (need to multiply this value by 4) |
+|`spi_ra_req_no_alloc_csn`       | Cycles      | Arb cycles with CSn req and no CSn alloc (need to multiply this value by 4) |
+| `spi_ra_res_stall_csn`         | Cycles      | Arb cycles with CSn req and no CSn fits (need to multiply this value by 4) |
+| `spi_ra_tmp_stall_csn[∗]`      | Cycles      | Cycles where CSn wants to req but does not fit in temp space |
+| `spi_ra_wave_simd_full_csn`    | SIMD-cycles | Sum of SIMD where WAVE cannot take csn wave when not fits    |
+| `spi_ra_vgpr_simd_full_csn[∗]` | SIMD-cycles | Sum of SIMD where VGPR cannot take csn wave when not fits    |
+| `spi_ra_sgpr_simd_full_csn[∗]` | SIMD-cycles | Sum of SIMD where SGPR cannot take csn wave when not fits    |
+| `spi_ra_lds_cu_full_csn`       | CUs         | Sum of CU where LDS cannot take csn wave when not fits       |
+| `spi_ra_bar_cu_full_csn[∗]`    | CUs         | Sum of CU where BARRIER cannot take csn wave when not fits   |
+| `spi_ra_bulky_cu_full_csn[∗]`  | CUs         | Sum of CU where BULKY cannot take csn wave when not fits     |
+| `spi_ra_tglim_cu_full_csn[∗]`  | Cycles      | Cycles where csn wants to req but all CUs are at tg_limit    |
+| `spi_ra_wvlim_cu_full_csn[∗]`  | Cycles      | Number of clocks csn is stalled due to WAVE LIMIT            |
+| `spi_vwc_csc_wr`               | Cycles      | Number of clocks to write CSC waves to VGPRs (need to multiply this value by 4) |
+| `spi_swc_csc_wr`               | Cycles      | Number of clocks to write CSC waves to SGPRs (need to multiply this value by 4) |
+
+### Compute unit
+
+The compute unit counters are further classified into instruction mix, MFMA operation counters, level counters, wavefront counters, wavefront cycle counters, local data share counters, and others.
+
+#### Instruction mix
+
+| Hardware Counter        | Unit   | Definition                                                               |
+| :-----------------------| :-----:| -----------------------------------------------------------------------: |
+| `sq_insts`                | Instr | Number of instructions issued                                             |
+| `sq_insts_valu`           | Instr | Number of VALU instructions issued, including MFMA                        |
+| `sq_insts_valu_add_f16`   | Instr | Number of VALU F16 Add instructions issued                                |
+| `sq_insts_valu_mul_f16`   | Instr | Number of VALU F16 Multiply instructions issued                           |
+| `sq_insts_valu_fma_f16`   | Instr | Number of VALU F16 FMA instructions issued                                |
+| `sq_insts_valu_trans_f16` | Instr | Number of VALU F16 Transcendental instructions issued                     |
+| `sq_insts_valu_add_f32`   | Instr | Number of VALU F32 Add instructions issued                                |
+| `sq_insts_valu_mul_f32`   | Instr | Number of VALU F32 Multiply instructions issued                           |
+| `sq_insts_valu_fma_f32`   | Instr | Number of VALU F32 FMA instructions issued                                |
+| `sq_insts_valu_trans_f32` | Instr | Number of VALU F32 Transcendental instructions issued                     |
+| `sq_insts_valu_add_f64`   | Instr | Number of VALU F64 Add instructions issued                                |
+| `sq_insts_valu_mul_f64`   | Instr | Number of VALU F64 Multiply instructions issued                           |
+| `sq_insts_valu_fma_f64`   | Instr | Number of VALU F64 FMA instructions issued                                |
+| `sq_insts_valu_trans_f64` | Instr | Number of VALU F64 Transcendental instructions issued                     |
+| `sq_insts_valu_int32`     | Instr | Number of VALU 32-bit integer instructions issued (signed or unsigned)    |
+| `sq_insts_valu_int64`     | Instr | Number of VALU 64-bit integer instructions issued (signed or unsigned)    |
+| `sq_insts_valu_cvt`       | Instr | Number of VALU Conversion instructions issued                             |
+| `sq_insts_valu_mfma_i8`   | Instr | Number of 8-bit Integer MFMA instructions issued                          |
+| `sq_insts_valu_mfma_f16`  | Instr | Number of F16 MFMA instructions issued                                    |
+| `sq_insts_valu_mfma_bf16` | Instr | Number of BF16 MFMA instructions issued                                   |
+| `sq_insts_valu_mfma_f32`  | Instr | Number of F32 MFMA instructions issued                                    |
+| `sq_insts_valu_mfma_f64`  | Instr | Number of F64 MFMA instructions issued                                    |
+| `sq_insts_mfma`           | Instr | Number of MFMA instructions issued                                        |
+| `sq_insts_vmem_wr`        | Instr | Number of VMEM write instructions issued                                  |
+| `sq_insts_vmem_rd`        | Instr | Number of VMEM read instructions issued                                   |
+| `sq_insts_vmem`           | Instr | Number of VMEM instructions issued, including both FLAT and buffer instructions |
+| `sq_insts_salu`           | Instr | Number of SALU instructions issued                                        |
+| `sq_insts_smem`           | Instr | Number of SMEM instructions issued                                        |
+| `sq_insts_smem_norm`      | Instr | Number of SMEM instructions issued to normalize to match `smem_level`. Used in measuring SMEM latency |
+| `sq_insts_flat`           | Instr | Number of FLAT instructions issued                                        |
+| `sq_insts_flat_lds_only`  | Instr | Number of FLAT instructions issued that read/write only from/to LDS       |
+| `sq_insts_lds`            | Instr | Number of LDS instructions issued                                         |
+| `sq_insts_gds`            | Instr | Number of GDS instructions issued                                         |
+| `sq_insts_exp_gds`        | Instr | Number of EXP and GDS instructions excluding skipped export instructions issued |
+| `sq_insts_branch`         | Instr | Number of Branch instructions issued                                      |
+| `sq_insts_sendmsg`        | Instr | Number of SENDMSG instructions including s_endpgm issued                  |
+| `sq_insts_vskipped[∗]`    | Instr | Number of VSkipped instructions issued                                    |
+
+#### MFMA operation counters
+
+| Hardware Counter             | Unit  | Definition                                      |
+| :----------------------------| :-----| ----------------------------------------------: |
+| `sq_insts_valu_mfma_mops_I8`   | IOP   | Number of 8-bit integer MFMA ops in unit of 512 |
+| `sq_insts_valu_mfma_mops_F16`  | FLOP  | Number of F16 floating MFMA ops in unit of 512  |
+| `sq_insts_valu_mfma_mops_BF16` | FLOP  | Number of BF16 floating MFMA ops in unit of 512 |
+| `sq_insts_valu_mfma_mops_F32`  | FLOP  | Number of F32 floating MFMA ops in unit of 512  |
+| `sq_insts_valu_mfma_mops_F64`  | FLOP  | Number of F64 floating MFMA ops in unit of 512  |
+
+#### Level counters
+
+| Hardware Counter    | Unit  | Definition                             |
+| :-------------------| :-----| -------------------------------------: |
+| `sq_accum_prev`       | Count | Accumulated counter sample value where accumulation takes place once every  four cycles |
+| `sq_accum_prev_hires` | Count | Accumulated counter sample value where accumulation takes place once every cycle |
+| `sq_level_waves`      | Waves | Number of inflight waves               |
+| `sq_insts_level_vmem` | Instr | Number of inflight VMEM instructions   |
+| `sq_insts_level_smem` | Instr | Number of inflight SMEM instructions   |
+| `sq_insts_level_lds`  | Instr | Number of inflight LDS instructions    |
+| `sq_ifetch_level`     | Instr | Number of inflight instruction fetches |
+
+#### Wavefront counters
+
+| Hardware Counter     | Unit  | Definition                                                        |
+| :--------------------| :-----| ----------------------------------------------------------------: |
+| `sq_waves`             | Waves | Number of wavefronts dispatch to SQs, including both new and restored wavefronts |
+| `sq_waves_saved[∗]`    | Waves | Number of context-saved wavefronts                                |
+| `sq_waves_restored[∗]` | Waves | Number of context-restored wavefronts                             |
+| `sq_waves_eq_64`       | Waves | Number of wavefronts with exactly 64 active threads sent to SQs   |
+| `sq_waves_lt_64`       | Waves | Number of wavefronts with less than 64 active threads sent to SQs |
+| `sq_waves_lt_48`       | Waves | Number of wavefronts with less than 48 active threads sent to SQs |
+| `sq_waves_lt_32`       | Waves | Number of wavefronts with less than 32 active threads sent to SQs |
+| `sq_waves_lt_16`       | Waves | Number of wavefronts with less than 16 active threads sent to SQs |
+
+#### Wavefront cycle counters
+
+| Hardware Counter         | Unit    | Definition                                                            |
+| :------------------------| :-------| --------------------------------------------------------------------: |
+| `sq_cycles`                | Cycles  | Free-running  SQ clocks                                               |
+| `sq_busy_cycles`           | Cycles  | Number of cycles while SQ reports it to be busy                       |
+| `sq_busy_cu_cycles`        | Qcycles | Number of quad cycles each CU is busy                                 |
+| `sq_valu_mfma_busy_cycles` | Cycles  | Number of cycles the MFMA ALU is busy                                 |
+| `sq_wave_cycles`           | Qcycles | Number of quad cycles spent by waves in the CUs                       |
+| `sq_wait_any`              | Qcycles | Number of quad cycles spent waiting for anything                      |
+| `sq_wait_inst_any`         | Qcycles | Number of quad cycles spent waiting for an issued instruction         |
+| `sq_active_inst_any`       | Qcycles | Number of quad cycles spent by each wave to work on an instruction    |
+| `sq_active_inst_vmem`      | Qcycles | Number of quad cycles spent by each wave to work on a non-FLAT VMEM instruction |
+| `sq_active_inst_lds`       | Qcycles | Number of quad cycles spent by each wave to work on an LDS instruction |
+| `sq_active_inst_valu`      | Qcycles | Number of quad cycles spent by each wave to work on a VALU instruction |
+| `sq_active_inst_sca`       | Qcycles | Number of quad cycles spent by each wave to work on an SCA instruction |
+| `sq_active_inst_exp_gds`   | Qcycles | Number of quad cycles spent by each wave to work on EXP or GDS instruction |
+| `sq_active_inst_misc`      | Qcycles | Number of quad cycles spent by each wave to work on an MISC instruction, including branch and sendmsg |
+| `sq_active_inst_flat`      | Qcycles | Number of quad cycles spent by each wave to work on a FLAT instruction |
+| `sq_inst_cycles_vmem_wr`   | Qcycles | Number of quad cycles  spent to send addr and cmd data for VMEM write instructions, including both FLAT and buffer |
+| `sq_inst_cycles_vmem_rd`   | Qcycles | Number of quad cycles  spent to send addr and cmd data for VMEM read instructions, including both FLAT and buffer |
+| `sq_inst_cycles_smem`      | Qcycles | Number of quad cycles  spent to execute scalar memory reads           |
+| `sq_inst_cycles_salu`      | Cycles  | Number of cycles spent to execute non-memory read scalar operations   |
+| `sq_thread_cycles_valu`    | Cycles  | Number of thread cycles spent to execute VALU operations              |
+
+#### Local data share
+
+| Hardware Counter           | Unit   | Definition                                                |
+| :--------------------------| :------| --------------------------------------------------------: |
+| `sq_lds_atomic_return`       | Cycles | Number of atomic return cycles in LDS                     |
+| `sq_lds_bank_conflict`       | Cycles | Number of cycles LDS is stalled by bank conflicts         |
+| `sq_lds_addr_conflict[∗]`    | Cycles | Number of cycles LDS is stalled by address conflicts      |
+| `sq_lds_unaligned_stalls[∗]` | Cycles | Number of cycles LDS is stalled processing flat unaligned load/store ops |
+| `sq_lds_mem_violations[∗]`   | Count  | Number of threads that have a memory violation in the LDS |
+
+#### Miscellaneous
+
+##### Local data share
+
+| Hardware Counter | Unit    | Definition                                                |
+| :----------------| :-------| --------------------------------------------------------: |
+| `sq_ifetch`        | Count   | Number of fetch requests from L1I cache, in 32-byte width |
+| `sq_items`         | Threads | Number of valid threads                                   |
+
+### L1I and sL1D caches
+
+#### L1I and sL1D caches
+
+| Hardware Counter             | Unit   | Definition                                                        |
+| :----------------------------| :------| ----------------------------------------------------------------: |
+| `sqc_icache_req`               | Req    | Number of L1I cache requests                                      |
+| `sqc_icache_hits`              | Count  | Number of L1I cache lookup-hits                                   |
+| `sqc_icache_misses`            | Count  | Number of L1I cache non-duplicate lookup-misses                   |
+| `sqc_icache_misses_duplicate`  | Count  | Number of d L1I cache duplicate lookup misses  whose previous lookup miss on the same cache line is not fulfilled yet |
+| `sqc_dcache_req`               | Req    | Number of sL1D cache requests                                       |
+| `sqc_dcache_input_valid_readb` | Cycles | Number of cycles while SQ input is valid but sL1D cache is not ready |
+| `sqc_dcache_hits`              | Count  | Number of sL1D cache lookup-hits                                  |
+| `sqc_dcache_misses`            | Count  | Number of sL1D non-duplicate lookup-misses                        |
+| `sqc_dcache_misses_duplicate`  | Count  | Number of sL1D duplicate lookup-misses                            |
+| `sqc_dcache_req_read_1`        | Req    | Number of read requests in a single 32-bit data word, DWORD (DW)  |
+| `sqc_dcache_req_read_2`        | Req    | Number of read requests in 2 DW                                   |
+| `sqc_dcache_req_read_4`        | Req    | Number of read requests in 4 DW                                   |
+| `sqc_dcache_req_read_8`        | Req    | Number of read requests in 8 DW                                   |
+| `sqc_dcache_req_read_16`       | Req    | Number of read requests in 16 DW                                  |
+| `sqc_dcache_atomic[∗]`         | Req    | Number of atomic requests                                         |
+| `sqc_tc_req`                   | Req    | Number of L2 cache requests that were issued by instruction and constant caches |
+| `sqc_tc_inst_req`              | Req    | Number of instruction cache line requests to L2 cache             |
+| `sqc_tc_data_read_req`         | Req    | Number of data read requests to the L2 cache                      |
+| `sqc_tc_data_write_req[∗]`     | Req    | Number of data write requests to the L2 cache                     |
+| `sqc_tc_data_atomic_req[∗]`    | Req    | Number of data atomic requests to the L2 cache                    |
+| `sqc_tc_stall[∗]`              | Cycles | Number of cycles while the valid requests to L2 cache are stalled |
+
+### Vector L1 cache subsystem
+
+The vector L1 cache subsystem counters are further classified into texture addressing unit, texture data unit, vector L1D cache, and texture cache arbiter.
+
+#### Texture addressing unit
+
+##### Texture addressing unit counters
+
+| Hardware Counter                 | Unit   | Definition                                        |
+| :--------------------------------| :------| ------------------------------------------------: |
+| `ta_ta_busy`                       | Cycles | texture addressing unit busy cycles                                    |
+| `ta_total_wavefronts`              | Instr  | Number of wavefront instructions                  |
+| `ta_buffer_wavefronts`             | Instr  | Number of buffer wavefront instructions           |
+| `ta_buffer_read_wavefronts`        | Instr  | Number of buffer read wavefront instructions      |
+| `ta_buffer_write_wavefronts`       | Instr  | Number of buffer write wavefront instructions     |
+| `ta_buffer_atomic_wavefronts[∗]`   | Instr  | Number of buffer atomic wavefront instructions    |
+| `ta_buffer_total_cycles`           | Cycles | Number of buffer cycles, including read and write |
+| `ta_buffer_coalesced_read_cycles`  | Cycles | Number of coalesced buffer read cycles            |
+| `ta_buffer_coalesced_write_cycles` | Cycles | Number of coalesced buffer write cycles           |
+| `ta_addr_stalled_by_tc`            | Cycles | Number of cycles texture addressing unit address is stalled by TCP     |
+| `ta_data_stalled_by_tc`            | Cycles | Number of cycles texture addressing unit data is stalled by TCP        |
+| `ta_addr_stalled_by_td_cycles[∗]`  | Cycles | Number of cycles texture addressing unit address is stalled by TD      |
+| `ta_flat_wavefronts`               | Instr  | Number of flat wavefront instructions             |
+| `ta_flat_read_wavefronts`          | Instr  | Number of flat read wavefront instructions        |
+| `ta_flat_write_wavefronts`         | Instr  | Number of flat write wavefront instructions       |
+| `ta_flat_atomic_wavefronts`        | Instr  | Number of flat atomic wavefront instructions      |
+
+#### Texture data unit
+
+##### Texture data unit counters
+
+| Hardware Counter         | Unit  | Definition                                           |
+| :------------------------| :-----| ---------------------------------------------------: |
+| `td_td_busy`               | Cycle | TD busy cycles                                       |
+| `td_tc_stall`              | Cycle | Number of cycles TD is stalled by TCP                |
+| `td_spi_stall[∗]`          | Cycle | Number of cycles TD is stalled by SPI                |
+| `td_load_wavefront`        | Instr | Number of wavefront instructions (read/write/atomic) |
+| `td_store_wavefront`       | Instr | Number of write wavefront instructions               |
+| `td_atomic_wavefront`      | Instr | Number of atomic wavefront instructions              |
+| `td_coalescable_wavefront` | Instr | Number of coalescable instructions                   |
+
+#### Vector L1D cache
+
+| Hardware Counter                    | Unit   | Definition                                                  |
+| :-----------------------------------| :------| ----------------------------------------------------------: |
+| `tcp_gate_en1`                        | Cycles | Number of cycles/ vL1D interface clocks are turned on    |
+| `tcp_gate_en2`                        | Cycles | Number of cycles vL1D core clocks are turned on           |
+| `tcp_td_tcp_stall_cycles`             | Cycles | Number of cycles TD stalls vL1D                           |
+| `tcp_tcr_tcp_stall_cycles`            | Cycles | Number of cycles TCR stalls vL1D                           |
+| `tcp_read_tagconflict_stall_cycles`   | Cycles | Number of cycles tagram conflict stalls on a read          |
+| `tcp_write_tagconflict_stall_cycles`  | Cycles | Number of cycles tagram conflict stalls on a write         |
+| `tcp_atomic_tagconflict_stall_cycles` | Cycles | Number of cycles tagram conflict stalls on an atomic       |
+| `tcp_pending_stall_cycles`            | Cycles | Number of cycles vL1D cache is stalled due to data pending from L2 cache |
+| `tcp_ta_tcp_state_read`               | Req    | Number of wavefront instruction requests to vL1D           |
+| `tcp_volatile[∗]`                     | Req    | Number of L1 volatile pixels/buffers from texture addressing unit               |
+| `tcp_total_accesses`                  | Req    | Number of vL1D accesses                                    |
+| `tcp_total_read`                      | Req    | Number of vL1D read accesses                               |
+| `tcp_total_write`                     | Req    | Number of vL1D write accesses                              |
+| `tcp_total_atomic_with_ret`           | Req    | Number of vL1D atomic with return                          |
+| `tcp_total_atomic_without_ret`        | Req    | Number of vL1D atomic without return                       |
+| `tcp_total_writeback_invalidates`     | Count  | Number of vL1D writebacks and Invalidates                  |
+| `tcp_utcl1_request`                   | Req    | Number of address translation requests to UTCL1            |
+| `tcp_utcl1_translation_hit`           | Req    | Number of UTCL1 translation hits                            |
+| `tcp_utcl1_translation_miss`          | Req    | Number of UTCL1 translation misses                          |
+| `tcp_utcl1_persmission_miss`          | Req    | Number of UTCL1 permission misses                           |
+| `tcp_total_cache_accesses`            | Req    | Number of vL1D cache accesses                               |
+| `tcp_tcp_latency`                     | Cycles | Accumulated wave access latency to vL1D over all wavefronts |
+| `tcp_tcc_read_req_latency`            | Cycles | Accumulated vL1D-L2 request latency over all wavefronts for reads and atomics with return |
+| `tcp_tcc_write_req_latency`           | Cycles | Accumulated vL1D-L2 request latency over all wavefronts for writes and atomics without return |
+| `tcp_tcc_read_req`                    | Req    | Number of read requests to L2 cache                        |
+| `tcp_tcc_write_req`                   | Req    | Number of write requests to L2 cache                       |
+| `tcp_tcc_atomic_with_ret_req`         | Req    | Number of atomic requests to L2 cache with return          |
+| `tcp_tcc_atomic_without_ret_req`      | Req    | Number of atomic requests to L2 cache without return       |
+| `tcp_tcc_nc_read_req`                 | Req    | Number of NC read requests to L2 cache                     |
+| `tcp_tcc_uc_read_req`                 | Req    | Number of UC read requests to L2 cache                     |
+| `tcp_tcc_cc_read_req`                 | Req    | Number of CC read requests to L2 cache                     |
+| `tcp_tcc_rw_read_req`                 | Req    | Number of RW read requests to L2 cache                     |
+| `tcp_tcc_nc_write_req`                | Req    | Number of NC write requests to L2 cache                    |
+| `tcp_tcc_uc_write_req`                | Req    | Number of UC write requests to L2 cache                    |
+| `tcp_tcc_cc_write_req`                | Req    | Number of CC write requests to L2 cache                    |
+| `tcp_tcc_rw_write_req`                | Req    | Number of RW write requests to L2 cache                    |
+| `tcp_tcc_nc_atomic_req`               | Req    | Number of NC atomic requests to L2 cache                   |
+| `tcp_tcc_uc_atomic_req`               | Req    | Number of UC atomic requests to L2 cache                   |
+| `tcp_tcc_cc_atomic_req`               | Req    | Number of CC atomic requests to L2 cache                   |
+| `tcp_tcc_rw_atomic_req`               | Req    | Number of RW atomic requests to L2 cache                   |
+
+#### TCA
+
+| Hardware Counter | Unit   | Definition                                  |
+| :----------------| :------| ------------------------------------------: |
+| `tca_cycle`        | Cycles | TCA cycles                                  |
+| `tca_busy`         | Cycles | Number of cycles  TCA has a pending request |
+
+### L2 cache access
+
+#### L2 cache access counters
+
+| Hardware Counter                 | Unit   | Definition                                                     |
+| :--------------------------------| :------| -------------------------------------------------------------: |
+| `tcc_cycle`                        |Cycle   | L2 cache free-running clocks                                  |
+| `tcc_busy`                         |Cycle   | L2 cache busy cycles                                          |
+| `tcc_req`                          |Req     | Number of L2 cache requests                                   |
+| `tcc_streaming_req[∗]`             |Req     | Number of L2 cache streaming requests                         |
+| `tcc_NC_req`                       |Req     | Number of NC requests                                         |
+| `tcc_UC_req`                       |Req     | Number of UC requests                                         |
+| `tcc_CC_req`                       |Req     | Number of CC requests                                         |
+| `tcc_RW_req`                       |Req     | Number of RW requests                                         |
+| `tcc_probe`                        |Req     | Number of L2 cache probe requests                             |
+| `tcc_probe_all[∗]`                 |Req     | Number of external probe requests with EA_TCC_preq_all== 1    |
+| `tcc_read_req`                     |Req     | Number of L2 cache read requests                              |
+| `tcc_write_req`                    |Req     | Number of L2 cache write requests                             |
+| `tcc_atomic_req`                   |Req     | Number of L2 cache atomic requests                            |
+| `tcc_hit`                          |Req     | Number of L2 cache lookup-hits                                |
+| `tcc_miss`                         |Req     | Number of L2 cache lookup-misses                              |
+| `tcc_writeback`                    |Req     | Number of lines written back to main memory, including writebacks of dirty lines and uncached write/atomic requests |
+| `tcc_ea_wrreq`                     |Req     | Total number of 32-byte and 64-byte write requests to EA      |
+| `tcc_ea_wrreq_64B`                 |Req     | Total number of 64-byte write requests to EA                  |
+| `tcc_ea_wr_uncached_32B`           |Req     | Number of 32-byte write/atomic going over the TC_EA_wrreq interface due to uncached traffic. Note that CC mtypes can produce uncached requests, and those are included in this. A 64-byte request is counted as 2. |
+| `tcc_ea_wrreq_stall`               | Cycles | Number of cycles a write request was stalled                  |
+| `tcc_ea_wrreq_io_credit_stall[∗]`  | Cycles | Number of cycles an EA write request runs out of IO credits   |
+| `tcc_ea_wrreq_gmi_credit_stall[∗]` | Cycles | Number of cycles an EA write request runs out of GMI credits  |
+| `tcc_ea_wrreq_dram_credit_stall`   | Cycles | Number of cycles an EA write request runs out of DRAM credits |
+| `tcc_too_many_ea_wrreqs_stall[∗]`  | Cycles | Number of cycles the L2 cache reaches maximum number of pending EA write requests |
+| `tcc_ea_wrreq_level`               | Req    | Accumulated number of L2 cache-EA write requests in flight    |
+| `tcc_ea_atomic`                    | Req    | Number of 32-byte and 64-byte atomic requests to EA           |
+| `tcc_ea_atomic_level`              | Req    | Accumulated number of L2 cache-EA atomic requests in flight   |
+| `tcc_ea_rdreq`                     | Req    | Total number of 32-byte and 64-byte read requests to EA       |
+| `tcc_ea_rdreq_32B`                 | Req    | Total number of 32-byte read requests to EA                   |
+| `tcc_ea_rd_uncached_32B`           | Req    | Number of 32-byte L2 cache-EA read due to uncached traffic. A 64-byte request is counted as 2. |
+| `tcc_ea_rdreq_io_credit_stall[∗]`  | Cycles | Number of cycles read request interface runs out of IO credits  |
+| `tcc_ea_rdreq_gmi_credit_stall[∗]` | Cycles | Number of cycles read request interface runs out of GMI credits |
+| `tcc_ea_rdreq_dram_credit_stall`   | Cycles | Number of cycles read request interface runs out of DRAM credits |
+| `tcc_ea_rdreq_level`               | Req    | Accumulated number of L2 cache-EA read requests in flight     |
+| `tcc_ea_rdreq_dram`                | Req    | Number of 32-byte and 64-byte read requests to HBM            |
+| `tcc_ea_wrreq_dram`                | Req    | Number of 32-byte and 64-byte write requests to HBM           |
+| `tcc_tag_stall`                    | Cycles | Number of cycles the normal request pipeline in the tag was stalled for any reason |
+| `tcc_normal_writeback`             | Req    | Number of L2 cache normal writeback                           |
+| `tcc_all_tc_op_wb_writeback[∗]`    | Req    | Number of instruction-triggered writeback requests            |
+| `tcc_normal_evict`                 | Req    | Number of L2 cache normal evictions                           |
+| `tcc_all_tc_op_inv_evict[∗]`       | Req    | Number of instruction-triggered eviction requests             |
+
+## MI200 derived metrics list
+
+### Derived metrics on MI200 GPUs
+
+| Derived Metric   | Description                                                                            |
+| :----------------| -------------------------------------------------------------------------------------: |
+| `VFetchInsts`      | The average number of vector fetch instructions from the video memory executed per work-item (affected by flow control). Excludes FLAT instructions that fetch from video memory               |
+| `VWriteInsts`      | The average number of vector write instructions to the video memory executed per work-item (affected by flow control). Excludes FLAT instructions that write to video memory                 |
+| `FlatVMemInsts`    | The average number of FLAT instructions that read from or write to the video memory executed per work item (affected by flow control). Includes FLAT instructions that read from or write to scratch |
+| `LDSInsts`         | The average number of LDS read/write instructions executed per work item (affected by flow control). Excludes FLAT instructions that read from or write to LDS |
+| `FlatLDSInsts`     | The average number of FLAT instructions that read or write to LDS executed per work item (affected by flow control) |
+| `VALUUtilization`  | The percentage of active vector ALU threads in a wave. A lower number can mean either more thread divergence in a wave or that the work-group size is not a multiple of 64. Value range: 0% (bad), 100% (ideal - no thread divergence) |
+| `VALUBusy`         | The percentage of GPU time vector ALU instructions are processed. Value range: 0% (bad) to 100% (optimal) |
+| `SALUBusy`         | The percentage of GPU time scalar ALU instructions are processed. Value range: 0% (bad) to 100% (optimal) |
+| `MemWrites32B`     | The total number of effective 32B write transactions to the memory                      |
+| `L2CacheHit`       | The percentage of fetch, write, atomic, and other instructions that hit the data in L2 cache. Value range: 0% (no hit) to 100% (optimal) |
+| `MemUnitStalled`   | The percentage of GPU time the memory unit is stalled. Try reducing the number or size of fetches and writes if possible. Value range: 0% (optimal) to 100% (bad) |
+| `WriteUnitStalled` | The percentage of GPU time the write unit is stalled. Value range: 0% to 100% (bad)      |
+| `LDSBankConflict`  | The percentage of GPU time LDS is stalled by bank conflicts. Value range: 0% (optimal) to 100% (bad) |
+
+## MI200 acronyms
+
+| Abbreviation | Meaning                                                                           |
+| :------------| --------------------------------------------------------------------------------: |
+| `ALU`          | Arithmetic logic unit |
+| `Arb`          | Arbiter |
+| `BF16`        | Brain floating point – 16 |
+| `CC`           | Coherently cached |
+| `CP`           | Command processor |
+| `CPC`         | Command processor – compute |
+| `CPF`         | Command processor – fetcher |
+| `CS`           | Compute shader |
+| `CSC`         | Compute shader controller |
+| `CSn`          | Compute Shader, the n-th pipe |
+| `CU`           | Compute unit |
+| `DW`           | 32-bit data word, DWORD |
+| `EA`           | Efficiency arbiter |
+| `F16`          | Half-precision floating point |
+| `FLAT`       | FLAT instructions allow read/write/atomic access to a generic memory address pointer, which can resolve to any of the following physical memories:<br>•   Global Memory<br>•   Scratch (“private”)<br>•   LDS (“shared”)<br>•   Invalid – MEM_VIOL TrapStatus |
+| `FMA`          | Fused multiply-add |
+| `GDS`          | Global data share |
+| `GRBM`         | Graphics register bus manager |
+| `HBM`          | High bandwidth memory |
+| `Instr`        | Instructions |
+| `IOP`          | Integer operation |
+| `L2`           | Level-2 cache |
+| `LDS`          | Local data share |
+| `ME1`          | Micro-engine, running packet processing firmware on CPC |
+| `MFMA`         | Matrix fused multiply-add |
+| `NC`           | Noncoherently cached |
+| `RW`           | Coherently cached with write |
+| `SALU`         | Scalar ALU |
+| `SGPR`         | Scalar GPR |
+| `SIMD`         | Single instruction multiple data |
+| `sL1D`         | Scalar Level-1 data cache |
+| `SMEM`         | Scalar memory |
+| `SPI`          | Shader processor input |
+| `SQ`           | Sequencer |
+| `TA`           | Texture addressing unit |
+| `TC`           | Texture cache |
+| `TCA`          | Texture cache arbiter |
+| `TCC`          | Texture cache per channel, known as L2 cache |
+| `TCIU`         | Texture cache interface unit, command processor’s interface to memory system |
+| `TCP`          | Texture cache per pipe, known as vector L1 cache |
+| `TCR`          | Texture cache router |
+| `TD`           | Texture data unit |
+| `UC`           | Uncached |
+| `UTCL1`        | Unified translation cache – level 1 |
+| `UTCL2`        | Unified translation cache – level 2 |
+| `VALU`         | Vector ALU |
+| `VGPR`         | Vector GPR |
+| `vL1D`         | Vector level 1 data cache |
+| `VMEM`         | Vector memory |

docs/conceptual/gpu-arch/mi250.md

@@ -0,0 +1,127 @@
+# AMD Instinct™ MI250 microarchitecture
+
+The microarchitecture of the AMD Instinct MI250 accelerators is based on the
+AMD CDNA 2 architecture that targets compute applications such as HPC,
+artificial intelligence (AI), and machine learning (ML) and that run on
+everything from individual servers to the world’s largest exascale
+supercomputers. The overall system architecture is designed for extreme
+scalability and compute performance.
+
+The following image shows the components of a single Graphics Compute Die (GCD) of the CDNA 2 architecture. On the top and the bottom are AMD Infinity Fabric™
+interfaces and their physical links that are used to connect the GPU die to the
+other system-level components of the node (see also Section 2.2). Both
+interfaces can drive four AMD Infinity Fabric links. One of the AMD Infinity
+Fabric links of the controller at the bottom can be configured as a PCIe link.
+Each of the AMD Infinity Fabric links between GPUs can run at up to 25 GT/sec,
+which correlates to a peak transfer bandwidth of 50 GB/sec for a 16-wide link (
+two bytes per transaction). Section 2.2 has more details on the number of AMD
+Infinity Fabric links and the resulting transfer rates between the system-level
+components.
+
+To the left and the right are memory controllers that attach the High Bandwidth
+Memory (HBM) modules to the GCD. AMD Instinct MI250 GPUs use HBM2e, which offers
+a peak memory bandwidth of 1.6 TB/sec per GCD.
+
+The execution units of the GPU are depicted in the following image as Compute
+Units (CU). The MI250 GCD has 104 active CUs. Each compute unit is further
+subdivided into four SIMD units that process SIMD instructions of 16 data
+elements per instruction (for the FP64 data type). This enables the CU to
+process 64 work items (a so-called “wavefront”) at a peak clock frequency of 1.7
+GHz. Therefore, the theoretical maximum FP64 peak performance per GCD is 45.3
+TFLOPS for vector instructions. The MI250 compute units also provide specialized
+execution units (also called matrix cores), which are geared toward executing
+matrix operations like matrix-matrix multiplications. For FP64, the peak
+performance of these units amounts to 90.5 TFLOPS.
+
+![Structure of a single GCD in the AMD Instinct MI250 accelerator.](../../data/conceptual/gpu-arch/image001.png "Structure of a single GCD in the AMD Instinct MI250 accelerator.")
+
+```{list-table} Peak-performance capabilities of the MI250 OAM for different data types.
+:header-rows: 1
+:name: mi250-perf-table
+
+*
+  - Computation and Data Type
+  - FLOPS/CLOCK/CU
+  - Peak TFLOPS
+*
+  - Matrix FP64
+  - 256
+  - 90.5
+*
+  - Vector FP64
+  - 128
+  - 45.3
+*
+  - Matrix FP32
+  - 256
+  - 90.5
+*
+  - Packed FP32
+  - 256
+  - 90.5
+*
+  - Vector FP32
+  - 128
+  - 45.3
+*
+  - Matrix FP16
+  - 1024
+  - 362.1
+*
+  - Matrix BF16
+  - 1024
+  - 362.1
+*
+  - Matrix INT8
+  - 1024
+  - 362.1
+```
+
+The above table summarizes the aggregated peak performance of the AMD
+Instinct MI250 OCP Open Accelerator Modules (OAM, OCP is short for Open Compute
+Platform) and its two GCDs for different data types and execution units. The
+middle column lists the peak performance (number of data elements processed in a
+single instruction) of a single compute unit if a SIMD (or matrix) instruction
+is being retired in each clock cycle. The third column lists the theoretical
+peak performance of the OAM module. The theoretical aggregated peak memory
+bandwidth of the GPU is 3.2 TB/sec (1.6 TB/sec per GCD).
+
+![Dual-GCD architecture of the AMD Instinct MI250 accelerators](../../data/conceptual/gpu-arch/image002.png "Dual-GCD architecture of the AMD Instinct MI250 accelerators")
+
+The following image shows the block diagram of an OAM package that consists
+of two GCDs, each of which constitutes one GPU device in the system. The two
+GCDs in the package are connected via four AMD Infinity Fabric links running at
+a theoretical peak rate of 25 GT/sec, giving 200 GB/sec peak transfer bandwidth
+between the two GCDs of an OAM, or a bidirectional peak transfer bandwidth of
+400 GB/sec for the same.
+
+## Node-level architecture
+
+The following image shows the node-level architecture of a system that is
+based on the AMD Instinct MI250 accelerator. The MI250 OAMs attach to the host
+system via PCIe Gen 4 x16 links (yellow lines). Each GCD maintains its own PCIe
+x16 link to the host part of the system. Depending on the server platform, the
+GCD can attach to the AMD EPYC processor directly or via an optional PCIe switch
+. Note that some platforms may offer an x8 interface to the GCDs, which reduces
+the available host-to-GPU bandwidth.
+
+![Block diagram of AMD Instinct MI250 Accelerators with 3rd Generation AMD EPYC processor](../../data/conceptual/gpu-arch/image003.png "Block diagram of AMD Instinct MI250 Accelerators with 3rd Generation AMD EPYC processor")
+
+The preceding image shows the node-level architecture of a system with AMD
+EPYC processors in a dual-socket configuration and four AMD Instinct MI250
+accelerators. The MI250 OAMs attach to the host processors system via PCIe Gen 4
+x16 links (yellow lines). Depending on the system design, a PCIe switch may
+exist to make more PCIe lanes available for additional components like network
+interfaces and/or storage devices. Each GCD maintains its own PCIe x16 link to
+the host part of the system or to the PCIe switch. Please note, some platforms
+may offer an x8 interface to the GCDs, which will reduce the available
+host-to-GPU bandwidth.
+
+Between the OAMs and their respective GCDs, a peer-to-peer (P2P) network allows
+for direct data exchange between the GPU dies via AMD Infinity Fabric links (
+black, green, and red lines). Each of these 16-wide links connects to one of the
+two GPU dies in the MI250 OAM and operates at 25 GT/sec, which corresponds to a
+theoretical peak transfer rate of 50 GB/sec per link (or 100 GB/sec
+bidirectional peak transfer bandwidth). The GCD pairs 2 and 6 as well as GCDs 0
+and 4 connect via two XGMI links, which is indicated by the thicker red line in
+the preceding image.

docs/conceptual/gpu-isolation.md

@@ -0,0 +1,109 @@
+# GPU isolation techniques
+
+Restricting the access of applications to a subset of GPUs, aka isolating
+GPUs allows users to hide GPU resources from programs. The programs by default
+will only use the "exposed" GPUs ignoring other (hidden) GPUs in the system.
+
+There are multiple ways to achieve isolation of GPUs in the ROCm software stack,
+differing in which applications they apply to and the security they provide.
+This page serves as an overview of the techniques.
+
+## Environment variables
+
+The runtimes in the ROCm software stack read these environment variables to
+select the exposed or default device to present to applications using them.
+
+Environment variables shouldn't be used for isolating untrusted applications,
+as an application can reset them before initializing the runtime.
+
+### `ROCR_VISIBLE_DEVICES`
+
+A list of device indices or {abbr}`UUID (universally unique identifier)`s
+that will be exposed to applications.
+
+Runtime
+: ROCm Platform Runtime. Applies to all applications using the user mode ROCm
+  software stack.
+
+```{code-block} shell
+:caption: Example to expose the 1. device and a device based on UUID.
+export ROCR_VISIBLE_DEVICES="0,GPU-DEADBEEFDEADBEEF"
+```
+
+### `GPU_DEVICE_ORDINAL`
+
+Devices indices exposed to OpenCL and HIP applications.
+
+Runtime
+: ROCm Common Language Runtime (`ROCclr`). Applies to applications and runtimes
+  using the `ROCclr` abstraction layer including HIP and OpenCL applications.
+
+```{code-block} shell
+:caption: Example to expose the 1. and 3. device in the system.
+export GPU_DEVICE_ORDINAL="0,2"
+```
+
+(hip_visible_devices)=
+
+### `HIP_VISIBLE_DEVICES`
+
+Device indices exposed to HIP applications.
+
+Runtime: HIP runtime. Applies only to applications using HIP on the AMD platform.
+
+```{code-block} shell
+:caption: Example to expose the 1. and 3. devices in the system.
+export HIP_VISIBLE_DEVICES="0,2"
+```
+
+### `CUDA_VISIBLE_DEVICES`
+
+Provided for CUDA compatibility, has the same effect as `HIP_VISIBLE_DEVICES`
+on the AMD platform.
+
+Runtime
+: HIP or CUDA Runtime. Applies to HIP applications on the AMD or NVIDIA platform
+  and CUDA applications.
+
+### `OMP_DEFAULT_DEVICE`
+
+Default device used for OpenMP target offloading.
+
+Runtime
+: OpenMP Runtime. Applies only to applications using OpenMP offloading.
+
+```{code-block} shell
+:caption: Example on setting the default device to the third device.
+export OMP_DEFAULT_DEVICE="2"
+```
+
+## Docker
+
+Docker uses Linux kernel namespaces to provide isolated environments for
+applications. This isolation applies to most devices by default, including
+GPUs. To access them in containers explicit access must be granted, please see
+{ref}`docker-access-gpus-in-container` for details.
+Specifically refer to {ref}`docker-restrict-gpus` on exposing just a subset
+of all GPUs.
+
+Docker isolation is more secure than environment variables, and applies
+to all programs that use the `amdgpu` kernel module interfaces.
+Even programs that don't use the ROCm runtime, like graphics applications
+using OpenGL or Vulkan, can only access the GPUs exposed to the container.
+
+## GPU passthrough to virtual machines
+
+Virtual machines achieve the highest level of isolation, because even the kernel
+of the virtual machine is isolated from the host. Devices physically installed
+in the host system can be passed to the virtual machine using PCIe passthrough.
+This allows for using the GPU with a different operating systems like a Windows
+guest from a Linux host.
+
+Setting up PCIe passthrough is specific to the hypervisor used. ROCm officially
+supports [VMware ESXi](https://www.vmware.com/products/esxi-and-esx.html)
+for select GPUs.
+
+<!--
+TODO: This should link to a page about virtualization that explains
+      pass-through and SR-IOV and how-tos for maybe `libvirt` and `VMWare`
+-->

docs/conceptual/gpu-memory.md

@@ -0,0 +1,234 @@
+# GPU memory
+
+For the HIP reference documentation, see:
+
+* {doc}`hip:.doxygen/docBin/html/group___memory`
+* {doc}`hip:.doxygen/docBin/html/group___memory_m`
+
+Host memory exists on the host (e.g. CPU) of the machine in random access memory (RAM).
+
+Device memory exists on the device (e.g. GPU) of the machine in video random access memory (VRAM).
+Recent architectures use graphics double data rate (GDDR) synchronous dynamic random-access memory (SDRAM)such as GDDR6, or high-bandwidth memory (HBM) such as HBM2e.
+
+## Memory allocation
+
+Memory can be allocated in two ways: pageable memory, and pinned memory.
+The following API calls with result in these allocations:
+
+| API                | Data location | Allocation |
+|--------------------|---------------|------------|
+| System allocated   | Host          | Pageable   |
+| `hipMallocManaged` | Host          | Managed    |
+| `hipHostMalloc`    | Host          | Pinned     |
+| `hipMalloc`        | Device        | Pinned     |
+
+:::{tip}
+`hipMalloc` and `hipFree` are blocking calls, however, HIP recently added non-blocking versions `hipMallocAsync` and `hipFreeAsync` which take in a stream as an additional argument.
+:::
+
+### Pageable memory
+
+Pageable memory is usually gotten when calling `malloc` or `new` in a C++ application.
+It is unique in that it exists on "pages" (blocks of memory), which can be migrated to other memory storage.
+For example, migrating memory between CPU sockets on a motherboard, or a system that runs out of space in RAM and starts dumping pages of RAM into the swap partition of your hard drive.
+
+### Pinned memory
+
+Pinned memory (or page-locked memory, or non-pageable memory) is host memory that is mapped into the address space of all GPUs, meaning that the pointer can be used on both host and device.
+Accessing host-resident pinned memory in device kernels is generally not recommended for performance, as it can force the data to traverse the host-device interconnect (e.g. PCIe), which is much slower than the on-device bandwidth (>40x on MI200).
+
+Pinned host memory can be allocated with one of two types of coherence support:
+
+:::{note}
+In HIP, pinned memory allocations are coherent by default (`hipHostMallocDefault`).
+There are additional pinned memory flags (e.g. `hipHostMallocMapped` and `hipHostMallocPortable`).
+On MI200 these options do not impact performance.
+<!-- TODO: link to programming_manual#memory-allocation-flags -->
+For more information, see the section *memory allocation flags* in the HIP Programming Guide: {doc}`hip:user_guide/programming_manual`.
+:::
+
+Much like how a process can be locked to a CPU core by setting affinity, a pinned memory allocator does this with the memory storage system.
+On multi-socket systems it is important to ensure that pinned memory is located on the same socket as the owning process, or else each cache line will be moved through the CPU-CPU interconnect, thereby increasing latency and potentially decreasing bandwidth.
+
+In practice, pinned memory is used to improve transfer times between host and device.
+For transfer operations, such as `hipMemcpy` or `hipMemcpyAsync`, using pinned memory instead of pageable memory on host can lead to a ~3x improvement in bandwidth.
+
+:::{tip}
+If the application needs to move data back and forth between device and host (separate allocations), use pinned memory on the host side.
+:::
+
+### Managed memory
+
+Managed memory refers to universally addressable, or unified memory available on the MI200 series of GPUs.
+Much like pinned memory, managed memory shares a pointer between host and device and (by default) supports fine-grained coherence, however, managed memory can also automatically migrate pages between host and device.
+The allocation will be managed by AMD GPU driver using the Linux HMM (Heterogeneous Memory Management) mechanism.
+
+If heterogenous memory management (HMM) is not available, then `hipMallocManaged` will default back to using system memory and will act like pinned host memory.
+Other managed memory API calls will have undefined behavior.
+It is therefore recommended to check for managed memory capability with: `hipDeviceGetAttribute` and `hipDeviceAttributeManagedMemory`.
+
+HIP supports additional calls that work with page migration:
+
+* `hipMemAdvise`
+* `hipMemPrefetchAsync`
+
+:::{tip}
+If the application needs to use data on both host and device regularly, does not want to deal with separate allocations, and is not worried about maxing out the VRAM on MI200 GPUs (64 GB per GCD), use managed memory.
+:::
+
+:::{tip}
+If managed memory performance is poor, check to see if managed memory is supported on your system and if page migration (XNACK) is enabled.
+:::
+
+## Access behavior
+
+Memory allocations for GPUs behave as follow:
+
+| API                | Data location | Host access  | Device access        |
+|--------------------|---------------|--------------|----------------------|
+| System allocated   | Host          | Local access | Unhandled page fault |
+| `hipMallocManaged` | Host          | Local access | Zero-copy            |
+| `hipHostMalloc`    | Host          | Local access | Zero-copy*           |
+| `hipMalloc`        | Device        | Zero-copy    | Local access         |
+
+Zero-copy accesses happen over the Infinity Fabric interconnect or PCI-E lanes on discrete GPUs.
+
+:::{note}
+While `hipHostMalloc` allocated memory is accessible by a device, the host pointer must be converted to a device pointer with `hipHostGetDevicePointer`.
+
+Memory allocated through standard system allocators such as `malloc`, can be accessed a device by registering the memory via `hipHostRegister`.
+The device pointer to be used in kernels can be retrieved with `hipHostGetDevicePointer`.
+Registered memory is treated like `hipHostMalloc` and will have similar performance.
+
+On devices that support and have [](#xnack) enabled, such as the MI250X, `hipHostRegister` is not required as memory accesses are handled via automatic page migration.
+:::
+
+### XNACK
+
+Normally, host and device memory are separate and data has to be transferred manually via `hipMemcpy`.
+
+On a subset of GPUs, such as the MI200, there is an option to automatically migrate pages of memory between host and device.
+This is important for managed memory, where the locality of the data is important for performance.
+Depending on the system, page migration may be disabled by default in which case managed memory will act like pinned host memory and suffer degraded performance.
+
+*XNACK* describes the GPUs ability to retry memory accesses that failed due a page fault (which normally would lead to a memory access error), and instead retrieve the missing page.
+
+This also affects memory allocated by the system as indicated by the following table:
+
+| API                | Data location | Host after device access | Device after host access |
+|--------------------|---------------|--------------------------|--------------------------|
+| System allocated   | Host          | Migrate page to host     | Migrate page to device   |
+| `hipMallocManaged` | Host          | Migrate page to host     | Migrate page to device   |
+| `hipHostMalloc`    | Host          | Local access             | Zero-copy                |
+| `hipMalloc`        | Device        | Zero-copy                | Local access             |
+
+To check if page migration is available on a platform, use `rocminfo`:
+
+```sh
+$ rocminfo | grep xnack
+      Name:                    amdgcn-amd-amdhsa--gfx90a:sramecc+:xnack-
+```
+
+Here, `xnack-` means that XNACK is available but is disabled by default.
+Turning on XNACK by setting the environment variable `HSA_XNACK=1` and gives the expected result, `xnack+`:
+
+```sh
+$ HSA_XNACK=1 rocminfo | grep xnack
+Name:                    amdgcn-amd-amdhsa--gfx90a:sramecc+:xnack+
+```
+
+`hipcc`by default will generate code that runs correctly with both XNACK enabled or disabled.
+Setting the `--offload-arch=`-option with `xnack+` or `xnack-` forces code to be only run with XNACK enabled or disabled respectively.
+
+```sh
+# Compiled kernels will run regardless if XNACK is enabled or is disabled. 
+hipcc --offload-arch=gfx90a
+
+# Compiled kernels will only be run if XNACK is enabled with XNACK=1.
+hipcc --offload-arch=gfx90a:xnack+
+
+# Compiled kernels will only be run if XNACK is disabled with XNACK=0.
+hipcc --offload-arch=gfx90a:xnack-
+```
+
+:::{tip}
+If you want to make use of page migration, use managed memory. While pageable memory will migrate correctly, it is not a portable solution and can have performance issues if the accessed data isn't page aligned.
+:::
+
+### Coherence
+
+* *Coarse-grained coherence* means that memory is only considered up to date at kernel boundaries, which can be enforced through `hipDeviceSynchronize`, `hipStreamSynchronize`, or any blocking operation that acts on the null stream (e.g. `hipMemcpy`).
+For example, cacheable memory is a type of coarse-grained memory where an up-to-date copy of the data can be stored elsewhere (e.g. in an L2 cache).
+* *Fine-grained coherence* means the coherence is supported while a CPU/GPU kernel is running.
+This can be useful if both host and device are operating on the same dataspace using system-scope atomic operations (e.g. updating an error code or flag to a buffer).
+Fine-grained memory implies that up-to-date data may be made visible to others regardless of kernel boundaries as discussed above.
+
+| API                     | Flag                         | Coherence      |
+|-------------------------|------------------------------|----------------|
+| `hipHostMalloc`         | `hipHostMallocDefault`       | Fine-grained   |
+| `hipHostMalloc`         | `hipHostMallocNonCoherent`   | Coarse-grained |
+
+| API                     | Flag                         | Coherence      |
+|-------------------------|------------------------------|----------------|
+| `hipExtMallocWithFlags` | `hipHostMallocDefault`       | Fine-grained   |
+| `hipExtMallocWithFlags` | `hipDeviceMallocFinegrained` | Coarse-grained |
+
+| API                     | `hipMemAdvise` argument      | Coherence      |
+|-------------------------|------------------------------|----------------|
+| `hipMallocManaged`      |                              | Fine-grained   |
+| `hipMallocManaged`      | `hipMemAdviseSetCoarseGrain` | Coarse-grained |
+| `malloc`                |                              | Fine-grained   |
+| `malloc`                | `hipMemAdviseSetCoarseGrain` | Coarse-grained |
+
+:::{tip}
+Try to design your algorithms to avoid host-device memory coherence (e.g. system scope atomics). While it can be a useful feature in very specific cases, it is not supported on all systems, and can negatively impact performance by introducing the host-device interconnect bottleneck.
+:::
+
+The availability of fine- and coarse-grained memory pools can be checked with `rocminfo`:
+
+```sh
+$ rocminfo
+...
+*******
+Agent 1
+*******
+Name:                    AMD EPYC 7742 64-Core Processor
+...
+Pool Info:
+Pool 1
+Segment:                 GLOBAL; FLAGS: FINE GRAINED
+...
+Pool 3
+Segment:                 GLOBAL; FLAGS: COARSE GRAINED
+...
+*******
+Agent 9
+*******
+Name:                    gfx90a
+...
+Pool Info:
+Pool 1
+Segment:                 GLOBAL; FLAGS: COARSE GRAINED
+...
+```
+
+## System direct memory access
+
+In most cases, the default behavior for HIP in transferring data from a pinned host allocation to device will run at the limit of the interconnect.
+However, there are certain cases where the interconnect is not the bottleneck.
+
+The primary way to transfer data onto and off of a GPU, such as the MI200, is to use the onboard System Direct Memory Access engine, which is used to feed blocks of memory to the off-device interconnect (either GPU-CPU or GPU-GPU).
+Each GCD has a separate SDMA engine for host-to-device and device-to-host memory transfers.
+Importantly, SDMA engines are separate from the computing infrastructure, meaning that memory transfers to and from a device will not impact kernel compute performance, though they do impact memory bandwidth to a limited extent.
+The SDMA engines are mainly tuned for PCIe-4.0 x16, which means they are designed to operate at bandwidths up to 32 GB/s.
+
+:::{note}
+An important feature of the MI250X platform is the Infinity Fabric™ interconnect between host and device.
+The Infinity Fabric interconnect supports improved performance over standard PCIe-4.0 (usually ~50% more bandwidth); however, since the SDMA engine does not run at this speed, it will not max out the bandwidth of the faster interconnect.
+:::
+
+The bandwidth limitation can be countered by bypassing the SDMA engine and replacing it with a type of copy kernel known as a "blit" kernel.
+Blit kernels will use the compute units on the GPU, thereby consuming compute resources, which may not always be beneficial.
+The easiest way to enable blit kernels is to set an environment variable `HSA_ENABLE_SDMA=0`, which will disable the SDMA engine.
+On systems where the GPU uses a PCIe interconnect instead of an Infinity Fabric interconnect, blit kernels will not impact bandwidth, but will still consume compute resources.
+The use of SDMA vs blit kernels also applies to MPI data transfers and GPU-GPU transfers.

docs/conceptual/using-gpu-sanitizer.md

@@ -0,0 +1,241 @@
+# Using the LLVM ASan on a GPU (beta release)
+
+The LLVM AddressSanitizer (ASan) provides a process that allows developers to detect runtime addressing errors in applications and libraries. The detection is achieved using a combination of compiler-added instrumentation and runtime techniques, including function interception and replacement.
+
+Until now, the LLVM ASan process was only available for traditional purely CPU applications. However, ROCm has extended this mechanism to additionally allow the detection of some addressing errors on the GPU in heterogeneous applications. Ideally, developers should treat heterogeneous HIP and OpenMP applications exactly like pure CPU applications. However, this simplicity has not been achieved yet.
+
+This document provides documentation on using ROCm ASan.
+For information about LLVM ASan, see the [LLVM documentation](https://clang.llvm.org/docs/AddressSanitizer.html).
+
+**Note**: The beta release of LLVM ASan for ROCm is currently tested and validated on Ubuntu 20.04.
+
+## Compiling for ASan
+
+The ASan process begins by compiling the application of interest with the ASan instrumentation.
+
+Recommendations for doing this are:
+
+* Compile as many application and dependent library sources as possible using an AMD-built clang-based compiler such as `amdclang++`.
+* Add the following options to the existing compiler and linker options:
+  * `-fsanitize=address` - enables instrumentation
+  * `-shared-libsan` - use shared version of runtime
+  * `-g` - add debug info for improved reporting
+* Explicitly use `xnack+` in the offload architecture option. For example, `--offload-arch=gfx90a:xnack+`
+Other architectures are allowed, but their device code will not be instrumented and a warning will be emitted.
+
+It is not an error to compile some files without ASan instrumentation, but doing so reduces the ability of the process to detect addressing errors. However, if the main program "`a.out`" does not directly depend on the ASan runtime (`libclang_rt.asan-x86_64.so`) after the build completes (check by running `ldd` (List Dynamic Dependencies) or `readelf`), the application will immediately report an error at runtime as described in the next section.
+
+### About compilation time
+
+When `-fsanitize=address` is used, the LLVM compiler adds instrumentation code around every memory operation. This added code must be handled by all of the downstream components of the compiler toolchain and results in increased overall compilation time. This increase is especially evident in the AMDGPU device compiler and has in a few instances raised the compile time to an unacceptable level.
+
+There are a few options if the compile time becomes unacceptable:
+
+* Avoid instrumentation of the files which have the worst compile times. This will reduce the effectiveness of the ASan process.
+* Add the option `-fsanitize-recover=address` to the compiles with the worst compile times. This option simplifies the added instrumentation resulting in faster compilation. See below for more information.
+* Disable instrumentation on a per-function basis by adding `__attribute__`((no_sanitize("address"))) to functions found to be responsible for the large compile time. Again, this will reduce the effectiveness of the process.
+
+## Installing ROCm GPU ASan packages
+
+For a complete ROCm GPU Sanitizer installation, including packages, instrumented HSA and HIP runtimes, tools, and math libraries, use the following instruction,
+
+```bash
+    sudo apt-get install rocm-ml-sdk-asan
+
+```
+
+## Using AMD-supplied ASan instrumented libraries
+
+ROCm releases have optional packages that contain additional ASan instrumented builds of the ROCm libraries (usually found in `/opt/rocm-<version>/lib`). The instrumented libraries have identical names to the regular uninstrumented libraries, and are located in `/opt/rocm-<version>/lib/asan`.
+These additional libraries are built using the `amdclang++` and `hipcc` compilers, while some uninstrumented libraries are built with g++. The preexisting build options are used but, as described above, additional options are used: `-fsanitize=address`, `-shared-libsan` and `-g`.
+
+These additional libraries avoid additional developer effort to locate repositories, identify the correct branch, check out the correct tags, and other efforts needed to build the libraries from the source. And they extend the ability of the process to detect addressing errors into the ROCm libraries themselves.
+
+When adjusting an application build to add instrumentation, linking against these instrumented libraries is unnecessary. For example, any `-L` `/opt/rocm-<version>/lib` compiler options need not be changed. However, the instrumented libraries should be used when the application is run. It is particularly important that the instrumented language runtimes, like `libamdhip64.so` and `librocm-core.so`, are used; otherwise, device invalid access detections may not be reported.
+
+## Running ASan instrumented applications
+
+### Preparing to run an instrumented application
+
+Here are a few recommendations to consider before running an ASan instrumented heterogeneous application.
+
+* Ensure the Linux kernel running on the system has Heterogeneous Memory Management (HMM) support. A kernel version of 5.6 or higher should be sufficient.
+* Ensure XNACK is enabled
+  * For `gfx90a` (MI-2X0) or `gfx940` (MI-3X0) use environment `HSA_XNACK = 1`.
+  * For `gfx906` (MI-50) or `gfx908` (MI-100) use environment `HSA_XNACK = 1` but also ensure the amdgpu kernel module is loaded with module argument `noretry=0`.
+This requirement is due to the fact that the XNACK setting for these GPUs is system-wide.
+
+* Ensure that the application will use the instrumented libraries when it runs. The output from the shell command `ldd <application name>` can be used to see which libraries will be used.
+If the instrumented libraries are not listed by `ldd`, the environment variable `LD_LIBRARY_PATH` may need to be adjusted, or in some cases an `RPATH` compiled into the application may need to be changed and the application recompiled.
+
+* Ensure that the application depends on the ASan runtime. This can be checked by running the command `readelf -d <application name> | grep NEEDED` and verifying that shared library: `libclang_rt.asan-x86_64.so` appears in the output.
+If it does not appear, when executed the application will quickly output an ASan error that looks like:
+
+```bash
+==3210==ASan runtime does not come first in initial library list; you should either link runtime to your application or manually preload it with LD_PRELOAD.
+```
+
+* Ensure that the application `llvm-symbolizer` can be executed, and that it is located in `/opt/rocm-<version>/llvm/bin`. This executable is not strictly required, but if found is used to translate ("symbolize") a host-side instruction address into a more useful function name, file name, and line number (assuming the application has been built to include debug information).
+
+There is an environment variable, `ASAN_OPTIONS`, that can be used to adjust the runtime behavior of the ASAN runtime itself. There are more than a hundred "flags" that can be adjusted (see an old list at [flags](https://github.com/google/sanitizers/wiki/AddressSanitizerFlags)) but the default settings are correct and should be used in most cases. It must be noted that these options only affect the host ASAN runtime. The device runtime only currently supports the default settings for the few relevant options.
+
+There are two `ASAN_OPTION` flags of particular note.
+
+* `halt_on_error=0/1 default 1`.
+
+This tells the ASAN runtime to halt the application immediately after detecting and reporting an addressing error. The default makes sense because the application has entered the realm of undefined behavior. If the developer wishes to have the application continue anyway, this option can be set to zero. However, the application and libraries should then be compiled with the additional option `-fsanitize-recover=address`. Note that the ROCm optional ASan instrumented libraries are not compiled with this option and if an error is detected within one of them, but halt_on_error is set to 0, more undefined behavior will occur.
+
+* `detect_leaks=0/1 default 1`.
+This option directs the ASan runtime to enable the [Leak Sanitizer](https://clang.llvm.org/docs/LeakSanitizer.html) (LSAN). Unfortunately, for heterogeneous applications, this default will result in significant output from the leak sanitizer when the application exits due to allocations made by the language runtime which are not considered to be to be leaks. This output can be avoided by adding `detect_leaks=0` to the `ASAN_OPTIONS`, or alternatively by producing an LSAN suppression file (syntax described [here](https://github.com/google/sanitizers/wiki/AddressSanitizerLeakSanitizer)) and activating it with environment variable `LSAN_OPTIONS=suppressions=/path/to/suppression/file`. When using a suppression file, a suppression report is printed by default. The suppression report can be disabled by using the `LSAN_OPTIONS` flag `print_suppressions=0`.
+
+## Runtime overhead
+
+Running an ASan instrumented application incurs
+overheads which may result in unacceptably long runtimes
+or failure to run at all.
+
+### Higher execution time
+
+ASan detection works by checking each address at runtime
+before the address is actually accessed by a load, store, or atomic
+instruction.
+This checking involves an additional load to "shadow" memory which
+records whether the address is "poisoned" or not, and additional logic
+that decides whether to produce an detection report or not.
+
+This extra runtime work can cause the application to slow down by
+a factor of three or more, depending on how many memory accesses are
+executed.
+For heterogeneous applications, the shadow memory must be accessible by all devices
+and this can mean that shadow accesses from some devices may be more costly
+than non-shadow accesses.
+
+### Higher memory use
+
+The address checking described above relies on the compiler to surround
+each program variable with a red zone and on ASan
+runtime to surround each runtime memory allocation with a red zone and
+fill the shadow corresponding to each red zone with poison.
+The added memory for the red zones is additional overhead on top
+of the 13% overhead for the shadow memory itself.
+
+Applications which consume most one or more available memory pools when
+run normally are likely to encounter allocation failures when run with
+instrumentation.
+
+## Runtime reporting
+
+It is not the intention of this document to provide a detailed explanation of all of the types of reports that can be output by the ASan runtime. Instead, the focus is on the differences between the standard reports for CPU issues, and reports for GPU issues.
+
+An invalid address detection report for the CPU always starts with
+
+```bash
+==<PID>==ERROR: AddressSanitizer: <problem type> on address <memory address> at pc <pc> bp <bp> sp <sp> <access> of size <N> at <memory address> thread T0
+```
+
+and continues with a stack trace for the access, a stack trace for the allocation and deallocation, if relevant, and a dump of the shadow near the <memory address>.
+
+In contrast, an invalid address detection report for the GPU always starts with
+
+```bash
+==<PID>==ERROR: AddressSanitizer: <problem type> on amdgpu device <device> at pc <pc> <access> of size <n> in workgroup id (<X>,<Y>,<Z>)
+```
+
+Above, `<device>` is the integer device ID, and `(<X>, <Y>, <Z>)` is the ID of the workgroup or block where the invalid address was detected.
+
+While the CPU report include a call stack for the thread attempting the invalid access, the GPU is currently to a call stack of size one, i.e. the (symbolized) of the invalid access, e.g.
+
+```bash
+#0 <pc> in <fuction signature> at /path/to/file.hip:<line>:<column>
+```
+
+This short call stack is followed by a GPU unique section that looks like
+
+```bash
+Thread ids and accessed addresses:
+<lid0> <maddr 0> : <lid1> <maddr1> : ...
+```
+
+where each `<lid j> <maddr j>` indicates the lane ID and the invalid memory address held by lane `j` of the wavefront attempting the invalid access.
+
+Additionally, reports for invalid GPU accesses to memory allocated by GPU code via `malloc` or new starting with, for example,
+
+```bash
+==1234==ERROR: AddressSanitizer: heap-buffer-overflow on amdgpu device 0 at pc 0x7fa9f5c92dcc
+```
+
+or
+
+```bash
+==5678==ERROR: AddressSanitizer: heap-use-after-free on amdgpu device 3 at pc 0x7f4c10062d74
+```
+
+currently may include one or two surprising CPU side tracebacks mentioning :`hostcall`". This is due to how `malloc` and `free` are implemented for GPU code and these call stacks can be ignored.
+
+### Running with `rocgdb`
+
+`rocgdb` can be used to further investigate ASan detected errors, with some preparation.
+
+Currently, the ASan runtime complains when starting `rocgdb` without preparation.
+
+```bash
+$ rocgdb my_app
+==1122==ASan` runtime does not come first in initial library list; you should either link runtime to your application or manually preload it with LD_PRELOAD.
+```
+
+This is solved by setting environment variable `LD_PRELOAD` to the path to the ASan runtime, whose path can be obtained using the command
+
+```bash
+amdclang++ -print-file-name=libclang_rt.asan-x86_64.so
+```
+
+It is also recommended to set the environment variable `HIP_ENABLE_DEFERRED_LOADING=0` before debugging HIP applications.
+
+After starting `rocgdb` breakpoints can be set on the ASan runtime error reporting entry points of interest. For example, if an ASan error report includes
+
+```bash
+WRITE of size 4 in workgroup id (10,0,0)
+```
+
+the `rocgdb` command needed to stop the program before the report is printed is
+
+```bash
+(gdb) break __asan_report_store4
+```
+
+Similarly, the appropriate command for a report including
+
+```bash
+READ of size <N> in workgroup ID (1,2,3)
+```
+
+is
+
+```bash
+(gdb) break __asan_report_load<N>
+```
+
+It is possible to set breakpoints on all ASan report functions using these commands:
+
+```bash
+$ rocgdb <path to application>
+(gdb) start <commmand line arguments>
+(gdb) rbreak ^__asan_report
+(gdb) c
+```
+
+### Using ASan with a short HIP application
+
+Refer to the following example to use ASan with a short HIP application,
+
+https://github.com/Rmalavally/rocm-examples/blob/Rmalavally-patch-1/LLVM_ASAN/Using-Address-Sanitizer-with-a-Short-HIP-Application.md
+
+### Known issues with using GPU sanitizer
+
+* Red zones must have limited size and it is possible for an invalid access to completely miss a red zone and not be detected.
+
+* Lack of detection or false reports can be caused by the runtime not properly maintaining red zone shadows.
+
+* Lack of detection on the GPU might also be due to the implementation not instrumenting accesses to all GPU specific address spaces. For example, in the current implementation accesses to "private" or "stack" variables on the GPU are not instrumented, and accesses to HIP shared variables (also known as "local data store" or "LDS") are also not instrumented.
+
+* It can also be the case that a memory fault is hit for an invalid address even with the instrumentation. This is usually caused by the invalid address being so wild that its shadow address is outside of any memory region, and the fault actually occurs on the access to the shadow address. It is also possible to hit a memory fault for the `NULL` pointer. While address 0 does have a shadow location, it is not poisoned by the runtime.

docs/conf.py

@@ -0,0 +1,107 @@
+# Configuration file for the Sphinx documentation builder.
+#
+# This file only contains a selection of the most common options. For a full
+# list see the documentation:
+# https://www.sphinx-doc.org/en/master/usage/configuration.html
+
+import shutil
+import jinja2
+import os
+
+from rocm_docs import ROCmDocs
+
+# Environement to process Jinja templates.
+jinja_env = jinja2.Environment(loader=jinja2.FileSystemLoader("."))
+
+# Jinja templates to render out.
+templates = [
+
+]
+
+# Render templates and output files without the last extension.
+# For example: 'install.md.jinja' becomes 'install.md'.
+for template in templates:
+    rendered = jinja_env.get_template(template).render()
+    with open(os.path.splitext(template)[0], 'w') as file:
+        file.write(rendered)
+
+shutil.copy2('../CONTRIBUTING.md','./contribute/index.md')
+shutil.copy2('../RELEASE.md','./about/release-notes.md')
+# Keep capitalization due to similar linking on GitHub's markdown preview.
+shutil.copy2('../CHANGELOG.md','./about/CHANGELOG.md')
+
+latex_engine = "xelatex"
+latex_elements = {
+    "fontpkg": r"""
+\usepackage{tgtermes}
+\usepackage{tgheros}
+\renewcommand\ttdefault{txtt}
+"""
+}
+
+# configurations for PDF output by Read the Docs
+project = "ROCm Documentation"
+author = "Advanced Micro Devices, Inc."
+copyright = "Copyright (c) 2023 Advanced Micro Devices, Inc. All rights reserved."
+version = "5.7.1"
+release = "5.7.1"
+setting_all_article_info = True
+all_article_info_os = ["linux", "windows"]
+all_article_info_author = ""
+
+# pages with specific settings
+article_pages = [
+    {
+        "file":"release",
+        "os":["linux", "windows"],
+        "date":"2023-07-27"
+    },
+
+    {"file":"install/windows/install-quick", "os":["windows"]},
+    {"file":"install/linux/install-quick", "os":["linux"]},
+
+    {"file":"install/linux/install", "os":["linux"]},
+    {"file":"install/linux/install-options", "os":["linux"]},
+    {"file":"install/linux/prerequisites", "os":["linux"]},
+
+    {"file":"install/docker", "os":["linux"]},
+    {"file":"install/magma-install", "os":["linux"]},
+    {"file":"install/pytorch-install", "os":["linux"]},
+    {"file":"install/tensorflow-install", "os":["linux"]},
+
+    {"file":"install/windows/install", "os":["windows"]},
+    {"file":"install/windows/prerequisites", "os":["windows"]},
+    {"file":"install/windows/cli/index", "os":["windows"]},
+    {"file":"install/windows/gui/index", "os":["windows"]},
+
+    {"file":"about/compatibility/linux-support", "os":["linux"]},
+    {"file":"about/compatibility/windows-support", "os":["windows"]},
+
+    {"file":"about/compatibility/docker-image-support-matrix", "os":["linux"]},
+    {"file":"about/compatibility/user-kernel-space-compat-matrix", "os":["linux"]},
+
+    {"file":"reference/library-index", "os":["linux"]},
+
+    {"file":"how-to/deep-learning-rocm", "os":["linux"]},
+    {"file":"how-to/gpu-enabled-mpi", "os":["linux"]},
+    {"file":"how-to/system-debugging", "os":["linux"]},
+    {"file":"how-to/tuning-guides", "os":["linux", "windows"]},
+
+    {"file":"rocm-a-z", "os":["linux", "windows"]},
+
+]
+
+exclude_patterns = ['temp']
+
+external_toc_path = "./sphinx/_toc.yml"
+
+docs_core = ROCmDocs("ROCm Documentation")
+docs_core.setup()
+
+external_projects_current_project = "rocm"
+
+for sphinx_var in ROCmDocs.SPHINX_VARS:
+    globals()[sphinx_var] = getattr(docs_core, sphinx_var)
+html_theme_options = {
+    "link_main_doc": False
+}

docs/contribute/building.md

@@ -0,0 +1,148 @@
+# Building documentation
+
+You can build our documentation via GitHub (in a pull request) or locally (using the command line or
+Visual Studio (VS) Code.
+
+## GitHub
+
+If you open a pull request on the `develop` branch of a ROCm repository and scroll to the bottom of
+the page, there is a summary panel. Next to the line
+`docs/readthedocs.com:advanced-micro-devices-demo`, there is a `Details` link. If you click this, it takes
+you to the Read the Docs build for your pull request.
+
+![Screenshot of the GitHub documentation build link](../data/contribute/github-docs-build.png)
+
+If you don't see this line, click `Show all checks` to get an itemized view.
+
+## Command line
+
+You can build our documentation via the command line using Python. We use Python 3.8; other
+versions may not support the build.
+
+Use the Python Virtual Environment (`venv`) and run the following commands from the project root:
+
+```sh
+python3 -mvenv .venv
+
+# Windows
+.venv/Scripts/python -m pip install -r docs/sphinx/requirements.txt
+.venv/Scripts/python -m sphinx -T -E -b html -d _build/doctrees -D language=en docs _build/html
+
+# Linux
+.venv/bin/python     -m pip install -r docs/sphinx/requirements.txt
+.venv/bin/python     -m sphinx -T -E -b html -d _build/doctrees -D language=en docs _build/html
+```
+
+Navigate to `_build/html/index.html` and open this file in a web browser.
+
+## Visual Studio Code
+
+With the help of a few extensions, you can create a productive environment to author and test
+documentation locally using Visual Studio (VS) Code. Follow these steps to configure VS Code:
+
+1. Install the required extensions:
+
+   * Python: `(ms-python.python)`
+   * Live Server: `(ritwickdey.LiveServer)`
+
+2. Add the following entries to `.vscode/settings.json`.
+
+    ```json
+      {
+        "liveServer.settings.root": "/.vscode/build/html",
+        "liveServer.settings.wait": 1000,
+        "python.terminal.activateEnvInCurrentTerminal": true
+      }
+    ```
+
+    * `liveServer.settings.root`: Sets the root of the output website for live previews. Must be changed
+      alongside the `tasks.json` command.
+    * `liveServer.settings.wait`: Tells the live server to wait with the update in order to give Sphinx time to
+      regenerate the site contents and not refresh before the build is complete.
+    * `python.terminal.activateEnvInCurrentTerminal`: Activates the automatic virtual environment, so you
+      can build the site from the integrated terminal.
+
+3. Add the following tasks to `.vscode/tasks.json`.
+
+    ```json
+      {
+        "version": "2.0.0",
+        "tasks": [
+          {
+            "label": "Build Docs",
+            "type": "process",
+            "windows": {
+              "command": "${workspaceFolder}/.venv/Scripts/python.exe"
+            },
+            "command": "${workspaceFolder}/.venv/bin/python3",
+            "args": [
+              "-m",
+              "sphinx",
+              "-j",
+              "auto",
+              "-T",
+              "-b",
+              "html",
+              "-d",
+              "${workspaceFolder}/.vscode/build/doctrees",
+              "-D",
+              "language=en",
+              "${workspaceFolder}/docs",
+              "${workspaceFolder}/.vscode/build/html"
+            ],
+            "problemMatcher": [
+              {
+                "owner": "sphinx",
+                "fileLocation": "absolute",
+                "pattern": {
+                  "regexp": "^(?:.*\\.{3}\\s+)?(\\/[^:]*|[a-zA-Z]:\\\\[^:]*):(\\d+):\\s+(WARNING|ERROR):\\s+(.*)$",
+                  "file": 1,
+                  "line": 2,
+                  "severity": 3,
+                  "message": 4
+                }
+              },
+              {
+              "owner": "sphinx",
+                "fileLocation": "absolute",
+                "pattern": {
+                  "regexp": "^(?:.*\\.{3}\\s+)?(\\/[^:]*|[a-zA-Z]:\\\\[^:]*):{1,2}\\s+(WARNING|ERROR):\\s+(.*)$",
+                  "file": 1,
+                  "severity": 2,
+                  "message": 3
+                }
+              }
+            ],
+            "group": {
+              "kind": "build",
+              "isDefault": true
+            }
+          }
+        ]
+      }
+    ```
+
+    > (Implementation detail: two problem matchers were needed to be defined,
+    > because VS Code doesn't tolerate some problem information being potentially
+    > absent. While a single regex could match all types of errors, if a capture
+    > group remains empty (the line number doesn't show up in all warning/error
+    > messages) but the `pattern` references said empty capture group, VS Code
+    > discards the message completely.)
+
+4. Configure the Python virtual environment (`venv`).
+
+    From the Command Palette, run `Python: Create Environment`. Select `venv` environment and
+    `docs/sphinx/requirements.txt`.
+
+5. Build the docs.
+
+    Launch the default build task using one of the following options:
+
+    * A hotkey (the default is `Ctrl+Shift+B`)
+    * Issuing the `Tasks: Run Build Task` from the Command Palette
+
+6. Open the live preview.
+
+    Navigate to the site output within VS Code: right-click on `.vscode/build/html/index.html` and
+    select `Open with Live Server`. The contents should update on every rebuild without having to
+    refresh the browser.

docs/contribute/feedback.md

@@ -0,0 +1,27 @@
+# How to provide feedback for ROCm documentation
+
+There are four standard ways to provide feedback for this repository.
+
+## Pull request
+
+All contributions to ROCm documentation should arrive via the
+[GitHub Flow](https://docs.github.com/en/get-started/quickstart/github-flow)
+targeting the develop branch of the repository. If you are unable to contribute
+via the GitHub Flow, feel free to email us at [rocm-feedback@amd.com](mailto:rocm-feedback@amd.com?subject=Documentation%20Feedback).
+
+## GitHub discussions
+
+To ask questions or view answers to frequently asked questions, refer to
+[GitHub Discussions](https://github.com/RadeonOpenCompute/ROCm/discussions).
+On GitHub Discussions, in addition to asking and answering questions,
+members can share updates, have open-ended conversations,
+and follow along on via public announcements.
+
+## GitHub issue
+
+Issues on existing or absent docs can be filed as
+[GitHub Issues](https://github.com/RadeonOpenCompute/ROCm/issues).
+
+## Email
+
+Send other feedback or questions to [rocm-feedback@amd.com](mailto:rocm-feedback@amd.com?subject=Documentation%20Feedback).

docs/contribute/toolchain.md

@@ -0,0 +1,71 @@
+# ROCm documentation toolchain
+
+Our documentation relies on several open source toolchains and sites.
+
+## `rocm-docs-core`
+
+[rocm-docs-core](https://github.com/RadeonOpenCompute/rocm-docs-core) is an AMD-maintained
+project that applies customization for our documentation. This
+project is the tool most ROCm repositories use as part of the documentation
+build. It is also available as a [pip package on PyPI](https://pypi.org/project/rocm-docs-core/).
+
+See the user and developer guides for rocm-docs-core at {doc}`rocm-docs-core documentation<rocm-docs-core:index>`.
+
+## Sphinx
+
+[Sphinx](https://www.sphinx-doc.org/en/master/) is a documentation generator
+originally used for Python. It is now widely used in the open source community.
+Originally, Sphinx supported reStructuredText (RST) based documentation, but
+Markdown support is now available.
+ROCm documentation plans to default to Markdown for new projects.
+Existing projects using RST are under no obligation to convert to Markdown. New
+projects that believe Markdown is not suitable should contact the documentation
+team prior to selecting RST.
+
+## Read the Docs
+
+[Read the Docs](https://docs.readthedocs.io/en/stable/) is the service that builds
+and hosts the HTML documentation generated using Sphinx to our end users.
+
+## Doxygen
+
+[Doxygen](https://www.doxygen.nl/) is a documentation generator that extracts
+information from inline code.
+ROCm projects typically use Doxygen for public API documentation unless the
+upstream project uses a different tool.
+
+### Breathe
+
+[Breathe](https://www.breathe-doc.org/) is a Sphinx plugin to integrate Doxygen
+content.
+
+### MyST
+
+[Markedly Structured Text (MyST)](https://myst-tools.org/docs/spec) is an extended
+flavor of Markdown ([CommonMark](https://commonmark.org/)) influenced by reStructuredText (RST) and Sphinx.
+It is integrated into ROCm documentation by the Sphinx extension [`myst-parser`](https://myst-parser.readthedocs.io/en/latest/).
+A cheat sheet that showcases how to use the MyST syntax is available over at
+the [Jupyter reference](https://jupyterbook.org/en/stable/reference/cheatsheet.html).
+
+### Sphinx External ToC
+
+[Sphinx External ToC](https://sphinx-external-toc.readthedocs.io/en/latest/intro.html)
+is a Sphinx extension used for ROCm documentation navigation. This tool generates a navigation menu on the left
+based on a YAML file that specifies the table of contents.
+It was selected due to its flexibility that allows scripts to operate on the
+YAML file. Please transition to this file for the project's navigation. You can
+see the `_toc.yml.in` file in this repository in the `docs/sphinx` folder for an
+example.
+
+### Sphinx-book-theme
+
+[Sphinx-book-theme](https://sphinx-book-theme.readthedocs.io/en/latest/) is a Sphinx theme
+that defines the base appearance for ROCm documentation.
+ROCm documentation applies some customization,
+such as a custom header and footer on top of the Sphinx Book Theme.
+
+### Sphinx design
+
+[Sphinx design](https://sphinx-design.readthedocs.io/en/latest/index.html) is a Sphinx extension that adds design
+functionality.
+ROCm documentation uses Sphinx Design for grids, cards, and synchronized tabs.