Merge remote-tracking branch 'upstream/main' into IFU-2024-02-01

.github/scripts/fbgemm_gpu_docs.bash

@@ -24,17 +24,32 @@ install_docs_tools () {
     echo "################################################################################"
     echo "# Install Documentation Tools"
     echo "#"
-    echo "# [TIMESTAMP] $(date --utc +%FT%T.%3NZ)"
+    echo "# [$(date --utc +%FT%T.%3NZ)] + ${FUNCNAME[0]} ${*}"
     echo "################################################################################"
     echo ""
   fi
 
-  echo "[INSTALL] Installing docs tools ..."
-  (exec_with_retries conda install -n "${env_name}" -c conda-forge -y \
-    doxygen) || return 1
+  test_network_connection || return 1
 
-  # Check binaries are visible in the PAATH
+  # shellcheck disable=SC2155
+  local env_prefix=$(env_name_or_prefix "${env_name}")
+
+  echo "[INSTALL] Installing documentation tools ..."
+
+  # shellcheck disable=SC2086
+  (exec_with_retries 3 conda install ${env_prefix} -c conda-forge -y \
+    doxygen \
+    graphviz \
+    make) || return 1
+
+  # Check binaries are visible in the PATH
+  (test_binpath "${env_name}" dot) || return 1
   (test_binpath "${env_name}" doxygen) || return 1
+  (test_binpath "${env_name}" make) || return 1
+
+  echo "[BUILD] Installing docs-build dependencies ..."
+  # shellcheck disable=SC2086
+  (exec_with_retries 3 conda run ${env_prefix} python -m pip install -r requirements.txt) || return 1
 
   echo "[INSTALL] Successfully installed all the docs tools"
 }
@@ -55,19 +70,39 @@ build_fbgemm_gpu_docs () {
     echo "################################################################################"
     echo "# Build FBGEMM-GPU Documentation"
     echo "#"
-    echo "# [TIMESTAMP] $(date --utc +%FT%T.%3NZ)"
+    echo "# [$(date --utc +%FT%T.%3NZ)] + ${FUNCNAME[0]} ${*}"
     echo "################################################################################"
     echo ""
   fi
 
-  echo "[BUILD] Installing docs-build dependencies ..."
-  (exec_with_retries conda run -n "${env_name}" python -m pip install -r requirements.txt) || return 1
+  # shellcheck disable=SC2155
+  local env_prefix=$(env_name_or_prefix "${env_name}")
+
+  echo "[DOCS] Running the first-pass build (i.e. documentation linting) ..."
+  # shellcheck disable=SC2086
+  print_exec conda env config vars set ${env_prefix} SPHINX_LINT=1
+
+  # Run the first build pass with linting enabled.  The purpose of this pass
+  # is only to perform the lint checks, as the generated output will be broken
+  # when linting is enabled.
+  # shellcheck disable=SC2086
+  if print_exec conda run ${env_prefix} make clean doxygen html; then
+    echo "[DOCS] Docs linting passed"
+  else
+    echo "[DOCS] Docs linting failed; showing build output ..."
+    # Show the buidl logs on error
+    cat build/html/output.txt || true
+    return 1
+  fi
 
-  echo "[BUILD] Running Doxygen build ..."
-  (exec_with_retries conda run -n "${env_name}" doxygen Doxyfile.in) || return 1
+  echo "[DOCS] Running the second-pass documentation build ..."
+  # shellcheck disable=SC2086
+  print_exec conda env config vars unset ${env_prefix} SPHINX_LINT
 
-  echo "[BUILD] Building HTML pages ..."
-  (exec_with_retries conda run -n "${env_name}" make html) || return 1
+  # Run the second build pass with linting disabled.  The generated output will
+  # then be used for publication.
+  # shellcheck disable=SC2086
+  (print_exec conda run ${env_prefix} make clean doxygen html) || return 1
 
-  echo "[INSTALL] FBGEMM-GPU documentation build completed"
+  echo "[DOCS] FBGEMM-GPU documentation build completed"
 }

.github/scripts/fbgemm_gpu_install.bash

@@ -0,0 +1,97 @@
+#!/bin/bash
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+
+# shellcheck disable=SC1091,SC2128
+. "$( dirname -- "$BASH_SOURCE"; )/utils_base.bash"
+# shellcheck disable=SC1091,SC2128
+. "$( dirname -- "$BASH_SOURCE"; )/utils_pip.bash"
+
+################################################################################
+# FBGEMM_GPU Install Functions
+################################################################################
+
+__fbgemm_gpu_post_install_checks () {
+  local env_name="$1"
+  # shellcheck disable=SC2155
+  local env_prefix=$(env_name_or_prefix "${env_name}")
+
+  echo "[INSTALL] Checking imports and symbols ..."
+  (test_python_import_package "${env_name}" fbgemm_gpu) || return 1
+  (test_python_import_package "${env_name}" fbgemm_gpu.split_embedding_codegen_lookup_invokers) || return 1
+  (test_python_import_symbol "${env_name}" fbgemm_gpu __version__) || return 1
+
+  echo "[CHECK] Printing out the FBGEMM-GPU version ..."
+  # shellcheck disable=SC2086,SC2155
+  local installed_fbgemm_gpu_version=$(conda run ${env_prefix} python -c "import fbgemm_gpu; print(fbgemm_gpu.__version__)")
+  echo "[CHECK] The installed version is: ${installed_fbgemm_gpu_version}"
+}
+
+install_fbgemm_gpu_wheel () {
+  local env_name="$1"
+  local wheel_path="$2"
+  if [ "$wheel_path" == "" ]; then
+    echo "Usage: ${FUNCNAME[0]} ENV_NAME WHEEL_NAME"
+    echo "Example(s):"
+    echo "    ${FUNCNAME[0]} build_env fbgemm_gpu.whl     # Install the package (wheel)"
+    return 1
+  else
+    echo "################################################################################"
+    echo "# Install FBGEMM-GPU from Wheel"
+    echo "#"
+    echo "# [$(date --utc +%FT%T.%3NZ)] + ${FUNCNAME[0]} ${*}"
+    echo "################################################################################"
+    echo ""
+  fi
+
+  echo "[INSTALL] Printing out FBGEMM-GPU wheel SHA: ${wheel_path}"
+  print_exec sha1sum "${wheel_path}"
+  print_exec sha256sum "${wheel_path}"
+  print_exec md5sum "${wheel_path}"
+
+  # shellcheck disable=SC2155
+  local env_prefix=$(env_name_or_prefix "${env_name}")
+
+  echo "[INSTALL] Installing FBGEMM-GPU wheel: ${wheel_path} ..."
+  # shellcheck disable=SC2086
+  (exec_with_retries 3 conda run ${env_prefix} python -m pip install "${wheel_path}") || return 1
+
+  __fbgemm_gpu_post_install_checks "${env_name}" || return 1
+
+  echo "[INSTALL] FBGEMM-GPU installation through wheel completed ..."
+}
+
+install_fbgemm_gpu_pip () {
+  local env_name="$1"
+  local fbgemm_gpu_channel_version="$2"
+  local fbgemm_gpu_variant_type_version="$3"
+  if [ "$fbgemm_gpu_variant_type_version" == "" ]; then
+    echo "Usage: ${FUNCNAME[0]} ENV_NAME FBGEMM_GPU_CHANNEL[/VERSION] FBGEMM_GPU_VARIANT_TYPE[/VARIANT_VERSION]"
+    echo "Example(s):"
+    echo "    ${FUNCNAME[0]} build_env 0.5.0 cpu                  # Install the CPU variant, specific version from release channel"
+    echo "    ${FUNCNAME[0]} build_env release cuda 12.1.1        # Install the CUDA variant, latest version from release channel"
+    echo "    ${FUNCNAME[0]} build_env test/0.6.0rc0 cuda 12.1.0  # Install the CUDA 12.1 variant, specific version from test channel"
+    echo "    ${FUNCNAME[0]} build_env nightly rocm 5.3           # Install the ROCM 5.3 variant, latest version from nightly channel"
+    return 1
+  else
+    echo "################################################################################"
+    echo "# Install FBGEMM-GPU Package from PIP"
+    echo "#"
+    echo "# [$(date --utc +%FT%T.%3NZ)] + ${FUNCNAME[0]} ${*}"
+    echo "################################################################################"
+    echo ""
+  fi
+
+  # Install the package from PyTorch PIP (not PyPI)
+  # The package's canonical name is 'fbgemm-gpu' (hyphen, not underscore)
+  install_from_pytorch_pip "${env_name}" fbgemm_gpu "${fbgemm_gpu_channel_version}" "${fbgemm_gpu_variant_type_version}" || return 1
+
+  # Run post-installation checks
+  __fbgemm_gpu_post_install_checks "${env_name}" || return 1
+
+  echo "[INSTALL] Successfully installed FBGEMM-GPU through PyTorch PIP"
+}

.github/scripts/fbgemm_gpu_lint.bash

@@ -24,13 +24,19 @@ install_lint_tools () {
     echo "################################################################################"
     echo "# Install Lint Tools"
     echo "#"
-    echo "# [TIMESTAMP] $(date --utc +%FT%T.%3NZ)"
+    echo "# [$(date --utc +%FT%T.%3NZ)] + ${FUNCNAME[0]} ${*}"
     echo "################################################################################"
     echo ""
   fi
 
+  test_network_connection || return 1
+
+  # shellcheck disable=SC2155
+  local env_prefix=$(env_name_or_prefix "${env_name}")
+
   echo "[INSTALL] Installing lint tools ..."
-  (exec_with_retries conda install -n "${env_name}" -c conda-forge -y \
+  # shellcheck disable=SC2086
+  (exec_with_retries 3 conda install ${env_prefix} -c conda-forge -y \
     click \
     flake8 \
     ufmt) || return 1
@@ -42,7 +48,7 @@ install_lint_tools () {
   # Check Python packages are importable
   local import_tests=( click )
   for p in "${import_tests[@]}"; do
-    (test_python_import "${env_name}" "${p}") || return 1
+    (test_python_import_package "${env_name}" "${p}") || return 1
   done
 
   echo "[INSTALL] Successfully installed all the lint tools"
@@ -63,17 +69,21 @@ lint_fbgemm_gpu_flake8 () {
     echo "################################################################################"
     echo "# Run FBGEMM_GPU Lint: flake8"
     echo "#"
-    echo "# [TIMESTAMP] $(date --utc +%FT%T.%3NZ)"
+    echo "# [$(date --utc +%FT%T.%3NZ)] + ${FUNCNAME[0]} ${*}"
     echo "################################################################################"
     echo ""
   fi
 
   echo "::add-matcher::fbgemm_gpu/test/lint/flake8_problem_matcher.json"
 
+  # shellcheck disable=SC2155
+  local env_prefix=$(env_name_or_prefix "${env_name}")
+
   # E501 = line too long
   # W503 = line break before binary operator (deprecated)
   # E203 = whitespace before ":"
-  (print_exec conda run -n "${env_name}" flake8 --ignore=E501,W503,E203 .) || return 1
+  # shellcheck disable=SC2086
+  (print_exec conda run ${env_prefix} flake8 --ignore=E501,W503,E203 .) || return 1
 
   echo "[TEST] Finished running flake8 lint checks"
 }
@@ -89,7 +99,7 @@ lint_fbgemm_gpu_ufmt () {
     echo "################################################################################"
     echo "# Run FBGEMM_GPU Lint: ufmt"
     echo "#"
-    echo "# [TIMESTAMP] $(date --utc +%FT%T.%3NZ)"
+    echo "# [$(date --utc +%FT%T.%3NZ)] + ${FUNCNAME[0]} ${*}"
     echo "################################################################################"
     echo ""
   fi
@@ -100,8 +110,12 @@ lint_fbgemm_gpu_ufmt () {
     fbgemm_gpu/bench
   )
 
+  # shellcheck disable=SC2155
+  local env_prefix=$(env_name_or_prefix "${env_name}")
+
   for p in "${lint_paths[@]}"; do
-    (print_exec conda run -n "${env_name}" ufmt diff "${p}") || return 1
+    # shellcheck disable=SC2086
+    (print_exec conda run ${env_prefix} ufmt diff "${p}") || return 1
   done
 
   echo "[TEST] Finished running ufmt lint checks"
@@ -118,7 +132,7 @@ lint_fbgemm_gpu_copyright () {
     echo "################################################################################"
     echo "# Run FBGEMM_GPU Lint: Meta Copyright Headers"
     echo "#"
-    echo "# [TIMESTAMP] $(date --utc +%FT%T.%3NZ)"
+    echo "# [$(date --utc +%FT%T.%3NZ)] + ${FUNCNAME[0]} ${*}"
     echo "################################################################################"
     echo ""
   fi
@@ -129,8 +143,12 @@ lint_fbgemm_gpu_copyright () {
     fbgemm_gpu/bench
   )
 
+  # shellcheck disable=SC2155
+  local env_prefix=$(env_name_or_prefix "${env_name}")
+
   for p in "${lint_paths[@]}"; do
-    (print_exec conda run -n "${env_name}" python fbgemm_gpu/test/lint/check_meta_header.py --path="${p}" --fixit=False) || return 1
+    # shellcheck disable=SC2086
+    (print_exec conda run ${env_prefix} python fbgemm_gpu/test/lint/check_meta_header.py --path="${p}" --fixit=False) || return 1
   done
 
   echo "[TEST] Finished running Meta Copyright Header checks"