#!/bin/sh # Manage a stack installation or an lsstsw clone # Installs conda if an existing one is not used # Creates lsst-scipipe-{version} environment if it doesn't exist # Records appropriate EUPS_PKGROOT in the environment's eups stack # Default values dryrun= rubinenv_version=latest update=false eups_root=https://eups.lsst.cloud/stack use_tarball=true use_source=true use_rsp=false python_pin= # channels is in _reverse_ priority order channels= print_error () { >&2 echo "$@" } fail () { print_error "$@" exit 1 } # Parse arguments usage () { cat <" suffix, and binary "tarball" eups packages are disabled because published tarballs are built only for the officially supported python. Not available with -X or a hash ref, whose exact environments already pin python. -p CONDA_PATH -- Specify the path to the conda installation. If a conda installation already exists there, it will be used. If it does not exist, it will be created. If a conda is activated, it will be used, ignoring this option. -P -- DO NOT use an existing activated conda; always install a new one. -C CHANNEL -- Use the given conda channel before the conda-forge channel. May be repeated; first has highest priority. -E EUPS_URL -- Select a different EUPS distribution server root URL (default=https://eups.lsst.cloud/stack). -B -- DO NOT use binary "tarball" eups packages. -S -- DO NOT use source eups packages. -b -- Due to backward compatibility, it will print help instead -c -- Due to backward compatibility, it will print help instead -t -- Due to backward compatibility, it will print help instead -h -- Display this help message. EOF exit 1 } while getopts nBSRT:X:v:e:udp:PC:E:y:bcth opt; do case "$opt" in n) dryrun="echo \$ " ;; B) use_tarball=false ;; S) use_source=false ;; R) use_rsp=true ;; T) if [ -n "$eups_tag" ] || [ "$rubinenv_version" != "latest" ]; then print_error "Only one of -T -X -v allowed" && usage fi eups_tag="$OPTARG" ;; X) if [ -n "$eups_tag" ] || [ "$rubinenv_version" != "latest" ]; then print_error "Only one of -T -X -v allowed" && usage fi eups_tag="$OPTARG"; exact=true ;; v) if [ -n "$eups_tag" ] || [ "$rubinenv_version" != "latest" ]; then print_error "Only one of -T -X -v allowed" && usage fi rubinenv_version="$OPTARG" ;; e) rubinenv_name="$OPTARG" ;; u) update="true" ;; d) add_developer="true" ;; p) conda_path="$OPTARG" ;; P) # Force conda installation by removing knowledge of current conda unset CONDA_EXE ;; C) channels="$OPTARG $channels" ;; E) eups_root="$OPTARG" ;; y) python_pin="$OPTARG" ;; b | c | t) usage ;; h) usage ;; *) usage ;; esac done # Checks for dangling arguments shift $((OPTIND - 1)) if [ $# -gt 0 ]; then print_error "Unexpected argument(s): $*" usage fi channels="conda-forge $channels" # Configure cwd=$(pwd) platform="$(uname -s)" case "$platform" in Linux | Darwin) ;; *) fail "Unknown platform: $platform" ;; esac arch="$(uname -m)" case "$arch" in x86_64 | aarch64 | arm64) ;; *) fail "Unknown architecture: $arch" ;; esac run_curl () { _c=curl # Silent, show errors _opt="-sS" # Use /usr/bin/curl if available for consistency [ -x /usr/bin/curl ] && _c=/usr/bin/curl # If running interactively, output progress [ -t 1 ] && _opt="-#" # Fail on error, follow redirects, and retry on transient errors $_c -fL --retry 3 $_opt "$@" } # Activate conda, installing if necessary if [ -n "$(command -v conda)" ] && [ -n "$CONDA_EXE" ]; then conda_path=$(expr "$CONDA_EXE" : '\(.*\)/bin/conda') else conda_path="${conda_path:-$cwd/conda}" fi if [ -x "$conda_path/bin/conda" ]; then echo "Using existing conda at $conda_path" else [ -e "$conda_path" ] \ && fail "$conda_path exists but does not appear to contain conda" echo "Installing Miniforge3 conda at $conda_path" url="https://github.com/conda-forge/miniforge/releases/latest/download" url="$url/Miniforge3-$platform-${arch}.sh" $dryrun run_curl -O "$url" \ || fail "Unable to get Miniforge3 script for $platform $arch from $url" $dryrun bash "Miniforge3-$platform-${arch}.sh" -b -p "$conda_path" \ || fail "Unable to install Miniforge3" $dryrun rm -f "Miniforge3-$platform-${arch}.sh" fi if [ -n "$dryrun" ]; then $dryrun setup conda shell functions else if ! type conda 2> /dev/null | grep "function" > /dev/null 2>&1; then if [ -z "$SHELL" ]; then if [ -n "$BASH_VERSION" ]; then SHELL=bash elif [ -n "$ZSH_NAME" ]; then SHELL=zsh else # fall back to lowest common denominator shell SHELL=dash fi fi __conda_setup=$("$conda_path/bin/conda" "shell.$(basename "$SHELL")" \ hook 2>/dev/null) || fail "Unknown shell" eval "$__conda_setup" || fail "Unable to start conda" fi for profile in "$conda_path/etc/profile.d"/*.sh; do # shellcheck source=/dev/null [ "$(basename "$profile")" = "mamba.sh" ] && continue # shellcheck source=/dev/null . "$profile" done fi command -v conda >/dev/null run_conda () { subcmd=$1; shift set -- --strict-channel-priority "$@" for c in $channels; do set -- -c "$c" "$@" done $dryrun conda "$subcmd" "$@" } # Determine rubin-env version src_root="${eups_root}/src" if [ -n "$eups_tag" ]; then # TODO Works for w_2020_15 or later # First conda-system hash is 46b24e8 w_2020_20; before uses devtools rubinenv_version=$(run_curl "$src_root/tags/${eups_tag}.list" \ | grep '^#CONDA_ENV=' | cut -d= -f2) if [ -z "$rubinenv_version" ]; then # check if tag was moved to old_tags echo "Checking old tags" rubinenv_version=$(run_curl "$src_root/tags/old_tags/${eups_tag}.list" \ | grep '^#CONDA_ENV=' | cut -d= -f2) [ -z "$rubinenv_version" ] && fail "Unable to determine conda env" fi env_hash=$(expr "$rubinenv_version" : 'https:.*@\(.*\)') [ -n "$env_hash" ] && rubinenv_version=$env_hash elif [ "$rubinenv_version" = latest ]; then rubinenv_version=$(run_curl \ "https://api.anaconda.org/package/conda-forge/rubin-env" \ | grep \"latest_version\" | cut -d\" -f4) elif expr "$rubinenv_version" : '[0-9a-f]*$' > /dev/null; then env_hash="$rubinenv_version" eups_tag="$rubinenv_version" fi echo "Selected rubin-env=$rubinenv_version" # Hash refs resolve to a scipipe_conda_env lock file, which predates rubin-env # and so has no RSP counterpart to install. if [ "$use_rsp" = true ] && [ -n "$env_hash" ]; then fail "-R is not available for pre-rubin-env environments ($rubinenv_version)" fi # Exact environments and pre-rubin-env lock files are installed from a fully # resolved package list, which already pins python, so an override there would # be silently ignored. if [ -n "$python_pin" ] && { [ "$exact" = true ] || [ -n "$env_hash" ]; }; then fail "-y is not available for exact environments (-X) or pre-rubin-env refs ($rubinenv_version)" fi # Published binary eups packages are built only for the officially supported # python. Their extension modules are ABI-untagged and link no libpython, so a # mismatched interpreter loads them and then fails unpredictably rather than # cleanly -- fall back to source packages instead. if [ -n "$python_pin" ]; then echo "Note: -y overrides the python version rubin-env would select." echo "Binary tarball eups packages are disabled; building from source." use_tarball=false fi # Determine EUPS binary root get_binary_root () { case "$arch" in x86_64) if [ "$platform" = "Linux" ]; then eups_platform="redhat/el7/conda-system/${1}-$rubinenv_version" else eups_platform="osx/10.9/conda-system/${1}-$rubinenv_version" fi echo "$eups_root/${eups_platform}" ;; arm64) eups_platform="osx/14-arm/conda-system/${1}-$rubinenv_version" echo "$eups_root/${eups_platform}" ;; aarch64) eups_platform="redhat/el8-arm/conda-system/${1}-$rubinenv_version" echo "$eups_root/${eups_platform}" ;; *) echo "" ;; esac } if [ -n "$env_hash" ]; then binary_root=$(get_binary_root miniconda3-py37-4.7.12) if [ -n "$binary_root" ] && run_curl "$binary_root" > /dev/null 2>&1; then next_root=$(get_binary_root miniconda3-py37_4.8.2) if run_curl "$next_root" > /dev/null 2>&1; then binary_root="$next_root|$binary_root" fi else binary_root=$(get_binary_root miniconda3-py37_4.8.2) fi else binary_root=$(get_binary_root miniconda3-py38_4.9.2) fi # Install rubin-env environment if necessary rubinenv_name=${rubinenv_name:-lsst-scipipe-$rubinenv_version} [ "$exact" = true ] && [ -z "$env_hash" ] && rubinenv_name="${rubinenv_name}-exact" rubinenv_pkg=rubin-env if [ "$use_rsp" = true ]; then rubinenv_pkg=rubin-env-rsp rubinenv_name="${rubinenv_name}-rsp" fi if [ -n "$python_pin" ]; then rubinenv_name="${rubinenv_name}-py$(echo "$python_pin" | tr -d .)" fi if [ -d "$conda_path/envs/$rubinenv_name" ]; then echo "Using existing environment $rubinenv_name" if [ "$update" = true ] && [ "$exact" != true ] && [ -z "$env_hash" ]; then echo "Updating $rubinenv_pkg=$rubinenv_version" run_conda update -y -n "$rubinenv_name" \ "$rubinenv_pkg=$rubinenv_version" ${python_pin:+python=$python_pin} fi elif [ "$exact" = true ] || [ -n "$env_hash" ]; then env_file="${eups_tag}.env" [ "$use_rsp" = true ] && env_file="${eups_tag}_rsp.env" if [ -n "$env_hash" ]; then if [ "$platform" = Linux ]; then env_platform=linux-64 else env_platform=osx-64 fi env_base="https://raw.githubusercontent.com/lsst/scipipe_conda_env" url="${env_base}/${env_hash}/etc/conda-${env_platform}.lock" elif [ -n "$binary_root" ]; then url="$binary_root/env/$env_file" else fail "No exact environment for source-only platform/architecture" fi if ! $dryrun run_curl -o "$env_file" "$url"; then print_error "Unable to download environment spec for tag $eups_tag from $url" [ "$use_rsp" = true ] && print_error \ "Tags built before RSP environments were published have no $env_file" exit 1 fi run_conda create -y -n "$rubinenv_name" --file "$env_file" \ || fail "Unable to create conda environment ${rubinenv_name}" $dryrun rm -f "$env_file" if [ -n "$env_hash" ]; then # scipipe_conda_env did not have eups run_conda install -y -n "$rubinenv_name" --no-update-deps eups \ || fail "Unable to install eups into environment ${rubinenv_name}" fi else run_conda create -y -n "$rubinenv_name" \ "$rubinenv_pkg=$rubinenv_version" ${python_pin:+python=$python_pin} \ || fail "Unable to create conda environment ${rubinenv_name}" fi if [ "$add_developer" = true ]; then # Note: doesn't actually fail if rubin-env-developer can't be installed, # but instead gives a message that an appropriate rubin-env can't be found. run_conda install -y -n "$rubinenv_name" --no-update-deps rubin-env-developer fi # Activate environment to set EUPS_PATH $dryrun conda activate "$rubinenv_name" \ || fail "Unable to activate environment ${rubinenv_name}" # Set EUPS_PKGROOT if [ "$use_source" = true ]; then if [ "$use_tarball" = true ] && [ -n "$binary_root" ]; then EUPS_PKGROOT="$binary_root|$src_root" else EUPS_PKGROOT="$src_root" fi else if [ "$use_tarball" = true ]; then EUPS_PKGROOT="$binary_root" else EUPS_PKGROOT="" fi fi if [ -n "$dryrun" ]; then $dryrun echo "$EUPS_PKGROOT" \> "\$EUPS_PATH/pkgroot" else [ -f "$EUPS_PATH/pkgroot" ] && echo "$EUPS_PATH/pkgroot exists; overwriting" echo "$EUPS_PKGROOT" > "$EUPS_PATH/pkgroot" || fail "Unable to write pkgroot file" fi # Copy postinstall callback_folder="$EUPS_PATH/site" hook="distribInstallPostHook.py" hook_url="https://raw.githubusercontent.com/RobertLuptonTheGood/eups/refs/heads/master/callbacks/$hook" if [ -n "$dryrun" ]; then $dryrun run_curl $hook_url \> "\$EUPS_PATH/site/$hook" elif [ -f "$callback_folder/$hook" ]; then echo "Callback already installed" else run_curl -o "$callback_folder/$hook" "$hook_url" fi # Create load scripts source_cmd=source if [ -f loadLSST.sh ]; then echo "loadLSST.sh exists; not overwriting" source_cmd="LSST_CONDA_ENV_NAME=${rubinenv_name} source" elif [ -n "$dryrun" ]; then $dryrun cat \> loadLSST.sh else cat > loadLSST.sh </dev/null)" \\ || { echo "Unknown shell"; exit 1; } eval "\$__conda_setup" || { echo "Unable to start conda"; exit 1; } for __profile in "$conda_path/etc/profile.d"/*.sh; do [ "\$(basename "\$__profile")" = "mamba.sh" ] && continue . "\$__profile" done unset __profile __conda_setup export LSST_CONDA_ENV_NAME=\${LSST_CONDA_ENV_NAME:-$rubinenv_name} conda activate "\$LSST_CONDA_ENV_NAME" && export EUPS_PKGROOT=\$(cat \$EUPS_PATH/pkgroot) EOF fi # For now, all of these are identical; csh is unsupported for ext in ash bash zsh; do if [ ! -f "loadLSST.$ext" ]; then $dryrun ln loadLSST.sh "loadLSST.$ext" fi done cat <