Allow installation of dxtbx with read-only conda_base/ (#413)

On some HPC systems, the root environment is read-only. This allows
installation and usage within the libtbx environment without having to
create extra layers of virtual environments.

It does this by falling back to inserting the editable source PYTHONPATH
into the tbx dispatcher system's list of extra python paths, and
requesting tbx dispatcher generation directly.

libtbx_refresh.py

@@ -1,7 +1,12 @@
+import inspect
+import os
+import random
+import site
 import subprocess
 import sys
+from pathlib import Path
 
-import libtbx.load_env
+import libtbx
 import libtbx.pkg_utils
 
 try:
@@ -40,8 +45,127 @@ def _install_dxtbx_setup():
     )
 
 
+def _install_dxtbx_setup_readonly_fallback():
+    """
+    Partially install package in the libtbx build folder.
+
+    This is a less complete installation - base python console_scripts
+    entrypoints will not be installed, but the basic package metadata
+    and other entrypoints will be enumerable through dispatcher black magic
+    """
+    dxtbx_root_path = Path(libtbx.env.dist_path("dxtbx"))
+    dxtbx_import_path = dxtbx_root_path / "src"
+
+    # Install this into a build/dxtbx subfolder
+    build_path = Path(abs(libtbx.env.build_path))
+    dxtbx_build_path = build_path / "dxtbx"
+    subprocess.run(
+        [
+            sys.executable,
+            "-m",
+            "pip",
+            "install",
+            "--prefix",
+            dxtbx_build_path,
+            "--no-build-isolation",
+            "--no-deps",
+            "-e",
+            dxtbx_root_path,
+        ],
+        check=True,
+    )
+
+    # Get the actual environment being configured (NOT libtbx.env)
+    env = _get_real_env_hack_hack_hack()
+
+    # Update the libtbx environment pythonpaths to point to the source
+    # location which now has an .egg-info folder; this will mean that
+    # the PYTHONPATH is written into the libtbx dispatchers
+    rel_path = libtbx.env.as_relocatable_path(str(dxtbx_import_path))
+    if rel_path not in env.pythonpath:
+        env.pythonpath.insert(0, rel_path)
+
+    # Update the sys.path so that we can find the .egg-info in this process
+    # if we do a full reconstruction of the working set
+    if str(dxtbx_import_path) not in sys.path:
+        sys.path.insert(0, str(dxtbx_import_path))
+
+    # ...and add to the existing pkg_resources working_set
+    if pkg_resources:
+        pkg_resources.working_set.add_entry(dxtbx_import_path)
+
+    # Also, since we can't re-export dispatchers, add the src/ folder
+    # as an extra command_line_locations.
+    #
+    # This is already generated by this point, but will get picked up
+    # on the second libtbx.refresh.
+    module = env.module_dict["dxtbx"]
+    if "src/dxtbx" not in module.extra_command_line_locations:
+        module.extra_command_line_locations.append("src/dxtbx")
+
+
+def _test_writable_dir(path: Path) -> bool:
+    """Test a path is writable. Based on pip's _test_writable_dir_win."""
+    # os.access doesn't work on windows
+    # os.access won't always work with network filesystems
+    # pip doesn't use tempfile on windows because https://bugs.python.org/issue22107
+    basename = "test_site_packages_writable_dxtbx_"
+    alphabet = "abcdefghijklmnopqrstuvwxyz0123456789"
+    for _ in range(10):
+        name = basename + "".join(random.choice(alphabet) for _ in range(6))
+        file = path / name
+        try:
+            fd = os.open(file, os.O_RDWR | os.O_CREAT | os.O_EXCL)
+        except FileExistsError:
+            pass
+        except PermissionError:
+            return False
+        else:
+            os.close(fd)
+            os.unlink(file)
+            return True
+
+
+def _get_real_env_hack_hack_hack():
+    """
+    Get the real, currently-being-configured libtbx.env environment.
+
+    This is not libtbx.env, because although libtbx.env_config.environment.cold_start
+    does:
+        self.pickle()
+        libtbx.env = self
+    the first time there is an "import libtbx.load_env" this environment
+    gets replaced by unpickling the freshly-written libtbx_env file onto
+    libtbx.env, thereby making the environment accessed via libtbx.env
+    *not* the actual one that is currently being constructed.
+
+    So, the only way to get this environment being configured in order
+    to - like - configure it, is to walk the stack trace and extract the
+    self object from environment.refresh directly.
+    """
+    for frame in inspect.stack():
+        if (
+            frame.filename.endswith("env_config.py")
+            and frame.function == "refresh"
+            and "self" in frame.frame.f_locals
+        ):
+            return frame.frame.f_locals["self"]
+
+    raise RuntimeError("Could not determine real libtbx.env_config.environment object")
+
+
+# Detect case where base python environment is read-only
+# e.g. on an LCLS session on a custom cctbx installation where the
+# source is editable but the conda_base is read-only
+#
+# We need to check before trying to install as pip does os.access-based
+# checks then installs with --user if it fails. We don't want that.
+if _test_writable_dir(Path(site.getsitepackages()[0])):
+    _install_dxtbx_setup()
+else:
+    print("Python site directory not writable - falling back to tbx install")
+    _install_dxtbx_setup_readonly_fallback()
+
 # Retain until after DIALS 3.6 is released to unregister the previous dispatcher handlers
 if not pkg_resources or any(x.key == "libtbx.dxtbx" for x in pkg_resources.working_set):
     libtbx.pkg_utils.define_entry_points({})
-
-_install_dxtbx_setup()

newsfragments/413.bugfix

@@ -0,0 +1 @@
+Handle installing dxtbx as a "real" package when the ``conda_base/`` is read-only