Update pre-commits - Take 2 (#468)

Various pre-commit updates:
* fast-fail on syntax errors
* control future imports
* update black to stable version
* add clang-format for C/C++
* run flake8-comprehensions checks

Co-authored-by: Nicholas Devenish <ndevenish@gmail.com>

.azure-pipelines/bootstrap.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import argparse
 import multiprocessing.pool
 import os

.azure-pipelines/flake8-validation.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import os
 import subprocess
 

.azure-pipelines/syntax-validation.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import ast
 import os
 import sys

.pre-commit-config.yaml

@@ -1,35 +1,54 @@
 repos:
+# Syntax validation and some basic sanity checks
+- repo: /~https://github.com/pre-commit/pre-commit-hooks
+  rev: v4.1.0
+  hooks:
+  - id: check-merge-conflict
+  - id: check-ast
+    fail_fast: True
+  - id: check-json
+  - id: check-added-large-files
+    args: ['--maxkb=200']
+  - id: check-yaml
+    args: ['--allow-multiple-documents']
+  - id: no-commit-to-branch
+    name: "Don't commit to 'main' directly"
 
 # Automatically sort imports
-- repo: /~https://github.com/PyCQA/isort.git
-  rev: 5.9.2
+- repo: /~https://github.com/PyCQA/isort
+  rev: 5.10.1
   hooks:
   - id: isort
+    args: [
+           '-a', 'from __future__ import annotations',        # 3.7-3.11
+           '--rm', 'from __future__ import absolute_import',  # -3.0
+           '--rm', 'from __future__ import division',         # -3.0
+           '--rm', 'from __future__ import generator_stop',   # -3.7
+           '--rm', 'from __future__ import generators',       # -2.3
+           '--rm', 'from __future__ import nested_scopes',    # -2.2
+           '--rm', 'from __future__ import print_function',   # -3.0
+           '--rm', 'from __future__ import unicode_literals', # -3.0
+           '--rm', 'from __future__ import with_statement',   # -2.6
+          ]
 
 # Automatic source code formatting
 - repo: /~https://github.com/psf/black
-  rev: 21.6b0
+  rev: 22.1.0
   hooks:
   - id: black
     args: [--safe, --quiet]
     files: \.pyi?$|SConscript$|^libtbx_config$
     types: [file]
 
+- repo: /~https://github.com/pre-commit/mirrors-clang-format
+  rev: v13.0.1
+  hooks:
+  - id: clang-format
+    files: \.c(c|pp|xx)?$|\.h(pp)?$
+
 # Linting
 - repo: /~https://github.com/PyCQA/flake8
-  rev: 3.9.2
+  rev: 4.0.1
   hooks:
   - id: flake8
-
-# Syntax validation and some basic sanity checks
-- repo: /~https://github.com/pre-commit/pre-commit-hooks
-  rev: v4.0.1
-  hooks:
-  - id: check-ast
-  - id: check-yaml
-    args: ['--allow-multiple-documents']
-  - id: check-merge-conflict
-  - id: check-added-large-files
-    args: ['--maxkb=200']
-  - id: no-commit-to-branch
-    name: "Don't commit to main directly"
+    additional_dependencies: ['flake8-comprehensions==3.8.0']

build.py

@@ -6,6 +6,8 @@
 offloads the unresolved build phases.
 """
 
+from __future__ import annotations
+
 import ast
 import itertools
 import re

cmake/read_env.py

@@ -8,6 +8,8 @@
 path anyway).
 """
 
+from __future__ import annotations
+
 import json
 import os
 import pickle

conftest.py

@@ -3,6 +3,8 @@
 # write and run pytest tests, and an overview of the available features.
 #
 
+from __future__ import annotations
+
 import os
 import socket
 

libtbx_refresh.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import inspect
 import os
 import random

newsfragments/468.misc

@@ -0,0 +1 @@
+Update pre-commit environments

setup.cfg

@@ -25,3 +25,5 @@ select =
     E401,E711,E712,E713,E714,E721,E722,E901,
     F401,F402,F403,F405,F541,F631,F632,F633,F811,F812,F821,F822,F841,F901,
     W191,W291,W292,W293,W602,W603,W604,W605,W606,
+    # flake8-comprehensions, /~https://github.com/adamchainz/flake8-comprehensions
+    C4,

setup.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 from pathlib import Path
 
 import setuptools

src/dxtbx/command_line/debug_memory.py

@@ -17,7 +17,7 @@ def run(args=None):
     parser.add_argument("image_frame", help="An image to reload repeatedly")
     options = parser.parse_args(args)
 
-    powers = [2 ** n for n in range(20)]
+    powers = [2**n for n in range(20)]
 
     for j in range(powers[-1] + 1):
         dxtbx.load(options.image_frame)

src/dxtbx/command_line/radial_average.py

@@ -346,7 +346,7 @@ def load_func(x):
                     results = results.select(xvals <= params.plot_x_max)
                     xvals = xvals.select(xvals <= params.plot_x_max)
                 if params.x_axis == "resolution":
-                    xvals = 1 / (xvals ** 2)
+                    xvals = 1 / (xvals**2)
                 if params.normalize:
                     plt.plot(
                         xvals.as_numpy_array(),
@@ -400,7 +400,7 @@ def resolution(x, pos):
                     / (2 * flex.asin((math.pi / 180) * tt.select(nonzero) / 2)),
                 )
                 vals = resolution
-                vals = 1 / (vals ** 2)
+                vals = 1 / (vals**2)
             elif params.x_axis == "two_theta":
                 vals = tt
 

src/dxtbx/datablock.py

@@ -162,79 +162,42 @@ def abspath_or_none(filename):
             if isinstance(iset, dxtbx.imageset.ImageSequence):
                 if iset.reader().is_single_file_reader():
                     result["imageset"].append(
-                        dict(
-                            [
-                                ("__id__", "ImageSequence"),
-                                (
-                                    "master",
-                                    os.path.abspath(iset.reader().master_path()),
-                                ),
-                                (
-                                    "mask",
-                                    abspath_or_none(iset.external_lookup.mask.filename),
-                                ),
-                                (
-                                    "gain",
-                                    abspath_or_none(iset.external_lookup.gain.filename),
-                                ),
-                                (
-                                    "pedestal",
-                                    abspath_or_none(
-                                        iset.external_lookup.pedestal.filename
-                                    ),
-                                ),
-                                (
-                                    "dx",
-                                    abspath_or_none(iset.external_lookup.dx.filename),
-                                ),
-                                (
-                                    "dy",
-                                    abspath_or_none(iset.external_lookup.dy.filename),
-                                ),
-                                ("beam", b.index(iset.get_beam())),
-                                ("detector", d.index(iset.get_detector())),
-                                ("goniometer", g.index(iset.get_goniometer())),
-                                ("scan", s.index(iset.get_scan())),
-                                ("images", list(iset.indices())),
-                                ("params", iset.params()),
-                            ]
-                        )
+                        {
+                            "__id__": "ImageSequence",
+                            "master": os.path.abspath(iset.reader().master_path()),
+                            "mask": abspath_or_none(iset.external_lookup.mask.filename),
+                            "gain": abspath_or_none(iset.external_lookup.gain.filename),
+                            "pedestal": abspath_or_none(
+                                iset.external_lookup.pedestal.filename
+                            ),
+                            "dx": abspath_or_none(iset.external_lookup.dx.filename),
+                            "dy": abspath_or_none(iset.external_lookup.dy.filename),
+                            "beam": b.index(iset.get_beam()),
+                            "detector": d.index(iset.get_detector()),
+                            "goniometer": g.index(iset.get_goniometer()),
+                            "scan": s.index(iset.get_scan()),
+                            "images": list(iset.indices()),
+                            "params": iset.params(),
+                        }
                     )
                 else:
                     result["imageset"].append(
-                        dict(
-                            [
-                                ("__id__", "ImageSequence"),
-                                ("template", os.path.abspath(iset.get_template())),
-                                (
-                                    "mask",
-                                    abspath_or_none(iset.external_lookup.mask.filename),
-                                ),
-                                (
-                                    "gain",
-                                    abspath_or_none(iset.external_lookup.gain.filename),
-                                ),
-                                (
-                                    "pedestal",
-                                    abspath_or_none(
-                                        iset.external_lookup.pedestal.filename
-                                    ),
-                                ),
-                                (
-                                    "dx",
-                                    abspath_or_none(iset.external_lookup.dx.filename),
-                                ),
-                                (
-                                    "dy",
-                                    abspath_or_none(iset.external_lookup.dy.filename),
-                                ),
-                                ("beam", b.index(iset.get_beam())),
-                                ("detector", d.index(iset.get_detector())),
-                                ("goniometer", g.index(iset.get_goniometer())),
-                                ("scan", s.index(iset.get_scan())),
-                                ("params", iset.params()),
-                            ]
-                        )
+                        {
+                            "__id__": "ImageSequence",
+                            "template": os.path.abspath(iset.get_template()),
+                            "mask": abspath_or_none(iset.external_lookup.mask.filename),
+                            "gain": abspath_or_none(iset.external_lookup.gain.filename),
+                            "pedestal": abspath_or_none(
+                                iset.external_lookup.pedestal.filename
+                            ),
+                            "dx": abspath_or_none(iset.external_lookup.dx.filename),
+                            "dy": abspath_or_none(iset.external_lookup.dy.filename),
+                            "beam": b.index(iset.get_beam()),
+                            "detector": d.index(iset.get_detector()),
+                            "goniometer": g.index(iset.get_goniometer()),
+                            "scan": s.index(iset.get_scan()),
+                            "params": iset.params(),
+                        }
                     )
             else:
                 imageset = {}

src/dxtbx/format/FormatCBFFull.py

@@ -25,7 +25,6 @@
     def cbf_read_buffer(handle, buffer, flags):
         return handle.read_buffer(buffer, flags)
 
-
 else:
     from dxtbx.format.image import cbf_read_buffer
 

src/dxtbx/format/FormatEigerStream.py

@@ -155,7 +155,7 @@ def get_raw_data(self, index):
         print("Get raw data")
 
         if info["type"] == "uint16":
-            bad_sel = data == 2 ** 16 - 1
+            bad_sel = data == 2**16 - 1
             data[bad_sel] = -1
 
         return flex.int(data)

src/dxtbx/format/FormatMRC.py

@@ -190,10 +190,10 @@ def _detector(self):
         camera = self._header_dictionary.get("camera", b"").lower()
         if b"ceta" in camera:
             gain = 26.0
-            saturation = 8000 * binning ** 2
+            saturation = 8000 * binning**2
         elif b"falcon" in camera:
             gain = 1.0
-            saturation = 8000 * binning ** 2
+            saturation = 8000 * binning**2
         else:
             gain = 1.0
             saturation = 1e6

src/dxtbx/format/FormatNXmx.py

@@ -77,9 +77,9 @@ def get_raw_data(self, index):
             # if 32 bit then it is a signed int, I think if 8, 16 then it is
             # unsigned with the highest two values assigned as masking values
             if self._bit_depth_readout == 32:
-                top = 2 ** 31
+                top = 2**31
             else:
-                top = 2 ** self._bit_depth_readout
+                top = 2**self._bit_depth_readout
             for data in raw_data:
                 d1d = data.as_1d()
                 d1d.set_selected(d1d == top - 1, -1)

src/dxtbx/format/FormatNexusEigerDLS.py

@@ -99,9 +99,9 @@ def get_raw_data(self, index):
             # if 32 bit then it is a signed int, I think if 8, 16 then it is
             # unsigned with the highest two values assigned as masking values
             if self._bit_depth_image == 32:
-                top = 2 ** 31
+                top = 2**31
             else:
-                top = 2 ** self._bit_depth_image
+                top = 2**self._bit_depth_image
             d1d = data.as_1d()
             d1d.set_selected(d1d == top - 1, -1)
             d1d.set_selected(d1d == top - 2, -2)

src/dxtbx/format/FormatSER.py

@@ -281,7 +281,7 @@ def _detector(self):
             pixel_size = 0.028, 0.028
             binning = 2
         binning = self._header_dictionary.get("Binning", binning)
-        saturation = 8000 * binning ** 2
+        saturation = 8000 * binning**2
         trusted_range = (-1000, saturation)
 
         distance = float(self._header_dictionary.get("Camera length [m]", 2)) * 1000

src/dxtbx/format/nexus.py

@@ -592,7 +592,7 @@ def load_model(self, index=None):
         self.index = index
 
         def get_wavelength(wavelength):
-            if wavelength.shape in (tuple(), (1,)):
+            if wavelength.shape in ((), (1,)):
                 wavelength_value = wavelength[()]
             else:
                 wavelength_value = wavelength[index]

src/dxtbx/masking/__init__.py

@@ -122,10 +122,10 @@ def dls_i23_kappa(goniometer):
 
         alpha = flex.double_range(180, 370, step=10) * math.pi / 180
         r = flex.double(alpha.size(), 33)
-        x = flex.sqrt(flex.pow2(r * flex.sin(alpha)) + 89.02 ** 2) * flex.cos(
+        x = flex.sqrt(flex.pow2(r * flex.sin(alpha)) + 89.02**2) * flex.cos(
             (50 * math.pi / 180) - flex.atan(r / 89.02 * flex.sin(alpha))
         )
-        y = flex.sqrt(flex.pow2(r * flex.sin(alpha)) + 89.02 ** 2) * flex.sin(
+        y = flex.sqrt(flex.pow2(r * flex.sin(alpha)) + 89.02**2) * flex.sin(
             (50 * math.pi / 180) - flex.atan(r / 89.02 * flex.sin(alpha))
         )
         z = -r * flex.cos(alpha)

src/dxtbx/model/experiment_list.py

@@ -1087,7 +1087,7 @@ def _groupby_template_is_none(
     for _, group in itertools.groupby(
         enumerate(records), key=lambda x: -1 if x[1].template is None else x[0]
     ):
-        yield list(x[1] for x in group)
+        yield [x[1] for x in group]
 
 
 def _openingpathiterator(pathnames: Iterable[str]):

src/dxtbx/sequence_filenames.py

@@ -164,7 +164,7 @@ def find_matching_images(image_name):
 
         matching_images = []
 
-        for j in range(0, 10 ** len_digits):
+        for j in range(0, 10**len_digits):
             if template_str % j in files_in_directory:
                 matching_images.append(os.path.join(directory, template_str % j))
 

src/dxtbx/util/dlsnxs2cbf.py

@@ -167,9 +167,9 @@ def make_cbf(
                 # if 32 bit then it is a signed int, I think if 8, 16 then it is
                 # unsigned with the highest two values assigned as masking values
                 if bit_depth_readout == 32:
-                    top = 2 ** 31
+                    top = 2**31
                 else:
-                    top = 2 ** bit_depth_readout
+                    top = 2**bit_depth_readout
                 d1d = data.as_1d()
                 d1d.set_selected(d1d == top - 1, -1)
                 d1d.set_selected(d1d == top - 2, -2)

tests/command_line/test_average.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import procrunner
 import pytest