Coverage for python / lsst / daf / butler / cli / utils.py: 33%
393 statements
« prev ^ index » next coverage.py v7.14.0, created at 2026-05-23 01:06 -0700
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This software is dual licensed under the GNU General Public License and also
10# under a 3-clause BSD license. Recipients may choose which of these licenses
11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
12# respectively. If you choose the GPL option then the following text applies
13# (but note that there is still no warranty even if you opt for BSD instead):
14#
15# This program is free software: you can redistribute it and/or modify
16# it under the terms of the GNU General Public License as published by
17# the Free Software Foundation, either version 3 of the License, or
18# (at your option) any later version.
19#
20# This program is distributed in the hope that it will be useful,
21# but WITHOUT ANY WARRANTY; without even the implied warranty of
22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
23# GNU General Public License for more details.
24#
25# You should have received a copy of the GNU General Public License
26# along with this program. If not, see <http://www.gnu.org/licenses/>.
27from __future__ import annotations
29__all__ = (
30 "ButlerCommand",
31 "LogCliRunner",
32 "MWArgument",
33 "MWArgumentDecorator",
34 "MWCommand",
35 "MWCtxObj",
36 "MWOption",
37 "MWOptionDecorator",
38 "MWPath",
39 "OptionGroup",
40 "OptionSection",
41 "addArgumentHelp",
42 "astropyTablesToStr",
43 "catch_and_exit",
44 "clickResultMsg",
45 "command_test_env",
46 "option_section",
47 "printAstropyTables",
48 "sortAstropyTable",
49 "split_commas",
50 "split_kv",
51 "textTypeStr",
52 "to_upper",
53 "unwrap",
54 "yaml_presets",
55)
58import importlib.metadata
59import itertools
60import logging
61import os
62import re
63import sys
64import textwrap
65import traceback
66import types
67import uuid
68import warnings
69from collections import Counter
70from collections.abc import Callable, Iterable, Iterator
71from contextlib import contextmanager
72from functools import partial, wraps
73from typing import TYPE_CHECKING, Any
74from unittest.mock import patch
76import click
77import click.core
78import click.exceptions
79import click.testing
80import yaml
81from packaging.version import Version
83from lsst.utils.iteration import ensure_iterable
85from .._config import Config
86from .cliLog import CliLog
88if TYPE_CHECKING:
89 from astropy.table import Table
91 from lsst.daf.butler import Dimension
93_click_version = Version(importlib.metadata.version("click"))
94if _click_version >= Version("8.2.0"): 94 ↛ 97line 94 didn't jump to line 97 because the condition on line 94 was always true
95 _click_make_metavar_has_context = True
96else:
97 _click_make_metavar_has_context = False
99# Starting from Click 8.3.0, a special `UNSET` sentinel value is used to
100# indicate the absence of a default value for a parameter. Prior to 8.3.0,
101# they just used `None`.
102_CLICK_UNSET_SENTINEL = getattr(click.core, "UNSET", None)
104log = logging.getLogger(__name__)
106# This is used as the metavar argument to Options that accept multiple string
107# inputs, which may be comma-separarated. For example:
108# --my-opt foo,bar --my-opt baz.
109# Other arguments to the Option should include multiple=true and
110# callback=split_kv.
111typeStrAcceptsMultiple = "TEXT ..."
112typeStrAcceptsSingle = "TEXT"
114# The standard help string for the --where option when it takes a WHERE clause.
115where_help = (
116 "A string expression similar to a SQL WHERE clause. May involve any column of a "
117 "dimension table or a dimension name as a shortcut for the primary key column of a "
118 "dimension table."
119)
122def astropyTablesToStr(tables: list[Table]) -> str:
123 """Render astropy tables to string as they are displayed in the CLI.
125 Output formatting matches ``printAstropyTables``.
127 Parameters
128 ----------
129 tables : `list` of `astropy.table.Table`
130 The tables to format.
132 Returns
133 -------
134 formatted : `str`
135 Tables formatted into a string.
136 """
137 ret = ""
138 for table in tables:
139 ret += "\n"
140 table.pformat()
141 ret += "\n"
142 return ret
145def printAstropyTables(tables: list[Table]) -> None:
146 """Print astropy tables to be displayed in the CLI.
148 Output formatting matches ``astropyTablesToStr``.
150 Parameters
151 ----------
152 tables : `list` of `astropy.table.Table`
153 The tables to print.
154 """
155 for table in tables:
156 print("")
157 table.pprint_all()
158 print("")
161def textTypeStr(multiple: bool) -> str:
162 """Get the text type string for CLI help documentation.
164 Parameters
165 ----------
166 multiple : `bool`
167 True if multiple text values are allowed, False if only one value is
168 allowed.
170 Returns
171 -------
172 textTypeStr : `str`
173 The type string to use.
174 """
175 return typeStrAcceptsMultiple if multiple else typeStrAcceptsSingle
178class ClickExitFailedNicely:
179 """Exit a Click command that failed.
181 This class is used to control the behavior when a click command has failed.
182 By default the exception will be logged and a non-zero exit status will
183 be used.
185 Parameters
186 ----------
187 exc_type : `type`
188 The type of the exception.
189 exc_value : `Exception`
190 The `Exception` object.
191 exc_tb : `types.TracebackType`
192 The traceback for this exception.
193 """
195 use_bad_status: bool = True
196 """Control how a bad command is exited. `True` indicates bad status and
197 `False` indicates a `click.ClickException`."""
199 def __init__(self, exc_type: type[BaseException], exc_value: BaseException, exc_tb: types.TracebackType):
200 self.exc_type = exc_type
201 self.exc_value = exc_value
202 self.exc_tb = self._clean_tb(exc_tb)
204 def _clean_tb(self, exc_tb: types.TracebackType) -> types.TracebackType:
205 if exc_tb.tb_next:
206 # Do not show the decorator in traceback.
207 exc_tb = exc_tb.tb_next
208 return exc_tb
210 def exit_click(self) -> None:
211 if self.use_bad_status:
212 self.exit_click_command_bad_status()
213 else:
214 self.exit_click_command_click_exception()
216 def exit_click_command_bad_status(self) -> None:
217 """Exit a click command with bad exit status and report log message."""
218 log.exception(
219 "Caught an exception, details are in traceback:",
220 exc_info=(self.exc_type, self.exc_value, self.exc_tb),
221 )
222 # Tell click to stop, this never returns.
223 click.get_current_context().exit(1)
225 def exit_click_command_click_exception(self) -> None:
226 """Exit a click command raising ClickException."""
227 tb = traceback.format_tb(self.exc_tb)
228 errmsg = "".join(tb) + str(self.exc_value)
229 raise click.ClickException(errmsg)
232class LogCliRunner(click.testing.CliRunner):
233 """A test runner to use when the logging system will be initialized by code
234 under test, calls CliLog.resetLog(), which undoes any logging setup that
235 was done with the CliLog interface.
237 lsst.log modules can not be set back to an uninitialized state (python
238 logging modules can be set back to NOTSET), instead they are set to
239 `CliLog.defaultLsstLogLevel`.
240 """
242 def invoke(self, *args: Any, **kwargs: Any) -> click.testing.Result:
243 # We want exceptions to be reported to the test runner rather than
244 # being converted to a simple exit status. The default is to
245 # use a logger but the click test infrastructure doesn't capture that
246 # in result.
247 with patch.object(ClickExitFailedNicely, "use_bad_status", False):
248 result = super().invoke(*args, **kwargs)
249 CliLog.resetLog()
250 if result.exception:
251 print("Failing command was: ", args)
252 return result
255def clickResultMsg(result: click.testing.Result) -> str:
256 """Get a standard assert message from a click result.
258 Parameters
259 ----------
260 result : click.testing.Result
261 The result object returned from `click.testing.CliRunner.invoke`.
263 Returns
264 -------
265 msg : `str`
266 The message string.
267 """
268 msg = f"""\noutput: {result.output}\nexception: {result.exception}"""
269 if result.exception:
270 msg += f"""\ntraceback: {"".join(traceback.format_tb(result.exception.__traceback__))}"""
271 return msg
274@contextmanager
275def command_test_env(runner: click.testing.CliRunner, commandModule: str, commandName: str) -> Iterator[None]:
276 """Context manager that creates (and then cleans up) an environment that
277 provides a CLI plugin command with the given name.
279 Parameters
280 ----------
281 runner : click.testing.CliRunner
282 The test runner to use to create the isolated filesystem.
283 commandModule : `str`
284 The importable module that the command can be imported from.
285 commandName : `str`
286 The name of the command being published to import.
287 """
288 with runner.isolated_filesystem():
289 with open("resources.yaml", "w") as f:
290 f.write(yaml.dump({"cmd": {"import": commandModule, "commands": [commandName]}}))
291 # Add a colon to the end of the path on the next line, this tests the
292 # case where the lookup in LoaderCLI._getPluginList generates an empty
293 # string in one of the list entries and verifies that the empty string
294 # is properly stripped out.
295 with patch.dict("os.environ", {"DAF_BUTLER_PLUGINS": f"{os.path.realpath(f.name)}:"}):
296 yield
299def addArgumentHelp(doc: str | None, helpText: str) -> str:
300 """Add a Click argument's help message to a function's documentation.
302 This is needed because click presents arguments in the order the argument
303 decorators are applied to a function, top down. But, the evaluation of the
304 decorators happens bottom up, so if arguments just append their help to the
305 function's docstring, the argument descriptions appear in reverse order
306 from the order they are applied in.
308 Parameters
309 ----------
310 doc : `str`
311 The function's docstring.
312 helpText : `str`
313 The argument's help string to be inserted into the function's
314 docstring.
316 Returns
317 -------
318 doc : `str`
319 Updated function documentation.
320 """
321 if doc is None: 321 ↛ 322line 321 didn't jump to line 322 because the condition on line 321 was never true
322 doc = helpText
323 else:
324 # See click documentation for details:
325 # https://click.palletsprojects.com/en/7.x/documentation/#truncating-help-texts
326 # In short, text for the click command help can be truncated by putting
327 # "\f" in the docstring, everything after it should be removed
328 if "\f" in doc: 328 ↛ 329line 328 didn't jump to line 329 because the condition on line 328 was never true
329 doc = doc.split("\f")[0]
331 doclines = doc.splitlines()
332 # The function's docstring may span multiple lines, so combine the
333 # docstring from all the first lines until a blank line is encountered.
334 # (Lines after the first blank line will be argument help.)
335 while len(doclines) > 1 and doclines[1]:
336 doclines[0] = " ".join((doclines[0], doclines.pop(1).strip()))
337 # Add standard indent to help text for proper alignment with command
338 # function documentation:
339 helpText = " " + helpText
340 doclines.insert(1, helpText)
341 doclines.insert(1, "\n")
342 doc = "\n".join(doclines)
343 return doc
346def split_commas(
347 context: click.Context | None, param: click.core.Option | None, values: str | Iterable[str] | None
348) -> tuple[str, ...]:
349 """Process a tuple of values, where each value may contain comma-separated
350 values, and return a single list of all the passed-in values.
352 This function can be passed to the 'callback' argument of a click.option to
353 allow it to process comma-separated values (e.g. "--my-opt a,b,c"). If
354 the comma is inside ``[]`` there will be no splitting.
356 Parameters
357 ----------
358 context : `click.Context` or `None`
359 The current execution context. Unused, but Click always passes it to
360 callbacks.
361 param : `click.core.Option` or `None`
362 The parameter being handled. Unused, but Click always passes it to
363 callbacks.
364 values : `~collections.abc.Iterable` of `str` or `str`
365 All the values passed for this option. Strings may contain commas,
366 which will be treated as delimiters for separate values unless they
367 are within ``[]``.
369 Returns
370 -------
371 results : `tuple` [`str`]
372 The passed in values separated by commas where appropriate and
373 combined into a single tuple.
374 """
375 if values is None:
376 return ()
377 valueList = []
378 for value in ensure_iterable(values):
379 # If we have [, or ,] we do the slow split. If square brackets
380 # are not matching then that is likely a typo that should result
381 # in a warning.
382 opens = "["
383 closes = "]"
384 if re.search(rf"\{opens}.*,|,.*\{closes}", value):
385 in_parens = False
386 current = ""
387 for c in value:
388 if c == opens:
389 if in_parens:
390 warnings.warn(
391 f"Found second opening {opens} without corresponding closing {closes}"
392 f" in {value!r}",
393 stacklevel=2,
394 )
395 in_parens = True
396 elif c == closes:
397 if not in_parens:
398 warnings.warn(
399 f"Found second closing {closes} without corresponding open {opens} in {value!r}",
400 stacklevel=2,
401 )
402 in_parens = False
403 elif c == "," and not in_parens:
404 # Split on this comma.
405 valueList.append(current)
406 current = ""
407 continue
408 current += c
409 if in_parens:
410 warnings.warn(
411 f"Found opening {opens} that was never closed in {value!r}",
412 stacklevel=2,
413 )
414 if current:
415 valueList.append(current)
416 else:
417 # Use efficient split since no parens.
418 valueList.extend(value.split(","))
419 return tuple(valueList)
422def split_kv(
423 context: click.Context,
424 param: click.core.Option,
425 values: list[str],
426 *,
427 choice: click.Choice | None = None,
428 multiple: bool = True,
429 normalize: bool = False,
430 separator: str = "=",
431 unseparated_okay: bool = False,
432 return_type: type[dict] | type[tuple] = dict,
433 default_key: str | None = "",
434 reverse_kv: bool = False,
435 add_to_default: bool = False,
436) -> dict[str | None, str] | tuple[tuple[str | None, str], ...]:
437 """Process a tuple of values that are key-value pairs separated by a given
438 separator. Multiple pairs may be comma separated. Return a dictionary of
439 all the passed-in values.
441 This function can be passed to the 'callback' argument of a click.option to
442 allow it to process comma-separated values (e.g. "--my-opt a=1,b=2").
444 Parameters
445 ----------
446 context : `click.Context` or `None`
447 The current execution context. Unused, but Click always passes it to
448 callbacks.
449 param : `click.core.Option` or `None`
450 The parameter being handled. Unused, but Click always passes it to
451 callbacks.
452 values : [`str`]
453 All the values passed for this option. Strings may contain commas,
454 which will be treated as delimiters for separate values.
455 choice : `click.Choice`, optional
456 If provided, verify each value is a valid choice using the provided
457 `click.Choice` instance. If None, no verification will be done. By
458 default `None`.
459 multiple : `bool`, optional
460 If true, the value may contain multiple comma-separated values. By
461 default True.
462 normalize : `bool`, optional
463 If True and ``choice.case_sensitive == False``, normalize the string
464 the user provided to match the choice's case. By default `False`.
465 separator : str, optional
466 The character that separates key-value pairs. May not be a comma or an
467 empty space (for space separators use Click's default implementation
468 for tuples; ``type=(str, str)``). By default "=".
469 unseparated_okay : `bool`, optional
470 If True, allow values that do not have a separator. They will be
471 returned in the values dict as a tuple of values in the key '', that
472 is: ``values[''] = (unseparated_values, )``. By default False.
473 return_type : `type`, must be `dict` or `tuple`
474 The type of the value that should be returned.
475 If `dict` then the returned object will be a dict, for each item in
476 values, the value to the left of the separator will be the key and the
477 value to the right of the separator will be the value.
478 If `tuple` then the returned object will be a tuple. Each item in the
479 tuple will be 2-item tuple, the first item will be the value to the
480 left of the separator and the second item will be the value to the
481 right. By default `dict`.
482 default_key : `str` or `None`
483 The key to use if a value is passed that is not a key-value pair.
484 `None` can imply no separator depending on how the results are handled.
485 (Passing values that are not key-value pairs requires
486 ``unseparated_okay`` to be `True`).
487 reverse_kv : bool
488 If true then for each item in values, the value to the left of the
489 separator is treated as the value and the value to the right of the
490 separator is treated as the key. By default `False`.
491 add_to_default : `bool`, optional
492 If True, then passed-in values will not overwrite the default value
493 unless the ``return_type`` is `dict` and passed-in value(s) have the
494 same key(s) as the default value.
496 Returns
497 -------
498 values : `dict` [`str`, `str`] or `tuple` [`tuple` [`str`, `str`], ...]
499 The passed-in values in dict form or tuple form.
501 Raises
502 ------
503 `click.ClickException`
504 Raised if the separator is not found in an entry, or if duplicate keys
505 are encountered.
506 """
508 def norm(val: str) -> str:
509 """If `normalize` is `True` and `choice` is not `None`, find the value
510 in the available choices and return the value as spelled in the
511 choices.
513 Assumes that val exists in choices; `split_kv` uses the `choice`
514 instance to verify val is a valid choice.
516 Parameters
517 ----------
518 val : `str`
519 Value to be found.
521 Returns
522 -------
523 val : `str`
524 The value that was found or the value that was given.
525 """
526 if normalize and choice is not None:
527 v = val.casefold()
528 for opt in choice.choices:
529 if opt.casefold() == v:
530 return opt
531 return val
533 class RetDict:
534 def __init__(self) -> None:
535 self.ret: dict[str | None, str] = {}
537 def add(self, key: str | None, val: str) -> None:
538 if reverse_kv:
539 key, val = val, key
540 self.ret[key] = val
542 def get(self) -> dict[str | None, str]:
543 return self.ret
545 class RetTuple:
546 def __init__(self) -> None:
547 self.ret: list[tuple[str | None, str]] = []
549 def add(self, key: str | None, val: str) -> None:
550 if reverse_kv:
551 key, val = val, key
552 self.ret.append((key, val))
554 def get(self) -> tuple[tuple[str | None, str], ...]:
555 return tuple(self.ret)
557 if separator in (",", " "):
558 raise RuntimeError(f"'{separator}' is not a supported separator for key-value pairs.")
559 vals = tuple(ensure_iterable(values)) # preserve the original argument for error reporting below.
561 if add_to_default:
562 default = param.get_default(context)
563 if default and default != _CLICK_UNSET_SENTINEL:
564 vals = tuple(
565 v for v in itertools.chain(ensure_iterable(default), vals)
566 ) # Convert to tuple for mypy
568 ret: RetDict | RetTuple
569 if return_type is dict:
570 ret = RetDict()
571 elif return_type is tuple:
572 ret = RetTuple()
573 else:
574 raise click.ClickException(
575 message=f"Internal error: invalid return type '{return_type}' for split_kv."
576 )
577 if multiple:
578 vals = split_commas(context, param, vals)
579 for val in ensure_iterable(vals):
580 if unseparated_okay and separator not in val:
581 if choice is not None:
582 choice(val) # will raise if val is an invalid choice
583 ret.add(default_key, norm(val))
584 else:
585 try:
586 k, v = val.split(separator)
587 if choice is not None:
588 choice(v) # will raise if val is an invalid choice
589 except ValueError as e:
590 raise click.ClickException(
591 message=f"Could not parse key-value pair '{val}' using separator '{separator}', "
592 f"with multiple values {'allowed' if multiple else 'not allowed'}: {e}"
593 ) from None
594 ret.add(k, norm(v))
595 return ret.get()
598def to_upper(context: click.Context, param: click.core.Option, value: str) -> str:
599 """Convert a value to upper case.
601 Parameters
602 ----------
603 context : `click.Context`
604 Context given by Click.
605 param : `click.core.Option`
606 Provided by Click. Ignored.
607 value : `str`
608 The value to be converted.
610 Returns
611 -------
612 str
613 A copy of the passed-in value, converted to upper case.
614 """
615 return value.upper()
618def unwrap(val: str) -> str:
619 """Remove newlines and leading whitespace from a multi-line string with
620 a consistent indentation level.
622 The first line of the string may be only a newline or may contain text
623 followed by a newline, either is ok. After the first line, each line must
624 begin with a consistant amount of whitespace. So, content of a
625 triple-quoted string may begin immediately after the quotes, or the string
626 may start with a newline. Each line after that must be the same amount of
627 indentation/whitespace followed by text and a newline. The last line may
628 end with a new line but is not required to do so.
630 Parameters
631 ----------
632 val : `str`
633 The string to change.
635 Returns
636 -------
637 strippedString : `str`
638 The string with newlines, indentation, and leading and trailing
639 whitespace removed.
640 """
642 def splitSection(val: str) -> str:
643 if not val.startswith("\n"): 643 ↛ 647line 643 didn't jump to line 647 because the condition on line 643 was always true
644 firstLine, _, val = val.partition("\n")
645 firstLine += " "
646 else:
647 firstLine = ""
648 return (firstLine + textwrap.dedent(val).replace("\n", " ")).strip()
650 return "\n\n".join([splitSection(s) for s in val.split("\n\n")])
653class option_section: # noqa: N801
654 """Decorator to add a section label between options in the help text of a
655 command.
657 Parameters
658 ----------
659 sectionText : `str`
660 The text to print in the section identifier.
661 """
663 def __init__(self, sectionText: str) -> None:
664 self.sectionText = "\n" + sectionText
666 def __call__(self, f: Any) -> click.Option:
667 # Generate a parameter declaration that will be unique for this
668 # section.
669 return click.option(
670 f"--option-section-{str(uuid.uuid4())}", sectionText=self.sectionText, cls=OptionSection
671 )(f)
674class MWPath(click.Path):
675 """Overrides `click.Path` to implement file-does-not-exist checking.
677 Changes the definition of ``exists`` so that `True` indicates the location
678 (file or directory) must exist, `False` indicates the location must *not*
679 exist, and `None` indicates that the file may exist or not. The standard
680 definition for the `click.Path` ``exists`` parameter is that for `True` a
681 location must exist, but `False` means it is not required to exist (not
682 that it is required to not exist).
684 Parameters
685 ----------
686 exists : `bool` or `None`, optional
687 If `True`, the location (file or directory) indicated by the caller
688 must exist. If `False` the location must not exist. If `None`, the
689 location may exist or not.
690 file_okay : `bool`, optional
691 Allow a file as a value.
692 dir_okay : `bool`, optional
693 Allow a directory as a value.
694 writable : `bool`, optional
695 If `True`, a writable check is performed.
696 readable : `bool`, optional
697 If `True`, a readable check is performed.
698 resolve_path : `bool`, optional
699 Resolve the path.
700 allow_dash : `bool`, optional
701 Allow single dash as value to mean a standard stream.
702 path_type : `type` or `None`, optional
703 Convert the incoming value to this type.
705 Notes
706 -----
707 All parameters other than ``exists`` come directly from `click.Path`.
708 """
710 def __init__(
711 self,
712 exists: bool | None = None,
713 file_okay: bool = True,
714 dir_okay: bool = True,
715 writable: bool = False,
716 readable: bool = True,
717 resolve_path: bool = False,
718 allow_dash: bool = False,
719 path_type: type | None = None,
720 ):
721 self.mustNotExist = exists is False
722 if exists is None: 722 ↛ 724line 722 didn't jump to line 724 because the condition on line 722 was always true
723 exists = False
724 super().__init__(
725 exists=exists,
726 file_okay=file_okay,
727 dir_okay=dir_okay,
728 writable=writable,
729 readable=readable,
730 resolve_path=resolve_path,
731 allow_dash=allow_dash,
732 path_type=path_type,
733 )
735 def convert(
736 self, value: str | os.PathLike[str], param: click.Parameter | None, ctx: click.Context | None
737 ) -> Any:
738 """Convert values through types.
740 Called by `click.ParamType` to "convert values through types".
741 `click.Path` uses this step to verify Path conditions.
743 Parameters
744 ----------
745 value : `str` or `os.PathLike`
746 File path.
747 param : `click.Parameter`
748 Parameters provided by Click.
749 ctx : `click.Context`
750 Context provided by Click.
751 """
752 if self.mustNotExist and os.path.exists(value):
753 self.fail(f'Path "{value}" should not exist.')
754 return super().convert(value, param, ctx)
757class MWOption(click.Option):
758 """Overrides click.Option with desired behaviors."""
760 def make_metavar(self, ctx: click.Context | None = None) -> str:
761 """Make the metavar for the help menu.
763 Parameters
764 ----------
765 ctx : `click.Context` or `None`
766 Context from the command.
768 Notes
769 -----
770 Overrides `click.Option.make_metavar`.
771 Adds a space and an ellipsis after the metavar name if
772 the option accepts multiple inputs, otherwise defers to the base
773 implementation.
775 By default click does not add an ellipsis when multiple is True and
776 nargs is 1. And when nargs does not equal 1 click adds an ellipsis
777 without a space between the metavar and the ellipsis, but we prefer a
778 space between.
780 Does not get called for some option types (e.g. flag) so metavar
781 transformation that must apply to all types should be applied in
782 get_help_record.
783 """
784 if _click_make_metavar_has_context:
785 metavar = super().make_metavar(ctx=ctx) # type: ignore
786 else:
787 metavar = super().make_metavar() # type: ignore
788 if self.multiple and self.nargs == 1:
789 metavar += " ..."
790 elif self.nargs != 1:
791 metavar = f"{metavar[:-3]} ..."
792 return metavar
795class MWArgument(click.Argument):
796 """Overrides click.Argument with desired behaviors."""
798 def make_metavar(self, ctx: click.Context | None = None) -> str:
799 """Make the metavar for the help menu.
801 Parameters
802 ----------
803 ctx : `click.Context` or `None`
804 Context from the command.
806 Notes
807 -----
808 Overrides `click.Option.make_metavar`.
809 Always adds a space and an ellipsis (' ...') after the
810 metavar name if the option accepts multiple inputs.
812 By default click adds an ellipsis without a space between the metavar
813 and the ellipsis, but we prefer a space between.
815 Returns
816 -------
817 metavar : `str`
818 The metavar value.
819 """
820 if _click_make_metavar_has_context:
821 metavar = super().make_metavar(ctx=ctx) # type: ignore
822 else:
823 metavar = super().make_metavar() # type: ignore
824 if self.nargs != 1:
825 metavar = f"{metavar[:-3]} ..."
826 return metavar
829class OptionSection(MWOption):
830 """Implements an Option that prints a section label in the help text and
831 does not pass any value to the command function.
833 This class does a bit of hackery to add a section label to a click command
834 help output: first, ``expose_value`` is set to `False` so that no value is
835 passed to the command function. Second, this class overrides
836 `click.Option.get_help_record` to return the section label string without
837 any prefix so that it stands out as a section label.
839 This class overrides the hidden attribute because our documentation build
840 tool, sphinx-click, implements its own `get_help_record` function which
841 builds the record from other option values (e.g. ``name``, ``opts``), which
842 breaks the hack we use to make `get_help_record` only return the
843 ``sectionText``. Fortunately, Click gets the value of `hidden` inside the
844 `click.Option`'s `get_help_record`, and sphinx-click calls ``opt.hidden``
845 before entering its ``_get_help_record`` function. So, making the hidden
846 property return True hides this option from sphinx-click, while allowing
847 the section text to be returned by our `get_help_record` method when using
848 Click.
850 The intention for this implementation is to do minimally invasive overrides
851 of the click classes so as to be robust and easy to fix if the click
852 internals change.
854 Parameters
855 ----------
856 sectionName : `str`
857 The parameter declaration for this option. It is not shown to the user,
858 it must be unique within the command. If using the ``section``
859 decorator to add a section to a command's options, the section name is
860 auto-generated.
861 sectionText : `str`
862 The text to print in the section identifier.
863 """
865 @property
866 def hidden(self) -> bool:
867 return True
869 @hidden.setter
870 def hidden(self, val: Any) -> None:
871 pass
873 def __init__(self, sectionName: str, sectionText: str) -> None:
874 super().__init__(sectionName, expose_value=False)
875 self.sectionText = sectionText
877 def get_help_record(self, ctx: click.Context | None) -> tuple[str, str]:
878 return (self.sectionText, "")
881class MWOptionDecorator:
882 """Wraps the click.option decorator to enable shared options to be declared
883 and allows inspection of the shared option.
885 Parameters
886 ----------
887 *param_decls : `typing.Any`
888 Parameters to be stored in the option.
889 **kwargs : `typing.Any`
890 Keyword arguments for the option.
891 """
893 def __init__(self, *param_decls: Any, **kwargs: Any) -> None:
894 self.partialOpt = partial(click.option, *param_decls, cls=partial(MWOption), **kwargs) # type: ignore
895 opt = click.Option(param_decls, **kwargs)
896 self._name = opt.name
897 self._opts = opt.opts
899 def name(self) -> str:
900 """Get the name that will be passed to the command function for this
901 option.
902 """
903 assert self._name is not None, "keep mypy happy"
904 return self._name
906 def opts(self) -> list[str]:
907 """Get the flags that will be used for this option on the command
908 line.
909 """
910 return self._opts
912 @property
913 def help(self) -> str:
914 """Get the help text for this option. Returns an empty string if no
915 help was defined.
916 """
917 return self.partialOpt.keywords.get("help", "")
919 def __call__(self, *args: Any, **kwargs: Any) -> Any:
920 return self.partialOpt(*args, **kwargs)
923class MWArgumentDecorator:
924 """Wraps the click.argument decorator to enable shared arguments to be
925 declared.
927 Parameters
928 ----------
929 *param_decls : `typing.Any`
930 Parameters to be stored in the argument.
931 **kwargs : `typing.Any`
932 Keyword arguments for the argument.
933 """
935 def __init__(self, *param_decls: Any, **kwargs: Any) -> None:
936 self._helpText = kwargs.pop("help", None)
937 self.partialArg = partial(click.argument, *param_decls, cls=MWArgument, **kwargs)
939 def __call__(self, *args: Any, help: str | None = None, **kwargs: Any) -> Callable:
940 def decorator(f: Any) -> Any:
941 if help is not None:
942 self._helpText = help
943 if self._helpText:
944 f.__doc__ = addArgumentHelp(f.__doc__, self._helpText)
945 return self.partialArg(*args, **kwargs)(f)
947 return decorator
950class MWCommand(click.Command):
951 """Command subclass that stores a copy of the args list for use by the
952 command.
954 Parameters
955 ----------
956 *args : `typing.Any`
957 Arguments for `click.Command`.
958 **kwargs : `typing.Any`
959 Keyword arguments for `click.Command`.
960 """
962 name = "butler"
963 extra_epilog: str | None = None
965 def __init__(self, *args: Any, **kwargs: Any) -> None:
966 # wrap callback method with catch_and_exit decorator
967 callback = kwargs.get("callback")
968 if callback is not None: 968 ↛ 971line 968 didn't jump to line 971 because the condition on line 968 was always true
969 kwargs = kwargs.copy()
970 kwargs["callback"] = catch_and_exit(callback)
971 super().__init__(*args, **kwargs)
973 def _capture_args(self, ctx: click.Context, args: list[str]) -> None:
974 """Capture the command line options and arguments.
976 See details about what is captured and the order in which it is stored
977 in the documentation of `MWCtxObj`.
979 Parameters
980 ----------
981 ctx : `click.Context`
982 The current Context.
983 args : `list` [`str`]
984 The list of arguments from the command line, split at spaces but
985 not at separators (like "=").
986 """
987 parser = self.make_parser(ctx)
988 opts, _, param_order = parser.parse_args(args=list(args))
989 # `param_order` is a list of click.Option and click.Argument, there is
990 # one item for each time the Option or Argument was used on the
991 # command line. Options will precede Arguments, within each sublist
992 # they are in the order they were used on the command line. Note that
993 # click.Option and click.Argument do not contain the value from the
994 # command line; values are in `opts`.
995 #
996 # `opts` is a dict where the key is the argument name to the
997 # click.Command function, this name matches the `click.Option.name` or
998 # `click.Argument.name`. For Options, an item will only be present if
999 # the Option was used on the command line. For Arguments, an item will
1000 # always be present and if no value was provided on the command line
1001 # the value will be `None`. If the option accepts multiple values, the
1002 # value in `opts` is a tuple, otherwise it is a single item.
1003 next_idx: Counter = Counter()
1004 captured_args = []
1005 for param in param_order:
1006 if isinstance(param, click.Option):
1007 param_name = param.name
1008 assert param_name is not None, "keep mypy happy"
1009 if param.multiple:
1010 val = opts[param_name][next_idx[param_name]]
1011 next_idx[param_name] += 1
1012 else:
1013 val = opts[param_name]
1014 if param.is_flag:
1015 # Bool options store their True flags in opts and their
1016 # False flags in secondary_opts.
1017 if val:
1018 flag = max(param.opts, key=len)
1019 else:
1020 flag = max(param.secondary_opts, key=len)
1021 captured_args.append(flag)
1022 else:
1023 captured_args.append(max(param.opts, key=len))
1024 captured_args.append(val)
1025 elif isinstance(param, click.Argument):
1026 param_name = param.name
1027 assert param_name is not None, "keep mypy happy"
1028 opt = opts[param_name]
1029 if opt is not None and opt != _CLICK_UNSET_SENTINEL:
1030 captured_args.append(opt)
1031 else:
1032 raise AssertionError("All parameters should be an Option or an Argument")
1033 MWCtxObj.getFrom(ctx).args = captured_args
1035 def parse_args(self, ctx: click.Context, args: Any) -> list[str]:
1036 """Given a context and a list of arguments this creates the parser and
1037 parses the arguments, then modifies the context as necessary. This is
1038 automatically invoked by make_context().
1040 This function overrides `click.Command.parse_args`.
1042 The call to `_capture_args` in this override stores the arguments
1043 (option names, option value, and argument values) that were used by the
1044 caller on the command line in the context object. These stored
1045 arguments can be used by the command function, e.g. to process options
1046 in the order they appeared on the command line (``pipetask`` uses this
1047 feature to create pipeline actions in an order from different options).
1049 Parameters
1050 ----------
1051 ctx : `click.core.Context`
1052 The current Context.
1053 args : `list` [`str`]
1054 The list of arguments from the command line, split at spaces but
1055 not at separators (like "=").
1056 """
1057 self._capture_args(ctx, args)
1058 return super().parse_args(ctx, args)
1060 @property
1061 def epilog(self) -> str | None:
1062 """Override the epilog attribute to add extra_epilog (if defined by a
1063 subclass) to the end of any epilog provided by a subcommand.
1064 """
1065 ret = self._epilog if self._epilog else ""
1066 if self.extra_epilog:
1067 if ret:
1068 ret += "\n\n"
1069 ret += self.extra_epilog
1070 return ret
1072 @epilog.setter
1073 def epilog(self, val: str | None) -> None:
1074 self._epilog = val
1077class ButlerCommand(MWCommand):
1078 """Command subclass with butler-command specific overrides."""
1080 extra_epilog = "See 'butler --help' for more options."
1083class OptionGroup:
1084 """Base class for an option group decorator. Requires the option group
1085 subclass to have a property called ``decorator``.
1086 """
1088 decorators: list[Any]
1090 def __call__(self, f: Any) -> Any:
1091 for decorator in reversed(self.decorators):
1092 f = decorator(f)
1093 return f
1096class MWCtxObj:
1097 """Helper object for managing the `click.Context.obj` parameter, allows
1098 obj data to be managed in a consistent way.
1100 `click.Context.obj` defaults to None. ``MWCtxObj.getFrom(ctx)`` can be used
1101 to initialize the obj if needed and return a new or existing `MWCtxObj`.
1103 The `args` attribute contains a list of options, option values, and
1104 argument values that is similar to the list of arguments and options that
1105 were passed in on the command line, with differences noted below:
1107 * Option names and option values are first in the list, and argument
1108 values come last. The order of options and option values is preserved
1109 within the options. The order of argument values is preserved.
1111 * The longest option name is used for the option in the `args` list, e.g.
1112 if an option accepts both short and long names ``"-o / --option"`` and
1113 the short option name ``"-o"`` was used on the command line, the longer
1114 name will be the one that appears in ``args``.
1116 * A long option name (which begins with two dashes ``"--"``) and its value
1117 may be separated by an equal sign; the name and value are split at the
1118 equal sign and it is removed. In ``args``, the option is in one list
1119 item, and the option value (without the equal sign) is in the next list
1120 item. e.g. ``"--option=foo"`` and ``"--option foo"`` both become
1121 ``["--opt", "foo"]`` in ``args``.
1123 * A short option name, (which begins with one dash ``"-"``) and its value
1124 are split immediately after the short option name, and if there is
1125 whitespace between the short option name and its value it is removed.
1126 Everything after the short option name (excluding whitespace) is included
1127 in the value. If the ``Option`` has a long name, the long name will be
1128 used in ``args`` e.g. for the option ``"-o / --option"``: ``"-ofoo"`` and
1129 ``"-o foo"`` become ``["--option", "foo"]``, and (note!) ``"-o=foo"``
1130 will become ``["--option", "=foo"]`` (because everything after the short
1131 option name, except whitespace, is used for the value (as is standard
1132 with unix command line tools).
1134 Attributes
1135 ----------
1136 args : `list` [`str`]
1137 A list of options, option values, and arguments similar to those that
1138 were passed in on the command line. See comments about captured options
1139 & arguments above.
1140 """
1142 def __init__(self) -> None:
1143 self.args = None
1145 @staticmethod
1146 def getFrom(ctx: click.Context) -> Any:
1147 """If needed, initialize ``ctx.obj`` with a new `MWCtxObj`, and return
1148 the new or already existing `MWCtxObj`.
1150 Parameters
1151 ----------
1152 ctx : `click.Context`
1153 Context provided by Click.
1154 """
1155 if ctx.obj is not None:
1156 return ctx.obj
1157 ctx.obj = MWCtxObj()
1158 return ctx.obj
1161def yaml_presets(ctx: click.Context, param: str, value: Any) -> None:
1162 """Read additional values from the supplied YAML file.
1164 Parameters
1165 ----------
1166 ctx : `click.Context`
1167 The context for the click operation. Used to extract the subcommand
1168 name and translate option & argument names.
1169 param : `str`
1170 The parameter name.
1171 value : `object`
1172 The value of the parameter.
1173 """
1175 def _name_for_option(ctx: click.Context, option: str) -> str:
1176 """Use a CLI option name to find the name of the argument to the
1177 command function.
1179 Parameters
1180 ----------
1181 ctx : `click.Context`
1182 The context for the click operation.
1183 option : `str`
1184 The option/argument name from the yaml file.
1186 Returns
1187 -------
1188 name : str
1189 The name of the argument to use when calling the click.command
1190 function, as it should appear in the `ctx.default_map`.
1192 Raises
1193 ------
1194 RuntimeError
1195 Raised if the option name from the yaml file does not exist in the
1196 command parameters. This catches misspellings and incorrect usage
1197 in the yaml file.
1198 """
1199 for param in ctx.command.params:
1200 # Remove leading dashes: they are not used for option names in the
1201 # yaml file.
1202 if option in [opt.lstrip("-") for opt in param.opts]:
1203 assert param.name is not None, "keep mypy happy"
1204 return param.name
1205 raise RuntimeError(f"'{option}' is not a valid option for {ctx.info_name}")
1207 ctx.default_map = ctx.default_map or {}
1208 cmd_name = ctx.info_name
1209 assert cmd_name is not None, "command name cannot be None"
1210 if value:
1211 try:
1212 overrides = _read_yaml_presets(value, cmd_name)
1213 options = list(overrides.keys())
1214 for option in options:
1215 name = _name_for_option(ctx, option)
1216 if name == option:
1217 continue
1218 overrides[name] = overrides.pop(option)
1219 except Exception as e:
1220 raise click.BadOptionUsage(
1221 option_name=param,
1222 message=f"Error reading overrides file: {e}",
1223 ctx=ctx,
1224 ) from None
1225 # Override the defaults for this subcommand
1226 ctx.default_map.update(overrides)
1227 return
1230def _read_yaml_presets(file_uri: str, cmd_name: str) -> dict[str, Any]:
1231 """Read file command line overrides from YAML config file.
1233 Parameters
1234 ----------
1235 file_uri : `str`
1236 URI of override YAML file containing the command line overrides.
1237 They should be grouped by command name.
1238 cmd_name : `str`
1239 The subcommand name that is being modified.
1241 Returns
1242 -------
1243 overrides : `dict` of [`str`, Any]
1244 The relevant command line options read from the override file.
1245 """
1246 log.debug("Reading command line overrides for subcommand %s from URI %s", cmd_name, file_uri)
1247 config = Config(file_uri)
1248 return config[cmd_name]
1251def sortAstropyTable(table: Table, dimensions: list[Dimension], sort_first: list[str] | None = None) -> Table:
1252 """Sort an astropy table.
1254 Prioritization is given to columns in this order:
1256 1. the provided named columns
1257 2. spatial and temporal columns
1258 3. the rest of the columns.
1260 The table is sorted in-place, and is also returned for convenience.
1262 Parameters
1263 ----------
1264 table : `astropy.table.Table`
1265 The table to sort.
1266 dimensions : `list` [``Dimension``]
1267 The dimensions of the dataIds in the table (the dimensions should be
1268 the same for all the dataIds). Used to determine if the column is
1269 spatial, temporal, or neither.
1270 sort_first : `list` [`str`]
1271 The names of columns that should be sorted first, before spatial and
1272 temporal columns.
1274 Returns
1275 -------
1276 `astropy.table.Table`
1277 For convenience, the table that has been sorted.
1278 """
1279 # For sorting we want to ignore the id
1280 # We also want to move temporal or spatial dimensions earlier
1281 sort_first = sort_first or []
1282 sort_early: list[str] = []
1283 sort_late: list[str] = []
1284 for dim in dimensions:
1285 if dim.spatial or dim.temporal:
1286 sort_early.extend(dim.required.names)
1287 else:
1288 sort_late.append(str(dim))
1289 sort_keys = sort_first + sort_early + sort_late
1290 # The required names above means that we have the possibility of
1291 # repeats of sort keys. Now have to remove them
1292 # (order is retained by dict creation).
1293 sort_keys = list(dict.fromkeys(sort_keys).keys())
1295 table.sort(sort_keys)
1296 return table
1299def catch_and_exit(func: Callable) -> Callable:
1300 """Catch all exceptions, prints an exception traceback
1301 and signals click to exit.
1303 Use as decorator.
1305 Parameters
1306 ----------
1307 func : `collections.abc.Callable`
1308 The function to be decorated.
1310 Returns
1311 -------
1312 `collections.abc.Callable`
1313 The decorated function.
1314 """
1316 @wraps(func)
1317 def inner(*args: Any, **kwargs: Any) -> None:
1318 try:
1319 func(*args, **kwargs)
1320 except (click.exceptions.ClickException, click.exceptions.Exit, click.exceptions.Abort):
1321 # this is handled by click itself
1322 raise
1323 except Exception:
1324 exc_type, exc_value, exc_tb = sys.exc_info()
1325 assert exc_type is not None
1326 assert exc_value is not None
1327 assert exc_tb is not None
1328 exit_hdl = ClickExitFailedNicely(exc_type, exc_value, exc_tb)
1329 exit_hdl.exit_click()
1331 return inner