Coverage for python / lsst / daf / butler / dimensions / _universe.py: 39%
178 statements
« prev ^ index » next coverage.py v7.14.0, created at 2026-05-27 01:07 -0700
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This software is dual licensed under the GNU General Public License and also
10# under a 3-clause BSD license. Recipients may choose which of these licenses
11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
12# respectively. If you choose the GPL option then the following text applies
13# (but note that there is still no warranty even if you opt for BSD instead):
14#
15# This program is free software: you can redistribute it and/or modify
16# it under the terms of the GNU General Public License as published by
17# the Free Software Foundation, either version 3 of the License, or
18# (at your option) any later version.
19#
20# This program is distributed in the hope that it will be useful,
21# but WITHOUT ANY WARRANTY; without even the implied warranty of
22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
23# GNU General Public License for more details.
24#
25# You should have received a copy of the GNU General Public License
26# along with this program. If not, see <http://www.gnu.org/licenses/>.
28from __future__ import annotations
30__all__ = ["DimensionUniverse"]
32import logging
33import pickle
34import warnings
35from collections import defaultdict
36from collections.abc import Iterable, Mapping, Sequence
37from typing import TYPE_CHECKING, Any, ClassVar, TypeVar, overload
39from lsst.utils.classes import cached_getter, immutable
41from .._config import Config
42from .._exceptions import InconsistentUniverseError
43from .._named import NamedValueAbstractSet, NamedValueSet
44from .._topology import TopologicalFamily, TopologicalSpace
45from .._utilities.thread_safe_cache import ThreadSafeCache
46from ._config import _DEFAULT_NAMESPACE, DimensionConfig
47from ._database import DatabaseDimensionElement
48from ._elements import Dimension, DimensionElement
49from ._governor import GovernorDimension
50from ._group import DimensionGroup
51from ._skypix import SkyPixDimension, SkyPixSystem
53if TYPE_CHECKING: # Imports needed only for type annotations; may be circular.
54 from .construction import DimensionConstructionBuilder
57E = TypeVar("E", bound=DimensionElement)
58_LOG = logging.getLogger(__name__)
61@immutable
62class DimensionUniverse: # numpydoc ignore=PR02
63 """Self-consistent set of dimensions.
65 A parent class that represents a complete, self-consistent set of
66 dimensions and their relationships.
68 `DimensionUniverse` is not a class-level singleton, but all instances are
69 tracked in a singleton map keyed by the version number and namespace
70 in the configuration they were loaded from. Because these universes
71 are solely responsible for constructing `DimensionElement` instances,
72 these are also indirectly tracked by that singleton as well.
74 Parameters
75 ----------
76 config : `Config`, optional
77 Configuration object from which dimension definitions can be extracted.
78 Ignored if ``builder`` is provided, or if ``version`` is provided and
79 an instance with that version already exists.
80 version : `int`, optional
81 Integer version for this `DimensionUniverse`. If not provided, a
82 version will be obtained from ``builder`` or ``config``.
83 namespace : `str`, optional
84 Namespace of this `DimensionUniverse`, combined with the version
85 to provide universe safety for registries that use different
86 dimension definitions.
87 builder : `DimensionConstructionBuilder`, optional
88 Builder object used to initialize a new instance. Ignored if
89 ``version`` is provided and an instance with that version already
90 exists. Should not have had `~DimensionConstructionBuilder.finish`
91 called; this will be called if needed by `DimensionUniverse`.
92 use_cache : `bool`, optional
93 If `True` or not provided, cache `DimensionUniverse` instances globally
94 to avoid creating more than one `DimensionUniverse` instance for a
95 given configuration.
96 """
98 _instances: ClassVar[ThreadSafeCache[tuple[int, str], DimensionUniverse]] = ThreadSafeCache()
99 """Singleton dictionary of all instances, keyed by version.
101 For internal use only.
102 """
104 def __new__(
105 cls,
106 config: Config | None = None,
107 *,
108 version: int | None = None,
109 namespace: str | None = None,
110 builder: DimensionConstructionBuilder | None = None,
111 use_cache: bool = True,
112 ) -> DimensionUniverse:
113 # Try to get a version first, to look for existing instances; try to
114 # do as little work as possible at this stage.
115 if version is None:
116 if builder is None:
117 config = DimensionConfig(config)
118 version = config["version"]
119 else:
120 version = builder.version
122 if use_cache is not True:
123 warnings.warn(
124 "use_cache parameter is no longer supported and is ignored. Will be removed after v30.",
125 category=FutureWarning,
126 stacklevel=2,
127 )
129 # Then a namespace.
130 if namespace is None:
131 if builder is None:
132 config = DimensionConfig(config)
133 namespace = config.get("namespace", _DEFAULT_NAMESPACE)
134 else:
135 namespace = builder.namespace
136 # if still None use the default
137 if namespace is None:
138 namespace = _DEFAULT_NAMESPACE
140 # See if an equivalent instance already exists.
141 existing_instance = cls._instances.get((version, namespace))
142 if existing_instance is not None:
143 return existing_instance
145 # Ensure we have a builder, building one from config if necessary.
146 if builder is None:
147 config = DimensionConfig(config)
148 builder = config.makeBuilder()
150 # Delegate to the builder for most of the construction work.
151 builder.finish()
153 # Create the universe instance and create core attributes, mostly
154 # copying from builder.
155 self: DimensionUniverse = object.__new__(cls)
156 assert self is not None
157 self._cached_groups = ThreadSafeCache()
158 self._dimensions = builder.dimensions
159 self._elements = builder.elements
160 self._topology = builder.topology
161 self.dimensionConfig = builder.config
162 commonSkyPix = self._dimensions[builder.commonSkyPixName]
163 assert isinstance(commonSkyPix, SkyPixDimension)
164 self.commonSkyPix = commonSkyPix
166 # Attach self to all elements.
167 for element in self._elements:
168 element.universe = self
170 # Add attribute for special subsets of the graph.
171 self._empty = DimensionGroup(self, (), _conform=False)
173 # Use the version number and namespace from the config as a key in
174 # the singleton dict containing all instances; that will let us
175 # transfer dimension objects between processes using pickle without
176 # actually going through real initialization, as long as a universe
177 # with the same version and namespace has already been constructed in
178 # the receiving process.
179 self._version = version
180 self._namespace = namespace
182 # Build mappings from element to index. These are used for
183 # topological-sort comparison operators in DimensionElement itself.
184 self._elementIndices = {name: i for i, name in enumerate(self._elements.names)}
185 # Same for dimension to index, sorted topologically across required
186 # and implied. This is used for encode/decode.
187 self._dimensionIndices = {name: i for i, name in enumerate(self._dimensions.names)}
189 self._populates = defaultdict(NamedValueSet)
190 for element in self._elements:
191 if element.populated_by is not None:
192 self._populates[element.populated_by.name].add(element)
194 return cls._instances.set_or_get((self._version, self._namespace), self)
196 @property
197 def version(self) -> int:
198 """The version number of this universe.
200 Returns
201 -------
202 version : `int`
203 An integer representing the version number of this universe.
204 Uniquely defined when combined with the `namespace`.
205 """
206 return self._version
208 @property
209 def namespace(self) -> str:
210 """The namespace associated with this universe.
212 Returns
213 -------
214 namespace : `str`
215 The namespace. When combined with the `version` can uniquely
216 define this universe.
217 """
218 return self._namespace
220 def isCompatibleWith(self, other: DimensionUniverse) -> bool:
221 """Check compatibility between this `DimensionUniverse` and another.
223 Parameters
224 ----------
225 other : `DimensionUniverse`
226 The other `DimensionUniverse` to check for compatibility.
228 Returns
229 -------
230 results : `bool`
231 If the other `DimensionUniverse` is compatible with this one return
232 `True`, else `False`.
233 """
234 # Different namespaces mean that these universes cannot be compatible.
235 if self.namespace != other.namespace:
236 return False
237 if self.version != other.version:
238 _LOG.info(
239 "Universes share a namespace %r but have differing versions (%d != %d). "
240 " This could be okay but may be responsible for dimension errors later.",
241 self.namespace,
242 self.version,
243 other.version,
244 )
246 # For now assume compatibility if versions differ.
247 return True
249 def __repr__(self) -> str:
250 return f"DimensionUniverse({self._version}, {self._namespace})"
252 def __getitem__(self, name: str) -> DimensionElement:
253 return self._elements[name]
255 def __contains__(self, name: Any) -> bool:
256 return name in self._elements
258 def get(self, name: str, default: DimensionElement | None = None) -> DimensionElement | None:
259 """Return the `DimensionElement` with the given name or a default.
261 Parameters
262 ----------
263 name : `str`
264 Name of the element.
265 default : `DimensionElement`, optional
266 Element to return if the named one does not exist. Defaults to
267 `None`.
269 Returns
270 -------
271 element : `DimensionElement`
272 The named element.
273 """
274 return self._elements.get(name, default)
276 def getStaticElements(self) -> NamedValueAbstractSet[DimensionElement]:
277 """Return a set of all static elements in this universe.
279 Non-static elements that are created as needed may also exist, but
280 these are guaranteed to have no direct relationships to other elements
281 (though they may have spatial or temporal relationships).
283 Returns
284 -------
285 elements : `NamedValueAbstractSet` [ `DimensionElement` ]
286 A frozen set of `DimensionElement` instances.
287 """
288 return self._elements
290 def getStaticDimensions(self) -> NamedValueAbstractSet[Dimension]:
291 """Return a set of all static dimensions in this universe.
293 Non-static dimensions that are created as needed may also exist, but
294 these are guaranteed to have no direct relationships to other elements
295 (though they may have spatial or temporal relationships).
297 Returns
298 -------
299 dimensions : `NamedValueAbstractSet` [ `Dimension` ]
300 A frozen set of `Dimension` instances.
301 """
302 return self._dimensions
304 def getGovernorDimensions(self) -> NamedValueAbstractSet[GovernorDimension]:
305 """Return a set of all `GovernorDimension` instances in this universe.
307 Returns
308 -------
309 governors : `NamedValueAbstractSet` [ `GovernorDimension` ]
310 A frozen set of `GovernorDimension` instances.
311 """
312 return self.governor_dimensions
314 def getDatabaseElements(self) -> NamedValueAbstractSet[DatabaseDimensionElement]:
315 """Return set of all `DatabaseDimensionElement` instances in universe.
317 This does not include `GovernorDimension` instances, which are backed
318 by the database but do not inherit from `DatabaseDimensionElement`.
320 Returns
321 -------
322 elements : `NamedValueAbstractSet` [ `DatabaseDimensionElement` ]
323 A frozen set of `DatabaseDimensionElement` instances.
324 """
325 return self.database_elements
327 @property
328 def elements(self) -> NamedValueAbstractSet[DimensionElement]:
329 """All dimension elements defined in this universe."""
330 return self._elements
332 @property
333 def dimensions(self) -> NamedValueAbstractSet[Dimension]:
334 """All dimensions defined in this universe."""
335 return self._dimensions
337 @property
338 @cached_getter
339 def governor_dimensions(self) -> NamedValueAbstractSet[GovernorDimension]:
340 """All governor dimensions defined in this universe.
342 Governor dimensions serve as special required dependencies of other
343 dimensions, with special handling in dimension query expressions and
344 collection summaries. Governor dimension records are stored in the
345 database but the set of such values is expected to be small enough
346 for all values to be cached by all clients.
347 """
348 return NamedValueSet(d for d in self._dimensions if isinstance(d, GovernorDimension)).freeze()
350 @property
351 @cached_getter
352 def skypix_dimensions(self) -> NamedValueAbstractSet[SkyPixDimension]:
353 """All skypix dimensions defined in this universe.
355 Skypix dimension records are always generated on-the-fly rather than
356 stored in the database, and they always represent a tiling of the sky
357 with no overlaps.
358 """
359 result = NamedValueSet[SkyPixDimension]()
360 for system in self.skypix:
361 result.update(system)
362 return result.freeze()
364 @property
365 @cached_getter
366 def database_elements(self) -> NamedValueAbstractSet[DatabaseDimensionElement]:
367 """All dimension elements whose records are stored in the database,
368 except governor dimensions.
369 """
370 return NamedValueSet(d for d in self._elements if isinstance(d, DatabaseDimensionElement)).freeze()
372 @property
373 @cached_getter
374 def skypix(self) -> NamedValueAbstractSet[SkyPixSystem]:
375 """All skypix systems known to this universe.
377 (`NamedValueAbstractSet` [ `SkyPixSystem` ]).
378 """
379 return NamedValueSet(
380 [
381 family
382 for family in self._topology[TopologicalSpace.SPATIAL]
383 if isinstance(family, SkyPixSystem)
384 ]
385 ).freeze()
387 def getElementIndex(self, name: str) -> int:
388 """Return the position of the named dimension element.
390 The position is in this universe's sorting of all elements.
392 Parameters
393 ----------
394 name : `str`
395 Name of the element.
397 Returns
398 -------
399 index : `int`
400 Sorting index for this element.
401 """
402 return self._elementIndices[name]
404 def getDimensionIndex(self, name: str) -> int:
405 """Return the position of the named dimension.
407 This position is in this universe's sorting of all dimensions.
409 Parameters
410 ----------
411 name : `str`
412 Name of the dimension.
414 Returns
415 -------
416 index : `int`
417 Sorting index for this dimension.
419 Notes
420 -----
421 The dimension sort order for a universe is consistent with the element
422 order (all dimensions are elements), and either can be used to sort
423 dimensions if used consistently. But there are also some contexts in
424 which contiguous dimension-only indices are necessary or at least
425 desirable.
426 """
427 return self._dimensionIndices[name]
429 def conform(
430 self,
431 dimensions: Iterable[str] | str | DimensionGroup,
432 /,
433 ) -> DimensionGroup:
434 """Construct a dimension group from an iterable of dimension names.
436 Parameters
437 ----------
438 dimensions : `~collections.abc.Iterable` [ `str` ], `str`, or \
439 `DimensionGroup`
440 Dimensions that must be included in the returned group; their
441 dependencies will be as well.
443 Returns
444 -------
445 group : `DimensionGroup`
446 A `DimensionGroup` instance containing all given dimensions.
447 """
448 match dimensions:
449 case DimensionGroup():
450 if dimensions.universe is not self:
451 raise InconsistentUniverseError(
452 f"DimensionGroup {dimensions} belongs to a different universe."
453 f" Expected universe {self}, but got {dimensions.universe}."
454 )
455 return dimensions
456 case str() as name:
457 return self[name].minimal_group
458 case iterable:
459 return DimensionGroup(self, set(iterable))
461 @overload
462 def sorted(self, elements: Iterable[Dimension], *, reverse: bool = False) -> Sequence[Dimension]: ... 462 ↛ exitline 462 didn't return from function 'sorted' because
464 @overload
465 def sorted( 465 ↛ exitline 465 didn't return from function 'sorted' because
466 self, elements: Iterable[DimensionElement | str], *, reverse: bool = False
467 ) -> Sequence[DimensionElement]: ...
469 def sorted(self, elements: Iterable[Any], *, reverse: bool = False) -> list[Any]:
470 """Return a sorted version of the given iterable of dimension elements.
472 The universe's sort order is topological (an element's dependencies
473 precede it), with an unspecified (but deterministic) approach to
474 breaking ties.
476 Parameters
477 ----------
478 elements : `~collections.abc.Iterable` of `DimensionElement`
479 Elements to be sorted.
480 reverse : `bool`, optional
481 If `True`, sort in the opposite order.
483 Returns
484 -------
485 sorted : `~collections.abc.Sequence` [ `Dimension` or \
486 `DimensionElement` ]
487 A sorted sequence containing the same elements that were given.
488 """
489 s = set(elements)
490 result = [element for element in self._elements if element in s or element.name in s]
491 if reverse:
492 result.reverse()
493 return result
495 def get_elements_populated_by(self, dimension: Dimension) -> NamedValueAbstractSet[DimensionElement]:
496 """Return the set of `DimensionElement` objects whose
497 `~DimensionElement.populated_by` attribute is the given dimension.
499 Parameters
500 ----------
501 dimension : `Dimension`
502 The dimension of interest.
504 Returns
505 -------
506 populated_by : `NamedValueAbstractSet` [ `DimensionElement` ]
507 The set of elements who say they are populated by the given
508 dimension.
509 """
510 return self._populates[dimension.name]
512 @property
513 def empty(self) -> DimensionGroup:
514 """The `DimensionGroup` that contains no dimensions."""
515 return self._empty
517 @classmethod
518 def _unpickle(cls, version: int, namespace: str | None = None) -> DimensionUniverse:
519 """Return an unpickled dimension universe.
521 Callable used for unpickling.
523 For internal use only.
524 """
525 if namespace is None:
526 # Old pickled universe.
527 namespace = _DEFAULT_NAMESPACE
528 instance = cls._instances.get((version, namespace))
529 if instance is None:
530 raise pickle.UnpicklingError(
531 f"DimensionUniverse with version '{version}' and namespace {namespace!r} "
532 "not found. Note that DimensionUniverse objects are not "
533 "truly serialized; when using pickle to transfer them "
534 "between processes, an equivalent instance with the same "
535 "version must already exist in the receiving process."
536 )
537 return instance
539 def __reduce__(self) -> tuple:
540 return (self._unpickle, (self._version, self._namespace))
542 def __deepcopy__(self, memo: dict) -> DimensionUniverse:
543 # DimensionUniverse is recursively immutable; see note in @immutable
544 # decorator.
545 return self
547 # Class attributes below are shadowed by instance attributes, and are
548 # present just to hold the docstrings for those instance attributes.
550 commonSkyPix: SkyPixDimension
551 """The special skypix dimension that is used to relate all other spatial
552 dimensions in the `Registry` database (`SkyPixDimension`).
553 """
555 dimensionConfig: DimensionConfig
556 """The configuration used to create this Universe (`DimensionConfig`)."""
558 _cached_groups: ThreadSafeCache[frozenset[str], DimensionGroup]
560 _dimensions: NamedValueAbstractSet[Dimension]
562 _elements: NamedValueAbstractSet[DimensionElement]
564 _empty: DimensionGroup
566 _topology: Mapping[TopologicalSpace, NamedValueAbstractSet[TopologicalFamily]]
568 _dimensionIndices: dict[str, int]
570 _elementIndices: dict[str, int]
572 _populates: defaultdict[str, NamedValueSet[DimensionElement]]
574 _version: int
576 _namespace: str