Coverage for tests/test_templates.py: 11%
189 statements
« prev ^ index » next coverage.py v7.14.1, created at 2026-05-30 01:35 -0700
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This software is dual licensed under the GNU General Public License and also
10# under a 3-clause BSD license. Recipients may choose which of these licenses
11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
12# respectively. If you choose the GPL option then the following text applies
13# (but note that there is still no warranty even if you opt for BSD instead):
14#
15# This program is free software: you can redistribute it and/or modify
16# it under the terms of the GNU General Public License as published by
17# the Free Software Foundation, either version 3 of the License, or
18# (at your option) any later version.
19#
20# This program is distributed in the hope that it will be useful,
21# but WITHOUT ANY WARRANTY; without even the implied warranty of
22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
23# GNU General Public License for more details.
24#
25# You should have received a copy of the GNU General Public License
26# along with this program. If not, see <http://www.gnu.org/licenses/>.
28"""Test file name templating."""
30import os.path
31import unittest
32import uuid
34from lsst.daf.butler import (
35 DataCoordinate,
36 DatasetId,
37 DatasetRef,
38 DatasetType,
39 DimensionUniverse,
40 StorageClass,
41)
42from lsst.daf.butler.datastore.file_templates import (
43 FileTemplate,
44 FileTemplates,
45 FileTemplatesConfig,
46 FileTemplateValidationError,
47)
49TESTDIR = os.path.abspath(os.path.dirname(__file__))
51PlaceHolder = StorageClass("PlaceHolder")
53REFUUID = DatasetId(int=uuid.uuid4().int)
56class TestFileTemplates(unittest.TestCase):
57 """Test creation of paths from templates."""
59 def makeDatasetRef(
60 self, datasetTypeName, dataId=None, storageClassName="DefaultStorageClass", run="run2", conform=True
61 ):
62 """Make a simple DatasetRef"""
63 if dataId is None:
64 dataId = self.dataId
65 if "physical_filter" in dataId and "band" not in dataId:
66 dataId["band"] = "b" # Add fake band.
67 dimensions = self.universe.conform(dataId.keys())
68 dataId = DataCoordinate.standardize(dataId, dimensions=dimensions)
70 # Pretend we have a parent if this looks like a composite
71 compositeName, componentName = DatasetType.splitDatasetTypeName(datasetTypeName)
72 parentStorageClass = PlaceHolder if componentName else None
74 datasetType = DatasetType(
75 datasetTypeName,
76 dimensions,
77 StorageClass(storageClassName),
78 parentStorageClass=parentStorageClass,
79 )
80 return DatasetRef(datasetType, dataId, id=REFUUID, run=run, conform=conform)
82 def setUp(self):
83 self.universe = DimensionUniverse()
84 self.dataId = {
85 "instrument": "dummy",
86 "visit": 52,
87 "physical_filter": "Most Amazing U Filter Ever",
88 "day_obs": 20200101,
89 "group": "2025-08-07>08:25:27.100",
90 }
92 def assertTemplate(self, template, answer, ref):
93 fileTmpl = FileTemplate(template)
94 path = fileTmpl.format(ref)
95 self.assertEqual(path, answer)
97 def testBasic(self):
98 tmplstr = "{run}/{datasetType}/{visit:05d}/{physical_filter}"
99 self.assertTemplate(
100 tmplstr,
101 "run2/calexp/00052/Most_Amazing_U_Filter_Ever",
102 self.makeDatasetRef("calexp"),
103 )
104 tmplstr = "{run}/{datasetType}/{visit:05d}/{physical_filter}-trail"
105 self.assertTemplate(
106 tmplstr,
107 "run2/calexp/00052/Most_Amazing_U_Filter_Ever-trail",
108 self.makeDatasetRef("calexp"),
109 )
111 tmplstr = "{run}/{datasetType}/{visit:05d}/{physical_filter}-trail-{run}"
112 self.assertTemplate(
113 tmplstr,
114 "run2/calexp/00052/Most_Amazing_U_Filter_Ever-trail-run2",
115 self.makeDatasetRef("calexp"),
116 )
117 self.assertTemplate(
118 tmplstr,
119 "run_2/calexp/00052/Most_Amazing_U_Filter_Ever-trail-run_2",
120 self.makeDatasetRef("calexp", run="run/2"),
121 )
123 # Check that the id is sufficient without any other information.
124 self.assertTemplate("{id}", str(REFUUID), self.makeDatasetRef("calexp", run="run2"))
126 self.assertTemplate("{run}/{id}", f"run2/{str(REFUUID)}", self.makeDatasetRef("calexp", run="run2"))
128 self.assertTemplate(
129 "fixed/{id}",
130 f"fixed/{str(REFUUID)}",
131 self.makeDatasetRef("calexp", run="run2"),
132 )
134 self.assertTemplate(
135 "fixed/{id}_{physical_filter}",
136 f"fixed/{str(REFUUID)}_Most_Amazing_U_Filter_Ever",
137 self.makeDatasetRef("calexp", run="run2"),
138 )
140 # Retain any "/" in run
141 tmplstr = "{run:/}/{datasetType}/{visit:05d}/{physical_filter}-trail-{run}"
142 self.assertTemplate(
143 tmplstr,
144 "run/2/calexp/00052/Most_Amazing_U_Filter_Ever-trail-run_2",
145 self.makeDatasetRef("calexp", run="run/2"),
146 )
148 # Check that "." are replaced in the file basename, but not directory.
149 dataId = {"instrument": "dummy", "visit": 52, "physical_filter": "g.10", "day_obs": 20250101}
150 self.assertTemplate(
151 tmplstr,
152 "run.2/calexp/00052/g_10-trail-run_2",
153 self.makeDatasetRef("calexp", run="run.2", dataId=dataId),
154 )
156 # Check that type conversion is applied
157 dataId = {"instrument": "dummy", "day_obs": 20250203}
158 tmplstr = "{run}/{datasetType}/{day_obs!s:.4s}"
159 self.assertTemplate(
160 tmplstr,
161 "run3/calexp/2025",
162 self.makeDatasetRef("calexp", run="run3", dataId=dataId),
163 )
165 tmplstr = "{run}/{datasetType}/{group}"
166 self.assertTemplate(
167 tmplstr,
168 "run2/calexp/2025-08-07_08_25_27_100",
169 self.makeDatasetRef("calexp"),
170 )
172 with self.assertRaises(FileTemplateValidationError):
173 FileTemplate("no fields at all")
175 with self.assertRaises(FileTemplateValidationError):
176 FileTemplate("{visit}")
178 with self.assertRaises(FileTemplateValidationError):
179 FileTemplate("{run}_{datasetType}")
181 with self.assertRaises(FileTemplateValidationError):
182 FileTemplate("{id}/fixed")
184 with self.assertRaises(FileTemplateValidationError):
185 FileTemplate("{run}/../{datasetType}_{visit}")
187 def testAlternates(self):
188 tmplstr = "{run}/{datasetType}/{visit:05d}/{physical_filter|day_obs}_{day_obs|physical_filter}"
189 self.assertTemplate(
190 tmplstr,
191 "run2/calexp/00052/Most_Amazing_U_Filter_Ever_20200101",
192 self.makeDatasetRef("calexp"),
193 )
194 tmplstr = "{run}/{datasetType}/{exposure|visit:05d}/{physical_filter|day_obs}_{tract|exposure:?}"
195 self.assertTemplate(
196 tmplstr,
197 "run2/calexp/00052/Most_Amazing_U_Filter_Ever",
198 self.makeDatasetRef("calexp"),
199 )
201 def testRunOrCollectionNeeded(self):
202 tmplstr = "{datasetType}/{visit:05d}/{physical_filter}"
203 with self.assertRaises(FileTemplateValidationError):
204 self.assertTemplate(tmplstr, "run2/calexp/00052/U", self.makeDatasetRef("calexp"))
206 def testNoRecord(self):
207 # Attaching records is not possible in this test code but we can check
208 # that a missing record when a metadata entry has been requested
209 # does fail.
210 tmplstr = "{run}/{datasetType}/{visit.name}/{physical_filter}"
211 with self.assertRaises(RuntimeError) as cm:
212 self.assertTemplate(tmplstr, "", self.makeDatasetRef("calexp"))
213 self.assertIn("No metadata", str(cm.exception))
215 def testOptional(self):
216 """Optional units in templates."""
217 ref = self.makeDatasetRef("calexp")
218 tmplstr = "{run}/{datasetType}/v{visit:05d}_f{physical_filter:?}_{skypix:?}"
219 self.assertTemplate(
220 tmplstr,
221 "run2/calexp/v00052_fMost_Amazing_U_Filter_Ever",
222 self.makeDatasetRef("calexp"),
223 )
225 du = {"visit": 48, "tract": 265, "skymap": "big", "instrument": "dummy", "htm7": 12345}
226 self.assertTemplate(tmplstr, "run2/calexpT/v00048_12345", self.makeDatasetRef("calexpT", du))
228 # Ensure that this returns a relative path even if the first field
229 # is optional
230 tmplstr = "{run}/{tract:?}/{visit:?}/f{physical_filter}"
231 self.assertTemplate(tmplstr, "run2/52/fMost_Amazing_U_Filter_Ever", ref)
233 # Ensure that // from optionals are converted to singles
234 tmplstr = "{run}/{datasetType}/{patch:?}/{tract:?}/f{physical_filter}"
235 self.assertTemplate(tmplstr, "run2/calexp/fMost_Amazing_U_Filter_Ever", ref)
237 # Optionals with some text between fields
238 tmplstr = "{run}/{datasetType}/p{patch:?}_t{tract:?}/f{physical_filter}"
239 self.assertTemplate(tmplstr, "run2/calexp/p/fMost_Amazing_U_Filter_Ever", ref)
240 tmplstr = "{run}/{datasetType}/p{patch:?}_t{visit:04d?}/f{physical_filter}"
241 self.assertTemplate(tmplstr, "run2/calexp/p_t0052/fMost_Amazing_U_Filter_Ever", ref)
243 def testComponent(self):
244 """Test handling of components in templates."""
245 refMetricOutput = self.makeDatasetRef("metric.output")
246 refMetric = self.makeDatasetRef("metric")
247 refMaskedImage = self.makeDatasetRef("calexp.maskedimage.variance")
248 refWcs = self.makeDatasetRef("calexp.wcs")
250 tmplstr = "{run}_c_{component}_v{visit}"
251 self.assertTemplate(tmplstr, "run2_c_output_v52", refMetricOutput)
253 # We want this template to have both a directory and basename, to
254 # test that the right parts of the output are replaced.
255 tmplstr = "{component:?}/{run}_{component:?}_{visit}"
256 self.assertTemplate(tmplstr, "run2_52", refMetric)
257 self.assertTemplate(tmplstr, "output/run2_output_52", refMetricOutput)
258 self.assertTemplate(tmplstr, "maskedimage.variance/run2_maskedimage_variance_52", refMaskedImage)
259 self.assertTemplate(tmplstr, "output/run2_output_52", refMetricOutput)
261 # Providing a component but not using it
262 tmplstr = "{run}/{datasetType}/v{visit:05d}"
263 with self.assertRaises(KeyError):
264 self.assertTemplate(tmplstr, "", refWcs)
266 def testFields(self):
267 # Template, mandatory fields, optional non-special fields,
268 # special fields, optional special fields
269 testData = (
270 (
271 "{run}/{datasetType}/{visit:05d}/{physical_filter}-trail",
272 {"visit", "physical_filter"},
273 set(),
274 {"run", "datasetType"},
275 set(),
276 ),
277 (
278 "{run}/{component:?}_{visit}",
279 {"visit"},
280 set(),
281 {"run"},
282 {"component"},
283 ),
284 (
285 "{run}/{component:?}_{visit:?}_{physical_filter}_{instrument}_{datasetType}",
286 {"physical_filter", "instrument"},
287 {"visit"},
288 {"run", "datasetType"},
289 {"component"},
290 ),
291 )
292 for tmplstr, mandatory, optional, special, optionalSpecial in testData:
293 with self.subTest(template=tmplstr):
294 tmpl = FileTemplate(tmplstr)
295 fields = tmpl.fields()
296 self.assertEqual(fields, mandatory)
297 fields = tmpl.fields(optionals=True)
298 self.assertEqual(fields, mandatory | optional)
299 fields = tmpl.fields(specials=True)
300 self.assertEqual(fields, mandatory | special)
301 fields = tmpl.fields(specials=True, optionals=True)
302 self.assertEqual(fields, mandatory | special | optional | optionalSpecial)
304 def testSimpleConfig(self):
305 """Test reading from config file"""
306 configRoot = os.path.join(TESTDIR, "config", "templates")
307 config1 = FileTemplatesConfig(os.path.join(configRoot, "templates-nodefault.yaml"))
308 templates = FileTemplates(config1, universe=self.universe)
309 ref = self.makeDatasetRef("calexp")
310 tmpl = templates.getTemplate(ref)
311 self.assertIsInstance(tmpl, FileTemplate)
313 # This config file should not allow defaulting
314 ref2 = self.makeDatasetRef("unknown")
315 with self.assertRaises(KeyError):
316 templates.getTemplate(ref2)
318 # This should fall through the datasetTypeName check and use
319 # StorageClass instead
320 ref3 = self.makeDatasetRef("unknown2", storageClassName="StorageClassX")
321 tmplSc = templates.getTemplate(ref3)
322 self.assertIsInstance(tmplSc, FileTemplate)
324 # Try with a component: one with defined formatter and one without
325 refWcs = self.makeDatasetRef("calexp.wcs")
326 refImage = self.makeDatasetRef("calexp.image")
327 tmplCalexp = templates.getTemplate(ref)
328 tmplWcs = templates.getTemplate(refWcs) # Should be special
329 tmpl_image = templates.getTemplate(refImage)
330 self.assertIsInstance(tmplCalexp, FileTemplate)
331 self.assertIsInstance(tmpl_image, FileTemplate)
332 self.assertIsInstance(tmplWcs, FileTemplate)
333 self.assertEqual(tmplCalexp, tmpl_image)
334 self.assertNotEqual(tmplCalexp, tmplWcs)
336 # Check dimensions lookup order.
337 # The order should be: dataset type name, dimension, storage class
338 # This one will not match name but might match storage class.
339 # It should match dimensions
340 refDims = self.makeDatasetRef(
341 "nomatch", dataId={"instrument": "LSST", "physical_filter": "z"}, storageClassName="StorageClassX"
342 )
343 tmplDims = templates.getTemplate(refDims)
344 self.assertIsInstance(tmplDims, FileTemplate)
345 self.assertNotEqual(tmplDims, tmplSc)
347 # Test that instrument overrides retrieve specialist templates
348 refPvi = self.makeDatasetRef("pvi")
349 refPviHsc = self.makeDatasetRef("pvi", dataId={"instrument": "HSC", "physical_filter": "z"})
350 refPviLsst = self.makeDatasetRef("pvi", dataId={"instrument": "LSST", "physical_filter": "z"})
352 tmplPvi = templates.getTemplate(refPvi)
353 tmplPviHsc = templates.getTemplate(refPviHsc)
354 tmplPviLsst = templates.getTemplate(refPviLsst)
355 self.assertEqual(tmplPvi, tmplPviLsst)
356 self.assertNotEqual(tmplPvi, tmplPviHsc)
358 # Have instrument match and dimensions look up with no name match
359 refNoPviHsc = self.makeDatasetRef(
360 "pvix", dataId={"instrument": "HSC", "physical_filter": "z"}, storageClassName="StorageClassX"
361 )
362 tmplNoPviHsc = templates.getTemplate(refNoPviHsc)
363 self.assertNotEqual(tmplNoPviHsc, tmplDims)
364 self.assertNotEqual(tmplNoPviHsc, tmplPviHsc)
366 # Format config file with defaulting
367 config2 = FileTemplatesConfig(os.path.join(configRoot, "templates-withdefault.yaml"))
368 templates = FileTemplates(config2, universe=self.universe)
369 tmpl = templates.getTemplate(ref2)
370 self.assertIsInstance(tmpl, FileTemplate)
372 # Format config file with bad format string
373 with self.assertRaises(FileTemplateValidationError):
374 FileTemplates(os.path.join(configRoot, "templates-bad.yaml"), universe=self.universe)
376 # Config file with no defaulting mentioned
377 config3 = os.path.join(configRoot, "templates-nodefault2.yaml")
378 templates = FileTemplates(config3, universe=self.universe)
379 with self.assertRaises(KeyError):
380 templates.getTemplate(ref2)
382 # Try again but specify a default in the constructor
383 default = "{run}/{datasetType}/{physical_filter}"
384 templates = FileTemplates(config3, default=default, universe=self.universe)
385 tmpl = templates.getTemplate(ref2)
386 self.assertEqual(tmpl.template, default)
388 def testValidation(self):
389 configRoot = os.path.join(TESTDIR, "config", "templates")
390 config1 = FileTemplatesConfig(os.path.join(configRoot, "templates-nodefault.yaml"))
391 templates = FileTemplates(config1, universe=self.universe)
393 entities = {}
394 entities["calexp"] = self.makeDatasetRef(
395 "calexp",
396 storageClassName="StorageClassX",
397 dataId={"instrument": "dummy", "physical_filter": "i", "visit": 52},
398 )
400 with self.assertLogs(level="WARNING") as cm:
401 templates.validateTemplates(entities.values(), logFailures=True)
402 self.assertIn("Unchecked keys", cm.output[0])
403 self.assertIn("StorageClassX", cm.output[0])
405 entities["pvi"] = self.makeDatasetRef(
406 "pvi", storageClassName="StorageClassX", dataId={"instrument": "dummy", "physical_filter": "i"}
407 )
408 entities["StorageClassX"] = self.makeDatasetRef(
409 "storageClass", storageClassName="StorageClassX", dataId={"instrument": "dummy", "visit": 2}
410 )
411 entities["calexp.wcs"] = self.makeDatasetRef(
412 "calexp.wcs",
413 storageClassName="StorageClassX",
414 dataId={"instrument": "dummy", "physical_filter": "i", "visit": 23},
415 conform=False,
416 )
418 entities["instrument+physical_filter"] = self.makeDatasetRef(
419 "filter_inst",
420 storageClassName="StorageClassX",
421 dataId={"physical_filter": "i", "instrument": "SCUBA"},
422 )
423 entities["hsc+pvi"] = self.makeDatasetRef(
424 "pvi", storageClassName="StorageClassX", dataId={"physical_filter": "i", "instrument": "HSC"}
425 )
427 entities["hsc+instrument+physical_filter"] = self.makeDatasetRef(
428 "filter_inst",
429 storageClassName="StorageClassX",
430 dataId={"physical_filter": "i", "instrument": "HSC"},
431 )
433 entities["metric6"] = self.makeDatasetRef(
434 "filter_inst",
435 storageClassName="Integer",
436 dataId={"physical_filter": "i", "instrument": "HSC"},
437 )
439 templates.validateTemplates(entities.values(), logFailures=True)
441 # Rerun but with a failure
442 entities["pvi"] = self.makeDatasetRef("pvi", storageClassName="StorageClassX", dataId={"band": "i"})
443 with self.assertRaises(FileTemplateValidationError):
444 with self.assertLogs(level="FATAL"):
445 templates.validateTemplates(entities.values(), logFailures=True)
448if __name__ == "__main__":
449 unittest.main()