Coverage for python / felis / cli.py: 39%
199 statements
« prev ^ index » next coverage.py v7.14.0, created at 2026-05-21 01:22 -0700
1"""Click command line interface."""
3# This file is part of felis.
4#
5# Developed for the LSST Data Management System.
6# This product includes software developed by the LSST Project
7# (https://www.lsst.org).
8# See the COPYRIGHT file at the top-level directory of this distribution
9# for details of code ownership.
10#
11# This program is free software: you can redistribute it and/or modify
12# it under the terms of the GNU General Public License as published by
13# the Free Software Foundation, either version 3 of the License, or
14# (at your option) any later version.
15#
16# This program is distributed in the hope that it will be useful,
17# but WITHOUT ANY WARRANTY; without even the implied warranty of
18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19# GNU General Public License for more details.
20#
21# You should have received a copy of the GNU General Public License
22# along with this program. If not, see <https://www.gnu.org/licenses/>.
24from __future__ import annotations
26import logging
27from collections.abc import Iterable
28from typing import IO
30import click
31from pydantic import ValidationError
33from . import __version__
34from .datamodel import Schema
35from .db.database_context import create_database_context
36from .diff import DatabaseDiff, FormattedSchemaDiff, SchemaDiff
37from .metadata import create_metadata
38from .tap_schema import DataLoader, MetadataInserter, TableManager
40__all__ = ["cli"]
42logger = logging.getLogger(__name__)
44loglevel_choices = ["CRITICAL", "FATAL", "ERROR", "WARNING", "INFO", "DEBUG"]
47@click.group()
48@click.version_option(__version__)
49@click.option(
50 "--log-level",
51 type=click.Choice(loglevel_choices),
52 envvar="FELIS_LOGLEVEL",
53 help="Felis log level",
54 default=logging.getLevelName(logging.INFO),
55)
56@click.option(
57 "--log-file",
58 type=click.Path(),
59 envvar="FELIS_LOGFILE",
60 help="Felis log file path",
61)
62@click.option(
63 "--id-generation/--no-id-generation",
64 is_flag=True,
65 help="Generate IDs for all objects that do not have them",
66 default=True,
67)
68@click.option(
69 "--column-ref-index-increment",
70 type=int,
71 help="Automatically set 'tap:column_index' on column references, using the specified increment "
72 "(must be at least 1)",
73 default=None,
74)
75@click.pass_context
76def cli(
77 ctx: click.Context,
78 log_level: str,
79 log_file: str | None,
80 id_generation: bool,
81 column_ref_index_increment: int | None,
82) -> None:
83 """Felis command line tools"""
84 ctx.ensure_object(dict)
86 # Configure logging on the felis package logger directly.
87 # Do not use basicConfig() as it is a no-op when handlers already exist
88 # (e.g., when running under pytest).
89 felis_logger = logging.getLogger("felis")
90 felis_logger.setLevel(log_level)
91 if log_file:
92 handler: logging.Handler = logging.FileHandler(log_file)
93 else:
94 handler = logging.StreamHandler()
95 handler.setLevel(log_level)
96 handler.setFormatter(logging.Formatter("%(levelname)s:%(name)s:%(message)s"))
97 felis_logger.addHandler(handler)
99 # Configure ID generation (flag can only turn it off)
100 ctx.obj["id_generation"] = id_generation
101 if ctx.obj["id_generation"]:
102 logger.info("ID generation is enabled")
103 else:
104 logger.info("ID generation is disabled")
106 # Configure automatic indexing of column references (optional)
107 if column_ref_index_increment is not None and column_ref_index_increment < 1:
108 raise click.ClickException("column_ref_index_increment must be at least 1")
109 ctx.obj["column_ref_index_increment"] = column_ref_index_increment
110 if ctx.obj["column_ref_index_increment"] is not None:
111 logger.info(
112 "Automatic indexing of column references is enabled with increment %s",
113 ctx.obj["column_ref_index_increment"],
114 )
117@cli.command("create", help="Create database objects from the Felis file")
118@click.option("--engine-url", envvar="FELIS_ENGINE_URL", help="SQLAlchemy Engine URL", default="sqlite://")
119@click.option("--schema-name", help="Alternate schema name to override Felis file")
120@click.option(
121 "--initialize",
122 is_flag=True,
123 help="Create the schema in the database if it does not exist (error if already exists)",
124)
125@click.option(
126 "--drop", is_flag=True, help="Drop schema if it already exists in the database (implies --initialize)"
127)
128@click.option("--echo", is_flag=True, help="Echo database commands as they are executed")
129@click.option("--dry-run", is_flag=True, help="Dry run only to print out commands instead of executing")
130@click.option(
131 "--output-file", "-o", type=click.File(mode="w"), help="Write SQL commands to a file instead of executing"
132)
133@click.option("--ignore-constraints", is_flag=True, help="Ignore constraints when creating tables")
134@click.option("--skip-indexes", is_flag=True, help="Skip creating indexes when building metadata")
135@click.argument("uri", type=str)
136@click.pass_context
137def create(
138 ctx: click.Context,
139 engine_url: str,
140 schema_name: str | None,
141 initialize: bool,
142 drop: bool,
143 echo: bool,
144 dry_run: bool,
145 output_file: IO[str] | None,
146 ignore_constraints: bool,
147 skip_indexes: bool,
148 uri: str,
149) -> None:
150 """Create database objects from the Felis file.
152 Parameters
153 ----------
154 engine_url
155 SQLAlchemy Engine URL.
156 schema_name
157 Alternate schema name to override Felis file.
158 initialize
159 Create the schema in the database if it does not exist.
160 drop
161 Drop schema if it already exists in the database.
162 echo
163 Echo database commands as they are executed.
164 dry_run
165 Dry run only to print out commands instead of executing.
166 output_file
167 Write SQL commands to a file instead of executing.
168 ignore_constraints
169 Ignore constraints when creating tables.
170 skip_indexes
171 Skip creating indexes when building metadata.
172 uri
173 URI of Felis file to read.
174 """
175 try:
176 schema = Schema.from_uri(uri, context={"id_generation": ctx.obj["id_generation"]})
178 metadata = create_metadata(
179 schema,
180 schema_name=schema_name,
181 ignore_constraints=ignore_constraints,
182 skip_indexes=skip_indexes,
183 engine_url=engine_url,
184 )
186 with create_database_context(
187 engine_url,
188 metadata,
189 echo=echo,
190 dry_run=dry_run,
191 output_file=output_file,
192 ) as db_ctx:
193 if drop and initialize:
194 raise ValueError("Cannot drop and initialize schema at the same time")
196 if drop:
197 logger.debug("Dropping schema if it exists")
198 db_ctx.drop()
199 initialize = True # If schema is dropped, it needs to be recreated.
201 if initialize:
202 logger.debug("Creating schema if not exists")
203 db_ctx.initialize()
205 db_ctx.create_all()
207 except Exception as e:
208 logger.exception(e)
209 raise click.ClickException(str(e))
212@cli.command("create-indexes", help="Create database indexes defined in the Felis file")
213@click.option("--engine-url", envvar="FELIS_ENGINE_URL", help="SQLAlchemy Engine URL", default="sqlite://")
214@click.option("--schema-name", help="Alternate schema name to override Felis file")
215@click.argument("uri", type=str)
216@click.pass_context
217def create_indexes(
218 ctx: click.Context,
219 engine_url: str,
220 schema_name: str | None,
221 uri: str,
222) -> None:
223 """Create indexes from a Felis YAML file in a target database.
225 Parameters
226 ----------
227 engine_url
228 SQLAlchemy Engine URL.
229 file
230 Felis file to read.
231 """
232 try:
233 schema = Schema.from_uri(uri, context={"id_generation": ctx.obj["id_generation"]})
235 metadata = create_metadata(schema, schema_name=schema_name, engine_url=engine_url)
237 with create_database_context(engine_url, metadata) as db_ctx:
238 db_ctx.create_indexes()
239 except Exception as e:
240 logger.exception(e)
241 raise click.ClickException("Error creating indexes: " + str(e))
244@cli.command("drop-indexes", help="Drop database indexes defined in the Felis file")
245@click.option("--engine-url", envvar="FELIS_ENGINE_URL", help="SQLAlchemy Engine URL", default="sqlite://")
246@click.option("--schema-name", help="Alternate schema name to override Felis file")
247@click.argument("uri", type=str)
248@click.pass_context
249def drop_indexes(
250 ctx: click.Context,
251 engine_url: str,
252 schema_name: str | None,
253 uri: str,
254) -> None:
255 """Drop indexes from a Felis YAML file in a target database.
257 Parameters
258 ----------
259 engine_url
260 SQLAlchemy Engine URL.
261 schema-name
262 Alternate schema name to override Felis file.
263 file
264 Felis file to read.
265 """
266 try:
267 schema = Schema.from_uri(uri, context={"id_generation": ctx.obj["id_generation"]})
269 metadata = create_metadata(schema, schema_name=schema_name, engine_url=engine_url)
270 with create_database_context(engine_url, metadata) as db:
271 db.drop_indexes()
272 except Exception as e:
273 logger.exception(e)
274 raise click.ClickException("Error dropping indexes: " + str(e))
277@cli.command("load-tap-schema", help="Load metadata from a Felis file into a TAP_SCHEMA database")
278@click.option("--engine-url", envvar="FELIS_ENGINE_URL", help="SQLAlchemy Engine URL")
279@click.option(
280 "--tap-schema-name", "-n", help="Name of the TAP_SCHEMA schema in the database (default: TAP_SCHEMA)"
281)
282@click.option(
283 "--tap-tables-postfix",
284 "-p",
285 help="Postfix which is applied to standard TAP_SCHEMA table names",
286 default="",
287)
288@click.option("--tap-schema-index", "-i", type=int, help="TAP_SCHEMA index of the schema in this environment")
289@click.option("--dry-run", "-D", is_flag=True, help="Execute dry run only. Does not insert any data.")
290@click.option("--echo", "-e", is_flag=True, help="Print out the generated insert statements to stdout")
291@click.option(
292 "--output-file", "-o", type=click.File(mode="w"), help="Write SQL commands to a file instead of executing"
293)
294@click.option(
295 "--force-unbounded-arraysize",
296 is_flag=True,
297 help="Use unbounded arraysize by default for all variable length string columns"
298 ", e.g., ``votable:arraysize: *`` (workaround for astropy bug #18099)",
299) # DM-50899: Variable-length bounded strings are not handled correctly in astropy
300@click.option(
301 "--unique-keys",
302 "-u",
303 is_flag=True,
304 help="Generate unique key_id values for keys and key_columns tables by prepending the schema name",
305 default=False,
306)
307@click.argument("uri", type=str)
308@click.pass_context
309def load_tap_schema(
310 ctx: click.Context,
311 engine_url: str,
312 tap_schema_name: str,
313 tap_tables_postfix: str,
314 tap_schema_index: int,
315 dry_run: bool,
316 echo: bool,
317 output_file: IO[str] | None,
318 force_unbounded_arraysize: bool,
319 unique_keys: bool,
320 uri: str,
321) -> None:
322 """Load TAP metadata from a Felis file.
324 Parameters
325 ----------
326 engine_url
327 SQLAlchemy Engine URL.
328 tap_tables_postfix
329 Postfix which is applied to standard TAP_SCHEMA table names.
330 tap_schema_index
331 TAP_SCHEMA index of the schema in this environment.
332 dry_run
333 Execute dry run only. Does not insert any data.
334 echo
335 Print out the generated insert statements to stdout.
336 output_file
337 Output file for writing generated SQL.
338 file
339 Felis file to read.
341 Notes
342 -----
343 The TAP_SCHEMA database must already exist or the command will fail. This
344 command will not initialize the TAP_SCHEMA tables.
345 """
346 # Create TableManager with automatic dialect detection
347 mgr = TableManager(
348 engine_url=engine_url,
349 schema_name=tap_schema_name,
350 table_name_postfix=tap_tables_postfix,
351 )
353 # Create DatabaseContext using TableManager's metadata
354 with create_database_context(
355 engine_url, mgr.metadata, echo=echo, dry_run=dry_run, output_file=output_file
356 ) as db_ctx:
357 schema = Schema.from_uri(
358 uri,
359 context={
360 "id_generation": ctx.obj["id_generation"],
361 "column_ref_index_increment": ctx.obj["column_ref_index_increment"],
362 "force_unbounded_arraysize": force_unbounded_arraysize,
363 },
364 )
366 DataLoader(
367 schema,
368 mgr,
369 db_context=db_ctx,
370 tap_schema_index=tap_schema_index,
371 dry_run=dry_run,
372 print_sql=echo,
373 output_file=output_file,
374 unique_keys=unique_keys,
375 ).load()
378@cli.command("init-tap-schema", help="Initialize a standard TAP_SCHEMA database")
379@click.option("--engine-url", envvar="FELIS_ENGINE_URL", help="SQLAlchemy Engine URL", required=True)
380@click.option("--tap-schema-name", help="Name of the TAP_SCHEMA schema in the database")
381@click.option(
382 "--extensions",
383 type=str,
384 default=None,
385 help=(
386 "Optional path to extensions YAML file (system path or resource:// URI). "
387 "If not provided, no extensions will be applied. "
388 "Example (default packaged extensions): "
389 "--extensions resource://felis/config/tap_schema/tap_schema_extensions.yaml"
390 ),
391)
392@click.option(
393 "--tap-tables-postfix", help="Postfix which is applied to standard TAP_SCHEMA table names", default=""
394)
395@click.option(
396 "--insert-metadata/--no-insert-metadata",
397 is_flag=True,
398 help="Insert metadata describing TAP_SCHEMA itself",
399 default=True,
400)
401@click.pass_context
402def init_tap_schema(
403 ctx: click.Context,
404 engine_url: str,
405 tap_schema_name: str,
406 extensions: str | None,
407 tap_tables_postfix: str,
408 insert_metadata: bool,
409) -> None:
410 """Initialize a standard TAP_SCHEMA database.
412 Parameters
413 ----------
414 engine_url
415 SQLAlchemy Engine URL.
416 tap_schema_name
417 Name of the TAP_SCHEMA schema in the database.
418 extensions
419 Extensions YAML file.
420 tap_tables_postfix
421 Postfix which is applied to standard TAP_SCHEMA table names.
422 insert_metadata
423 Insert metadata describing TAP_SCHEMA itself.
424 If set to False, only the TAP_SCHEMA tables will be created, but no
425 metadata will be inserted.
426 """
427 # Create TableManager with automatic dialect detection
428 mgr = TableManager(
429 engine_url=engine_url,
430 schema_name=tap_schema_name,
431 table_name_postfix=tap_tables_postfix,
432 extensions_path=extensions,
433 )
435 # Create DatabaseContext using TableManager's metadata
436 with create_database_context(engine_url, mgr.metadata) as db_ctx:
437 mgr.initialize_database(db_context=db_ctx)
438 if insert_metadata:
439 MetadataInserter(mgr, db_context=db_ctx).insert_metadata()
442@cli.command("validate", help="Validate one or more Felis YAML files")
443@click.option(
444 "--check-description", is_flag=True, help="Check that all objects have a description", default=False
445)
446@click.option(
447 "--check-redundant-datatypes", is_flag=True, help="Check for redundant datatype overrides", default=False
448)
449@click.option(
450 "--check-tap-table-indexes",
451 is_flag=True,
452 help="Check that every table has a unique TAP table index",
453 default=False,
454)
455@click.option(
456 "--check-tap-principal",
457 is_flag=True,
458 help="Check that at least one column per table is flagged as TAP principal",
459 default=False,
460)
461@click.argument("uris", nargs=-1, type=str)
462@click.pass_context
463def validate(
464 ctx: click.Context,
465 check_description: bool,
466 check_redundant_datatypes: bool,
467 check_tap_table_indexes: bool,
468 check_tap_principal: bool,
469 uris: Iterable[str],
470) -> None:
471 """Validate one or more felis YAML files.
473 Parameters
474 ----------
475 check_description
476 Check that all objects have a valid description.
477 check_redundant_datatypes
478 Check for redundant type overrides.
479 check_tap_table_indexes
480 Check that every table has a unique TAP table index.
481 check_tap_principal
482 Check that at least one column per table is flagged as TAP principal.
483 uris
484 A list of URIs representing the Felis YAML files to validate. These can
485 be relative or absolute file paths or a URI with a supported scheme
486 such as ``resource://`` for package resources.
488 Raises
489 ------
490 click.exceptions.Exit
491 Raised if any validation errors are found. The ``ValidationError``
492 which is thrown when a schema fails to validate will be logged as an
493 error message.
495 Notes
496 -----
497 All of the ``check`` flags are turned off by default and represent
498 optional validations controlled by the Pydantic context.
499 """
500 rc = 0
501 for uri in uris:
502 logger.info("Validating schema at '%s'", uri)
503 try:
504 Schema.from_uri(
505 uri,
506 context={
507 "check_description": check_description,
508 "check_redundant_datatypes": check_redundant_datatypes,
509 "check_tap_table_indexes": check_tap_table_indexes,
510 "check_tap_principal": check_tap_principal,
511 "id_generation": ctx.obj["id_generation"],
512 "column_ref_index_increment": ctx.obj["column_ref_index_increment"],
513 },
514 )
515 logger.info("Successfully validated schema at '%s'", uri)
516 except ValidationError as e:
517 logger.error(e)
518 rc = 1
519 if rc:
520 raise click.exceptions.Exit(rc)
523@cli.command(
524 "diff",
525 help="""
526 Compare two schemas or a schema and a database for changes
528 Examples:
530 felis diff schema1.yaml schema2.yaml
532 felis diff -c alembic schema1.yaml schema2.yaml
534 felis diff --engine-url sqlite:///test.db schema.yaml
535 """,
536)
537@click.option("--engine-url", envvar="FELIS_ENGINE_URL", help="SQLAlchemy Engine URL")
538@click.option(
539 "-c",
540 "--comparator",
541 type=click.Choice(["alembic", "deepdiff"], case_sensitive=False),
542 help="Comparator to use for schema comparison",
543 default="deepdiff",
544)
545@click.option("-E", "--error-on-change", is_flag=True, help="Exit with error code if schemas are different")
546@click.argument("uris", nargs=-1, type=str)
547@click.pass_context
548def diff(
549 ctx: click.Context,
550 engine_url: str | None,
551 comparator: str,
552 error_on_change: bool,
553 uris: Iterable[str],
554) -> None:
555 uri_list = list(uris)
556 schemas = [Schema.from_uri(uri, context={"id_generation": ctx.obj["id_generation"]}) for uri in uri_list]
557 diff: SchemaDiff
558 if len(schemas) == 2:
559 if comparator == "alembic":
560 metadata = create_metadata(schemas[0], engine_url=engine_url)
561 with create_database_context(
562 engine_url if engine_url else "sqlite:///:memory:", metadata
563 ) as db_ctx:
564 db_ctx.initialize()
565 db_ctx.create_all()
566 diff = DatabaseDiff(schemas[1], db_ctx.engine)
567 else:
568 diff = FormattedSchemaDiff(schemas[0], schemas[1])
569 elif len(schemas) == 1 and engine_url is not None:
570 # Create minimal metadata for the context manager
571 from sqlalchemy import MetaData
573 metadata = MetaData()
575 with create_database_context(engine_url, metadata) as db_ctx:
576 diff = DatabaseDiff(schemas[0], db_ctx.engine)
577 else:
578 raise click.ClickException(
579 "Invalid arguments - provide two schemas or a single schema and a database engine URL"
580 )
582 diff.print()
584 if diff.has_changes and error_on_change:
585 raise click.ClickException("Schema was changed")
588@cli.command(
589 "dump",
590 help="""
591 Dump a schema file to YAML or JSON format
593 Example:
595 felis dump schema.yaml schema.json
597 felis dump schema.yaml schema_dump.yaml
598 """,
599)
600@click.option(
601 "--strip-ids/--no-strip-ids",
602 is_flag=True,
603 help="Strip IDs from the output schema",
604 default=False,
605)
606@click.option(
607 "--dereference-resources/--no-dereference-resources",
608 is_flag=True,
609 help="Remove any column references from resources and inline the full column definitions in the output",
610 default=False,
611)
612@click.option(
613 "--sort-columns/--no-sort-columns",
614 is_flag=True,
615 help="Sort columns alphabetically by name in the output",
616 default=False,
617)
618@click.argument("uris", nargs=2, type=str)
619@click.pass_context
620def dump(
621 ctx: click.Context,
622 strip_ids: bool,
623 dereference_resources: bool,
624 sort_columns: bool,
625 uris: list[str],
626) -> None:
627 if strip_ids:
628 logger.info("Stripping IDs from the output schema")
629 if sort_columns:
630 logger.info("Sorting output columns alphabetically")
631 if uris[1].endswith(".json"):
632 format = "json"
633 elif uris[1].endswith(".yaml"):
634 format = "yaml"
635 else:
636 raise click.ClickException("Output file must have a .json or .yaml extension")
637 schema = Schema.from_uri(
638 uris[0],
639 context={"id_generation": ctx.obj["id_generation"], "dereference_resources": dereference_resources},
640 )
641 with open(uris[1], "w") as f:
642 if format == "yaml":
643 schema.dump_yaml(f, strip_ids=strip_ids, sort_columns=sort_columns)
644 elif format == "json":
645 schema.dump_json(f, strip_ids=strip_ids, sort_columns=sort_columns)
646 logger.info("Dumped '%s' to '%s'", uris[0], uris[1])
649if __name__ == "__main__": 649 ↛ 650line 649 didn't jump to line 650 because the condition on line 649 was never true
650 cli()