Feature/712 add logging

doc/api.rst

@@ -0,0 +1,17 @@
+.. _api:
+
+:octicon:`cpu` API Reference
+=============================
+
+.. _workflow_exceptions:
+
+Workflow Exceptions
+-------------------
+These custom exceptions are associated with generating
+the workflows provided by the PTB. They are located in the
+``exasol.toolbox.util.workflows.exceptions`` module.
+
+.. currentmodule:: exasol.toolbox.util.workflows.exceptions
+
+.. automodule:: exasol.toolbox.util.workflows.exceptions
+   :members:

doc/changes/unreleased.md

@@ -5,6 +5,7 @@
 ## Feature
 
 * #691: Started customization of PTB workflows by defining the YML schema
+* #712: Added basic logging to workflow processing
 
 ## Documentation
 

doc/conf.py

@@ -42,7 +42,7 @@
     "sphinx_copybutton",
     "exasol.toolbox.sphinx.multiversion",
 ]
-
+add_module_names = False
 intersphinx_mapping = {"python": ("https://docs.python.org/3", None)}
 
 # Make sure the target is unique

doc/index.rst

@@ -31,6 +31,12 @@ Documentation of the Exasol-Toolbox
 
         Custom GitHub Actions providing functionality that is commonly needed in our projects.
 
+    .. grid-item-card:: :octicon:`cpu` API Reference
+        :link: api
+        :link-type: ref
+
+        Comprehensive technical documentation for API endpoints and methods
+
     .. grid-item-card:: :octicon:`repo` Design Document
         :link: design_document
         :link-type: ref
@@ -46,4 +52,5 @@ Documentation of the Exasol-Toolbox
    developer_guide/developer_guide
    tools
    github_actions/github_actions
+   api
    changes/changelog

doc/user_guide/features/github_workflows/index.rst

@@ -8,6 +8,7 @@ Enabling GitHub Workflows
     :hidden:
 
     configuration
+    troubleshooting
 
 The PTB ships with configurable GitHub workflow templates covering the most common
 CI/CD setup variants for Python projects. The templates are defined in:

doc/user_guide/features/github_workflows/troubleshooting.rst

@@ -0,0 +1,9 @@
+.. _workflows_troubleshooting:
+
+Troubleshooting
+===============
+
+.. toctree::
+    :maxdepth: 2
+
+    ../../troubleshooting/debug_github_workflows

doc/user_guide/troubleshooting/debug_github_workflows.rst

@@ -0,0 +1,26 @@
+.. _debug_workflows_troubleshooting:
+
+Debugging Generated GitHub Workflows
+====================================
+
+This troubleshooting guide is helpful if you run into issues installing or updating
+the GitHub workflows provided by the PTB.
+
+Enabling Debug Logging
+----------------------
+
+To get more detailed output, set the ``LOG_LEVEL`` environment variable to ``DEBUG`` before executing a CLI command.
+By default, the ``LOG_LEVEL`` is set to ``INFO``.
+
+.. code-block:: bash
+
+   export LOG_LEVEL=DEBUG
+
+Checking Custom Exceptions
+----------------------------
+
+Certain pain points are associated with custom exceptions. These give a brief statement
+on what could be wrong and in which file. For further information, check the traceback.
+
+For the list of the custom exceptions for installing or updating the GitHub workflows,
+see the :ref:`workflow_exceptions`.

doc/user_guide/troubleshooting/ignore_ruff_findings.rst

@@ -4,13 +4,13 @@ Ignoring Ruff Findings
 ======================
 
 A typical example is when importing all PTB's Nox sessions in your
-``noxfile.py``, which may cause ruff to report error "F403 unused import".
+``noxfile.py``, which may cause ruff to report error "F401 unused import".
 
 You can ignore this finding by appending a comment to the code line:
 
 .. code-block:: python
 
-    from exasol.toolbox.nox.tasks import *  # noqa: F403
+    from exasol.toolbox.nox.tasks import *  # noqa: F401
 
 See also
 

doc/user_guide/troubleshooting/index.rst

@@ -16,5 +16,6 @@ proposed mitigations, some potentially specific to the related tool.
    format_check_errors_due_to_configuration_issues
    format_check_reports_unmodified_files
    formatting_disable
-   "F403 unused import" (reported by Ruff) <ignore_ruff_findings>
+   "F401 unused import" (reported by Ruff) <ignore_ruff_findings>
    Sonar findings <../features/metrics/ignore_findings>
+   debug_github_workflows

exasol/toolbox/__init__.py

@@ -0,0 +1,33 @@
+import logging
+import os
+
+import structlog
+from structlog.dev import ConsoleRenderer
+from structlog.processors import (
+    CallsiteParameter,
+    CallsiteParameterAdder,
+    TimeStamper,
+    add_log_level,
+    format_exc_info,
+)
+
+log_level = os.getenv("LOG_LEVEL", "INFO").upper()
+
+structlog.configure(
+    wrapper_class=structlog.make_filtering_bound_logger(getattr(logging, log_level)),
+    processors=[
+        # 1. Enrich the data first
+        add_log_level,
+        TimeStamper(fmt="iso"),
+        CallsiteParameterAdder(
+            {
+                CallsiteParameter.MODULE,
+                CallsiteParameter.FUNC_NAME,
+            }
+        ),
+        # 2. Handle exceptions
+        format_exc_info,
+        # 3. Rendering option
+        ConsoleRenderer(),
+    ],
+)

exasol/toolbox/util/workflows/__init__.py

@@ -0,0 +1,3 @@
+import structlog
+
+logger = structlog.get_logger(__name__).bind(subsystem="workflows")

exasol/toolbox/util/workflows/exceptions.py

@@ -26,7 +26,7 @@ class YamlOutputError(YamlError):
 class YamlParsingError(YamlError):
     """
     Raised when the rendered template is not a valid YAML file, as it cannot be
-     parsed by ruamel-yaml.
+    parsed by ruamel-yaml.
     """
 
     message_template = (

exasol/toolbox/util/workflows/render_yaml.py

@@ -15,6 +15,7 @@
 )
 from ruamel.yaml.error import YAMLError
 
+from exasol.toolbox.util.workflows import logger
 from exasol.toolbox.util.workflows.exceptions import (
     TemplateRenderingError,
     YamlOutputError,
@@ -60,6 +61,11 @@ def _render_with_jinja(self, input_str: str) -> str:
         """
         Render the template with Jinja.
         """
+        logger.debug(
+            "Render template with Jinja",
+            jinja_dict_source="PROJECT_CONFIG.github_template_dict",
+            jinja_dict_values=self.github_template_dict,
+        )
         jinja_template = jinja_env.from_string(input_str)
         return jinja_template.render(self.github_template_dict)
 
@@ -72,6 +78,7 @@ def get_yaml_dict(self) -> CommentedMap:
         try:
             workflow_string = self._render_with_jinja(raw_content)
             yaml = self._get_standard_yaml()
+            logger.debug("Parse template with ruamel-yaml")
             return yaml.load(workflow_string)
         except TemplateError as ex:
             raise TemplateRenderingError(file_path=self.file_path) from ex
@@ -84,6 +91,7 @@ def get_as_string(self, yaml_dict: CommentedMap) -> str:
         """
         yaml = self._get_standard_yaml()
         try:
+            logger.debug("Output workflow as string")
             with io.StringIO() as stream:
                 yaml.dump(yaml_dict, stream)
                 workflow_string = stream.getvalue()

exasol/toolbox/util/workflows/workflow.py

@@ -5,7 +5,11 @@
     BaseModel,
     ConfigDict,
 )
+from structlog.contextvars import (
+    bound_contextvars,
+)
 
+from exasol.toolbox.util.workflows import logger
 from exasol.toolbox.util.workflows.exceptions import YamlError
 from exasol.toolbox.util.workflows.process_template import WorkflowRenderer
 
@@ -17,18 +21,21 @@ class Workflow(BaseModel):
 
     @classmethod
     def load_from_template(cls, file_path: Path, github_template_dict: dict[str, Any]):
-        if not file_path.exists():
-            raise FileNotFoundError(file_path)
-
-        try:
-            workflow_renderer = WorkflowRenderer(
-                github_template_dict=github_template_dict,
-                file_path=file_path,
-            )
-            workflow = workflow_renderer.render()
-            return cls(content=workflow)
-        except YamlError as ex:
-            raise ex
-        except Exception as ex:
-            # Wrap all other "non-special" exceptions
-            raise ValueError(f"Error rendering file: {file_path}") from ex
+        with bound_contextvars(template_file_name=file_path.name):
+            logger.info("Load workflow from template")
+
+            if not file_path.exists():
+                raise FileNotFoundError(file_path)
+
+            try:
+                workflow_renderer = WorkflowRenderer(
+                    github_template_dict=github_template_dict,
+                    file_path=file_path,
+                )
+                workflow = workflow_renderer.render()
+                return cls(content=workflow)
+            except YamlError as ex:
+                raise ex
+            except Exception as ex:
+                # Wrap all other "non-special" exceptions
+                raise ValueError(f"Error rendering file: {file_path}") from ex

pyproject.toml

@@ -55,9 +55,10 @@ dependencies = [
     "sphinx-inline-tabs>=2023.4.21,<2024",
     "sphinx-design>=0.5.0,<1",
     "sphinx-toolbox>=4.0.0,<5",
+    "sphinxcontrib-mermaid (>=2.0.0,<3.0.0)",
+    "structlog (>=25.5.0,<26.0.0)",
     "typer[all]>=0.7.0",
     "twine>=6.1.0,<7",
-    "sphinxcontrib-mermaid (>=2.0.0,<3.0.0)",
 ]
 
 [project.scripts]

test/integration/tools/workflow_integration_test.py

@@ -1,6 +1,7 @@
 from unittest.mock import patch
 
 import pytest
+from structlog.testing import capture_logs
 
 from exasol.toolbox.tools.workflow import CLI
 
@@ -67,10 +68,11 @@ def test_show_workflow(cli_runner):
     ],
 )
 def test_diff_workflow(cli_runner, tmp_path, workflow):
-    # set up with file in tmp_path so checks files are the same
-    cli_runner.invoke(CLI, ["install", workflow, str(tmp_path)])
+    with capture_logs():
+        # set up with file in tmp_path so checks files are the same
+        cli_runner.invoke(CLI, ["install", workflow, str(tmp_path)])
 
-    result = cli_runner.invoke(CLI, ["diff", workflow, str(tmp_path)])
+        result = cli_runner.invoke(CLI, ["diff", workflow, str(tmp_path)])
 
     assert result.exit_code == 0
     # as the files are the same, we expect no difference
@@ -118,7 +120,8 @@ def test_install_twice_no_error(cli_runner, tmp_path):
 class TestUpdateWorkflow:
     @staticmethod
     def test_when_file_does_not_previously_exist(cli_runner, tmp_path):
-        result = cli_runner.invoke(CLI, ["update", "checks", str(tmp_path)])
+        with capture_logs():
+            result = cli_runner.invoke(CLI, ["update", "checks", str(tmp_path)])
 
         assert result.exit_code == 0
         assert result.output.strip() == f"Updated checks in \n{tmp_path}/checks.yml"