Standardize Datadog log format (#46)

Author: ikretz

requirements-dev.txt

@@ -16,6 +16,21 @@
 
 _log = logging.getLogger(__name__)
 
+DD_SOURCE = "scfw"
+"""
+Source value for Datadog logging.
+"""
+
+DD_SERVICE = "scfw"
+"""
+Service value for Datadog logging.
+"""
+
+DD_ENV = "dev"
+"""
+Default environment value for Datadog logging.
+"""
+
 DD_API_KEY_VAR = "DD_API_KEY"
 """
 The environment variable under which the firewall looks for a Datadog API key.
@@ -165,8 +180,8 @@ def _configure_agent_logging(port: str):
         "logs:\n"
         "  - type: tcp\n"
         f"    port: {port}\n"
-        '    service: "scfw"\n'
-        '    source: "scfw"\n'
+        f'    service: "{DD_SERVICE}"\n'
+        f'    source: "{DD_SOURCE}"\n'
     )
 
     try:

requirements.txt

@@ -2,51 +2,19 @@
 Configures a logger for sending firewall logs to a local Datadog Agent.
 """
 
-import json
 import logging
 import os
 import socket
 
-import scfw
 from scfw.configure import DD_AGENT_PORT_VAR
 from scfw.logger import FirewallLogger
-from scfw.loggers.dd_logger import DDLogger
+from scfw.loggers.dd_logger import DDLogFormatter, DDLogger
 
 _log = logging.getLogger(__name__)
 
 _DD_LOG_NAME = "dd_agent_log"
 
 
-class _DDLogFormatter(logging.Formatter):
-    """
-    A custom JSON formatter for firewall logs.
-    """
-    DD_ENV = "dev"
-    DD_VERSION = scfw.__version__
-    DD_SERVICE = DD_SOURCE = "scfw"
-
-    def format(self, record) -> str:
-        """
-        Format a log record as a JSON string.
-
-        Args:
-            record: The log record to be formatted.
-        """
-        log_record = {
-            "source": self.DD_SOURCE,
-            "service": self.DD_SERVICE,
-            "version": self.DD_VERSION
-        }
-
-        env = os.getenv("DD_ENV")
-        log_record["env"] = env if env else self.DD_ENV
-
-        for key in {"action", "created", "ecosystem", "msg", "targets"}:
-            log_record[key] = record.__dict__[key]
-
-        return json.dumps(log_record) + '\n'
-
-
 class _DDLogHandler(logging.Handler):
     def emit(self, record):
         """
@@ -72,7 +40,7 @@ def emit(self, record):
 
 # Configure a single logging handle for all `DDAgentLogger` instances to share
 _handler = _DDLogHandler() if os.getenv(DD_AGENT_PORT_VAR) else logging.NullHandler()
-_handler.setFormatter(_DDLogFormatter())
+_handler.setFormatter(DDLogFormatter())
 
 _ddlog = logging.getLogger(_DD_LOG_NAME)
 _ddlog.setLevel(logging.INFO)

scfw/configure.py

@@ -13,14 +13,12 @@
 from datadog_api_client.v2.model.http_log_item import HTTPLogItem
 
 import scfw
-from scfw.configure import DD_API_KEY_VAR
+from scfw.configure import DD_API_KEY_VAR, DD_ENV, DD_SERVICE, DD_SOURCE
 from scfw.logger import FirewallLogger
-from scfw.loggers.dd_logger import DDLogger
+from scfw.loggers.dd_logger import DDLogFormatter, DDLogger
 
 _DD_LOG_NAME = "dd_api_log"
 
-_DD_LOG_FORMAT = "%(asctime)s %(levelname)s [%(name)s] [%(filename)s:%(lineno)d] - %(message)s"
-
 
 class _DDLogHandler(logging.Handler):
     """
@@ -29,36 +27,29 @@ class _DDLogHandler(logging.Handler):
 
     In addition to USM tags, install targets are tagged with the `target` tag and included.
     """
-    DD_SOURCE = "scfw"
-    DD_ENV = "dev"
-    DD_VERSION = scfw.__version__
-
-    def __init__(self):
-        super().__init__()
-
     def emit(self, record):
         """
         Format and send a log to Datadog.
 
         Args:
             record: The log record to be forwarded.
         """
-        env = os.getenv("DD_ENV", self.DD_ENV)
-        service = os.getenv("DD_SERVICE", record.__dict__.get("ecosystem", self.DD_SOURCE))
-
-        usm_tags = {f"env:{env}", f"version:{self.DD_VERSION}"}
+        usm_tags = {
+            f"env:{os.getenv('DD_ENV', DD_ENV)}",
+            f"version:{scfw.__version__}"
+        }
 
         targets = record.__dict__.get("targets", {})
         target_tags = set(map(lambda e: f"target:{e}", targets))
 
         body = HTTPLog(
             [
                 HTTPLogItem(
-                    ddsource=self.DD_SOURCE,
+                    ddsource=DD_SOURCE,
                     ddtags=",".join(usm_tags | target_tags),
                     hostname=socket.gethostname(),
                     message=self.format(record),
-                    service=service,
+                    service=DD_SERVICE,
                 ),
             ]
         )
@@ -71,7 +62,7 @@ def emit(self, record):
 
 # Configure a single logging handle for all `DDAPILogger` instances to share
 _handler = _DDLogHandler() if os.getenv(DD_API_KEY_VAR) else logging.NullHandler()
-_handler.setFormatter(logging.Formatter(_DD_LOG_FORMAT))
+_handler.setFormatter(DDLogFormatter())
 
 _ddlog = logging.getLogger(_DD_LOG_NAME)
 _ddlog.setLevel(logging.INFO)

scfw/loggers/dd_agent_logger.py

@@ -2,12 +2,14 @@
 Provides a `FirewallLogger` class for sending logs to Datadog.
 """
 
+import json
 import logging
 import os
 
 import dotenv
 
-from scfw.configure import DD_LOG_LEVEL_VAR
+import scfw
+from scfw.configure import DD_ENV, DD_LOG_LEVEL_VAR, DD_SERVICE, DD_SOURCE
 from scfw.ecosystem import ECOSYSTEM
 from scfw.logger import FirewallAction, FirewallLogger
 from scfw.target import InstallTarget
@@ -20,6 +22,30 @@
 dotenv.load_dotenv()
 
 
+class DDLogFormatter(logging.Formatter):
+    """
+    A custom JSON formatter for firewall logs.
+    """
+    def format(self, record) -> str:
+        """
+        Format a log record as a JSON string.
+
+        Args:
+            record: The log record to be formatted.
+        """
+        log_record = {
+            "source": DD_SOURCE,
+            "service": DD_SERVICE,
+            "version": scfw.__version__,
+            "env": os.getenv("DD_ENV", DD_ENV),
+        }
+
+        for key in {"action", "created", "ecosystem", "msg", "targets"}:
+            log_record[key] = record.__dict__[key]
+
+        return json.dumps(log_record) + '\n'
+
+
 class DDLogger(FirewallLogger):
     """
     An implementation of `FirewallLogger` for sending logs to Datadog.
@@ -58,15 +84,7 @@ def log(
         if not self._level or action < self._level:
             return
 
-        match action:
-            case FirewallAction.ALLOW:
-                message = f"Command '{' '.join(command)}' was allowed"
-            case FirewallAction.ABORT:
-                message = f"Command '{' '.join(command)}' was aborted"
-            case FirewallAction.BLOCK:
-                message = f"Command '{' '.join(command)}' was blocked"
-
         self._logger.info(
-            message,
+            f"Command '{' '.join(command)}' was {str(action).lower()}ed",
             extra={"action": str(action), "ecosystem": str(ecosystem), "targets": list(map(str, targets))}
         )