#6920

Committed 22 Nov 2025 06:13PM UTC coverage: 78.218% (+0.007%) from 78.211%

Build # #6920

Build Type

push

coveralls-python

Committed by

teodorescuserban

Commit Message

HDX-10015 Add production config and development tools

Production configuration:
- Add boto3 to requirements.txt for SES API support
- Configure plugin in prod.ini.tpl and common-config-ini.txt
- Add HDX_SMTP_DOMAIN to hdx_theme config declaration
- Update setup_py_helper.sh for plugin packaging

Development tools:
- run-plugin-tests.sh: Quick test runner with docker compose
  - Auto-initializes test environment if needed
  - Reuses running stack for fast iteration
  - Shows coverage for specified plugin only
- TESTING.md: Comprehensive testing documentation

Run Details

13915 of 17790 relevant lines covered (78.22%)

0.78 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

93.24

/ckanext-hdx_smtp_assumerole/ckanext/hdx_smtp_assumerole/plugin.py

# encoding: utf-8

import logging
import re
from typing import Dict, Any
import ckan.plugins as plugins
import ckan.plugins.toolkit as tk

from ckanext.hdx_smtp_assumerole.helpers.smtp_assume_role import SMTPAssumeRoleException
from ckanext.hdx_smtp_assumerole.helpers.credentials_manager import SMTPCredentialsManager
from ckanext.hdx_smtp_assumerole.helpers.mailer_patches import patch_mailer_functions
from ckanext.hdx_smtp_assumerole.helpers.hdx_users_mailer_patches import patch_hdx_users_mailer

log = logging.getLogger(__name__)


def _validate_region(region: str) -> None:
    """
    Validate AWS region format.

    :param region: AWS region string
    :raises SMTPAssumeRoleException: If region format is invalid
    """
    # Common AWS regions - not exhaustive but covers most cases
    # Format: service-region-number (e.g., us-east-1, eu-west-2)
    # Direction part must be at least 4 chars (east, west, north, south, central)
    valid_region_pattern = r'^[a-z]{2}-[a-z]{4,}-\d+$'

    if not re.match(valid_region_pattern, region):
        raise SMTPAssumeRoleException(
            f'Invalid AWS region format: {region}. '
            f'Expected format like "us-east-1" or "eu-west-2"'
        )


def _validate_role_arn(role_arn: str) -> None:
    """
    Validate IAM role ARN or role name format.

    :param role_arn: Role ARN or role name
    :raises SMTPAssumeRoleException: If format is invalid
    """
    # If it starts with 'arn:', validate full ARN format
    if role_arn.startswith('arn:'):
        # ARN format: arn:aws:iam::123456789012:role/RoleName or arn:aws:iam::123456789012:role/path/RoleName
        arn_pattern = r'^arn:aws:iam::\d{12}:role/[\w+=,.@/-]+$'
        if not re.match(arn_pattern, role_arn):
            raise SMTPAssumeRoleException(
                f'Invalid IAM role ARN format: {role_arn}. '
                f'Expected format: arn:aws:iam::ACCOUNT_ID:role/ROLE_NAME or arn:aws:iam::ACCOUNT_ID:role/path/ROLE_NAME'
            )
    else:
        # Validate role name format (alphanumeric + special chars allowed by IAM)
        role_name_pattern = r'^[\w+=,.@-]+$'
        if not re.match(role_name_pattern, role_arn):
            raise SMTPAssumeRoleException(
                f'Invalid IAM role name: {role_arn}. '
                f'Role names can contain alphanumeric characters and: + = , . @ -'
            )


def run_on_startup(config: Dict[str, Any]) -> None:
    """
    Run startup tasks for SMTP AssumeRole plugin.

    If AssumeRole is enabled, this will:
    1. Initialize the credentials manager with config
    2. Load initial credentials via AssumeRole
    3. Apply monkey patches to email senders to use SES API
    4. Set up email domain defaults if configured

    The credentials manager will automatically refresh credentials
    when they are about to expire (< 5 minutes).

    If AssumeRole is disabled, CKAN will use static SMTP credentials
    from the config file (backward compatibility).

    :param config: CKAN config object
    :type config: dict
    """
    use_assume_role = tk.asbool(config.get('ckanext.hdx_smtp_assumerole.use_assume_role', False))

    if not use_assume_role:
        log.debug('SMTP AssumeRole is DISABLED. Using static SMTP credentials from config.')
        return

    log.debug('SMTP AssumeRole is ENABLED. Switching to SES API with auto-refresh...')

    try:
        # Validate required parameters
        role_name_or_arn = config.get('ckanext.hdx_smtp_assumerole.role_arn')
        region = config.get('ckanext.hdx_smtp_assumerole.region')

        if not role_name_or_arn:
            raise SMTPAssumeRoleException(
                'ckanext.hdx_smtp_assumerole.role_arn is required when use_assume_role is enabled'
            )

        if not region:
            raise SMTPAssumeRoleException(
                'ckanext.hdx_smtp_assumerole.region is required when use_assume_role is enabled'
            )

        # Validate formats
        _validate_role_arn(role_name_or_arn)
        _validate_region(region)

        # Initialize credentials manager (loads initial credentials)
        manager = SMTPCredentialsManager.get_instance()
        manager.initialize(config)

        # Apply monkey patches to replace SMTP with SES API
        log.debug('Applying patches to replace SMTP with SES API')
        patch_mailer_functions()  # Patch ckan.lib.mailer
        patch_hdx_users_mailer()  # Patch ckanext.hdx_users.helpers.mailer

        # Set up email domain defaults if configured
        smtp_domain = config.get('ckanext.hdx_smtp_assumerole.smtp_domain', '')
        if smtp_domain:
            if not config.get('email_to'):
                config['email_to'] = f'ckan@{smtp_domain}'
            if not config.get('error_email_from'):
                config['error_email_from'] = f'ckan@{smtp_domain}'
            if not config.get('smtp.mail_from'):
                config['smtp.mail_from'] = f'hdx@{smtp_domain}'

        # Get credentials info for logging
        creds_info = manager.get_credentials_info()

        # Single concise success message
        log.info(f'SES API with AssumeRole enabled: region={region}, role={role_name_or_arn}, expires={creds_info.get("expiration_time")}')

    except SMTPAssumeRoleException as e:
        log.error(f'SES AssumeRole configuration failed: {str(e)}')
        log.error('Email functionality will NOT work with AssumeRole.')
        log.error('To use static SMTP credentials, set: ckanext.hdx_smtp_assumerole.use_assume_role = false')
        raise
    except Exception as e:
        log.error(f'Unexpected error during SES AssumeRole setup: {str(e)}')
        log.error('Email functionality will NOT work with AssumeRole.')
        log.error('To use static SMTP credentials, set: ckanext.hdx_smtp_assumerole.use_assume_role = false')
        raise


@tk.blanket.config_declarations
class HDXSMTPAssumeRolePlugin(plugins.SingletonPlugin):
    """
    CKAN plugin for AWS SES API with AssumeRole support and automatic credential refresh.

    This plugin allows CKAN to send emails via AWS SES API (not SMTP) using temporary
    credentials obtained through AWS STS AssumeRole. This solves the problem that
    SES SMTP doesn't support session tokens from temporary credentials.

    Features:
    - SES API instead of SMTP: Supports temporary credentials with session tokens
    - Automatic credential refresh: Credentials refresh when < 5 minutes to expiry
    - Per-container independent: Each container manages its own credentials
    - Thread-safe: Safe for multi-threaded WSGI servers
    - Lazy loading: Credentials only refresh when sending email
    - No restart needed: Credentials refresh automatically before expiry
    - Backward compatible: Disable use_assume_role to use static SMTP credentials

    Why SES API instead of SMTP?
    - SMTP only supports username/password authentication
    - Temporary credentials from AssumeRole include a session token
    - SES SMTP cannot use session tokens -> authentication fails
    - SES API supports full temporary credentials with session token

    Configuration example in .ini file:

        # Enable SES API with AssumeRole (disable for static SMTP)
        ckanext.hdx_smtp_assumerole.use_assume_role = true

        # Role name (will deduce account ID) or full ARN
        ckanext.hdx_smtp_assumerole.role_arn = my-ses-role
        # OR
        # ckanext.hdx_smtp_assumerole.role_arn = arn:aws:iam::123456789012:role/my-ses-role

        # AWS region for SES
        ckanext.hdx_smtp_assumerole.region = us-east-1

        # Optional: Session name (default: ckan-smtp-session)
        ckanext.hdx_smtp_assumerole.session_name = ckan-smtp-session

        # Optional: Email domain for default addresses
        ckanext.hdx_smtp_assumerole.smtp_domain = example.com

    Backward Compatibility:
    To use static SMTP credentials (old system), set use_assume_role = false
    or simply don't set it (defaults to false). The plugin will not patch
    email senders and CKAN will use standard SMTP with static credentials.
    """

    plugins.implements(plugins.IConfigurer, inherit=False)
    plugins.implements(plugins.IMiddleware, inherit=True)

    __startup_tasks_done = False

    def update_config(self, config: Dict[str, Any]) -> None:
        """
        Update CKAN config with plugin settings.

        :param config: CKAN config object
        :type config: dict
        """
        # No template directories needed for this plugin
        pass

    def make_middleware(self, app: Any, config: Dict[str, Any]) -> Any:
        """
        Called during application initialization.
        This is where we run the AssumeRole logic at startup.

        :param app: WSGI application
        :param config: CKAN config object
        :return: WSGI application
        """
        if not HDXSMTPAssumeRolePlugin.__startup_tasks_done:
            run_on_startup(config)
            HDXSMTPAssumeRolePlugin.__startup_tasks_done = True
        return app

1	# encoding: utf-8
2
3	import logging	1✔
4	import re	1✔
5	from typing import Dict, Any	1✔
6	import ckan.plugins as plugins	1✔
7	import ckan.plugins.toolkit as tk	1✔
8
9	from ckanext.hdx_smtp_assumerole.helpers.smtp_assume_role import SMTPAssumeRoleException	1✔
10	from ckanext.hdx_smtp_assumerole.helpers.credentials_manager import SMTPCredentialsManager	1✔
11	from ckanext.hdx_smtp_assumerole.helpers.mailer_patches import patch_mailer_functions	1✔
12	from ckanext.hdx_smtp_assumerole.helpers.hdx_users_mailer_patches import patch_hdx_users_mailer	1✔
13
14	log = logging.getLogger(__name__)	1✔
15
16
17	def _validate_region(region: str) -> None:	1✔
18	"""
19	Validate AWS region format.
20
21	:param region: AWS region string
22	:raises SMTPAssumeRoleException: If region format is invalid
23	"""
24	# Common AWS regions - not exhaustive but covers most cases
25	# Format: service-region-number (e.g., us-east-1, eu-west-2)
26	# Direction part must be at least 4 chars (east, west, north, south, central)
27	valid_region_pattern = r'^[a-z]{2}-[a-z]{4,}-\d+$'	1✔
28
29	if not re.match(valid_region_pattern, region):	1✔
30	raise SMTPAssumeRoleException(	1✔
31	f'Invalid AWS region format: {region}. '
32	f'Expected format like "us-east-1" or "eu-west-2"'
33	)
34
35
36	def _validate_role_arn(role_arn: str) -> None:	1✔
37	"""
38	Validate IAM role ARN or role name format.
39
40	:param role_arn: Role ARN or role name
41	:raises SMTPAssumeRoleException: If format is invalid
42	"""
43	# If it starts with 'arn:', validate full ARN format
44	if role_arn.startswith('arn:'):	1✔
45	# ARN format: arn:aws:iam::123456789012:role/RoleName or arn:aws:iam::123456789012:role/path/RoleName
46	arn_pattern = r'^arn:aws:iam::\d{12}:role/[\w+=,.@/-]+$'	1✔
47	if not re.match(arn_pattern, role_arn):	1✔
48	raise SMTPAssumeRoleException(	1✔
49	f'Invalid IAM role ARN format: {role_arn}. '
50	f'Expected format: arn:aws:iam::ACCOUNT_ID:role/ROLE_NAME or arn:aws:iam::ACCOUNT_ID:role/path/ROLE_NAME'
51	)
52	else:
53	# Validate role name format (alphanumeric + special chars allowed by IAM)
54	role_name_pattern = r'^[\w+=,.@-]+$'	1✔
55	if not re.match(role_name_pattern, role_arn):	1✔
56	raise SMTPAssumeRoleException(	1✔
57	f'Invalid IAM role name: {role_arn}. '
58	f'Role names can contain alphanumeric characters and: + = , . @ -'
59	)
60
61
62	def run_on_startup(config: Dict[str, Any]) -> None:	1✔
63	"""
64	Run startup tasks for SMTP AssumeRole plugin.
65
66	If AssumeRole is enabled, this will:
67	1. Initialize the credentials manager with config
68	2. Load initial credentials via AssumeRole
69	3. Apply monkey patches to email senders to use SES API
70	4. Set up email domain defaults if configured
71
72	The credentials manager will automatically refresh credentials
73	when they are about to expire (< 5 minutes).
74
75	If AssumeRole is disabled, CKAN will use static SMTP credentials
76	from the config file (backward compatibility).
77
78	:param config: CKAN config object
79	:type config: dict
80	"""
81	use_assume_role = tk.asbool(config.get('ckanext.hdx_smtp_assumerole.use_assume_role', False))	1✔
82
83	if not use_assume_role:	1✔
84	log.debug('SMTP AssumeRole is DISABLED. Using static SMTP credentials from config.')	1✔
85	return	1✔
86
87	log.debug('SMTP AssumeRole is ENABLED. Switching to SES API with auto-refresh...')	1✔
88
89	try:	1✔
90	# Validate required parameters
91	role_name_or_arn = config.get('ckanext.hdx_smtp_assumerole.role_arn')	1✔
92	region = config.get('ckanext.hdx_smtp_assumerole.region')	1✔
93
94	if not role_name_or_arn:	1✔
95	raise SMTPAssumeRoleException(	1✔
96	'ckanext.hdx_smtp_assumerole.role_arn is required when use_assume_role is enabled'
97	)
98
99	if not region:	1✔
100	raise SMTPAssumeRoleException(	1✔
101	'ckanext.hdx_smtp_assumerole.region is required when use_assume_role is enabled'
102	)
103
104	# Validate formats
105	_validate_role_arn(role_name_or_arn)	1✔
106	_validate_region(region)	1✔
107
108	# Initialize credentials manager (loads initial credentials)
109	manager = SMTPCredentialsManager.get_instance()	1✔
110	manager.initialize(config)	1✔
111
112	# Apply monkey patches to replace SMTP with SES API
113	log.debug('Applying patches to replace SMTP with SES API')	1✔
114	patch_mailer_functions() # Patch ckan.lib.mailer	1✔
115	patch_hdx_users_mailer() # Patch ckanext.hdx_users.helpers.mailer	1✔
116
117	# Set up email domain defaults if configured
118	smtp_domain = config.get('ckanext.hdx_smtp_assumerole.smtp_domain', '')	1✔
119	if smtp_domain:	1✔
120	if not config.get('email_to'):	1✔
121	config['email_to'] = f'ckan@{smtp_domain}'	1✔
122	if not config.get('error_email_from'):	1✔
123	config['error_email_from'] = f'ckan@{smtp_domain}'	1✔
124	if not config.get('smtp.mail_from'):	1✔
125	config['smtp.mail_from'] = f'hdx@{smtp_domain}'	1✔
126
127	# Get credentials info for logging
128	creds_info = manager.get_credentials_info()	1✔
129
130	# Single concise success message
131	log.info(f'SES API with AssumeRole enabled: region={region}, role={role_name_or_arn}, expires={creds_info.get("expiration_time")}')	1✔
132
133	except SMTPAssumeRoleException as e:	1✔
134	log.error(f'SES AssumeRole configuration failed: {str(e)}')	1✔
135	log.error('Email functionality will NOT work with AssumeRole.')	1✔
136	log.error('To use static SMTP credentials, set: ckanext.hdx_smtp_assumerole.use_assume_role = false')	1✔
137	raise	1✔
138	except Exception as e:	×
139	log.error(f'Unexpected error during SES AssumeRole setup: {str(e)}')	×
140	log.error('Email functionality will NOT work with AssumeRole.')	×
141	log.error('To use static SMTP credentials, set: ckanext.hdx_smtp_assumerole.use_assume_role = false')	×
142	raise	×
143
144
145	@tk.blanket.config_declarations	1✔
146	class HDXSMTPAssumeRolePlugin(plugins.SingletonPlugin):	1✔
147	"""
148	CKAN plugin for AWS SES API with AssumeRole support and automatic credential refresh.
149
150	This plugin allows CKAN to send emails via AWS SES API (not SMTP) using temporary
151	credentials obtained through AWS STS AssumeRole. This solves the problem that
152	SES SMTP doesn't support session tokens from temporary credentials.
153
154	Features:
155	- SES API instead of SMTP: Supports temporary credentials with session tokens
156	- Automatic credential refresh: Credentials refresh when < 5 minutes to expiry
157	- Per-container independent: Each container manages its own credentials
158	- Thread-safe: Safe for multi-threaded WSGI servers
159	- Lazy loading: Credentials only refresh when sending email
160	- No restart needed: Credentials refresh automatically before expiry
161	- Backward compatible: Disable use_assume_role to use static SMTP credentials
162
163	Why SES API instead of SMTP?
164	- SMTP only supports username/password authentication
165	- Temporary credentials from AssumeRole include a session token
166	- SES SMTP cannot use session tokens -> authentication fails
167	- SES API supports full temporary credentials with session token
168
169	Configuration example in .ini file:
170
171	# Enable SES API with AssumeRole (disable for static SMTP)
172	ckanext.hdx_smtp_assumerole.use_assume_role = true
173
174	# Role name (will deduce account ID) or full ARN
175	ckanext.hdx_smtp_assumerole.role_arn = my-ses-role
176	# OR
177	# ckanext.hdx_smtp_assumerole.role_arn = arn:aws:iam::123456789012:role/my-ses-role
178
179	# AWS region for SES
180	ckanext.hdx_smtp_assumerole.region = us-east-1
181
182	# Optional: Session name (default: ckan-smtp-session)
183	ckanext.hdx_smtp_assumerole.session_name = ckan-smtp-session
184
185	# Optional: Email domain for default addresses
186	ckanext.hdx_smtp_assumerole.smtp_domain = example.com
187
188	Backward Compatibility:
189	To use static SMTP credentials (old system), set use_assume_role = false
190	or simply don't set it (defaults to false). The plugin will not patch
191	email senders and CKAN will use standard SMTP with static credentials.
192	"""
193
194	plugins.implements(plugins.IConfigurer, inherit=False)	1✔
195	plugins.implements(plugins.IMiddleware, inherit=True)	1✔
196
197	__startup_tasks_done = False	1✔
198
199	def update_config(self, config: Dict[str, Any]) -> None:	1✔
200	"""
201	Update CKAN config with plugin settings.
202
203	:param config: CKAN config object
204	:type config: dict
205	"""
206	# No template directories needed for this plugin
207	pass	1✔
208
209	def make_middleware(self, app: Any, config: Dict[str, Any]) -> Any:	1✔
210	"""
211	Called during application initialization.
212	This is where we run the AssumeRole logic at startup.
213
214	:param app: WSGI application
215	:param config: CKAN config object
216	:return: WSGI application
217	"""
218	if not HDXSMTPAssumeRolePlugin.__startup_tasks_done:	1✔
219	run_on_startup(config)	1✔
220	HDXSMTPAssumeRolePlugin.__startup_tasks_done = True	1✔
221	return app	1✔

OCHA-DAP / hdx-ckan / #6920

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous