diff --git a/.github/workflows/update-flagsmith-environment.yml b/.github/workflows/update-flagsmith-environment.yml index 87a9c9cf5a0f..69ccd4231dfa 100644 --- a/.github/workflows/update-flagsmith-environment.yml +++ b/.github/workflows/update-flagsmith-environment.yml @@ -9,6 +9,9 @@ defaults: run: working-directory: api +permissions: + contents: read + jobs: update_server_defaults: runs-on: depot-ubuntu-latest diff --git a/api/app/routers.py b/api/app/routers.py index 0ba86350f31d..35753699d773 100644 --- a/api/app/routers.py +++ b/api/app/routers.py @@ -3,6 +3,7 @@ from django.db.models import Model AnalyticsDatabaseName = Literal["analytics"] +ClickHouseDatabaseName = Literal["clickhouse"] class AnalyticsRouter: @@ -38,3 +39,38 @@ def allow_migrate(self, db: str, app_label: str, **hints: Any) -> bool | None: if db == "analytics": return app_label in self.route_app_labels return None + + +class ClickHouseRouter: + route_app_labels = ["clickhouse"] + + def db_for_read( + self, model: type[Model], **hints: Any + ) -> ClickHouseDatabaseName | None: + if model._meta.app_label in self.route_app_labels: + return "clickhouse" + return None + + def db_for_write( + self, model: type[Model], **hints: Any + ) -> ClickHouseDatabaseName | None: + if model._meta.app_label in self.route_app_labels: + return "clickhouse" + return None + + def allow_relation(self, obj1: Model, obj2: Model, **hints: Any) -> bool | None: + # ClickHouse has no FKs and we don't expose CH-app models, so any + # relation involving this app is forbidden. + if ( + obj1._meta.app_label in self.route_app_labels + or obj2._meta.app_label in self.route_app_labels + ): + return False + return None + + def allow_migrate(self, db: str, app_label: str, **hints: Any) -> bool | None: + if db == "clickhouse": + return app_label in self.route_app_labels + if app_label in self.route_app_labels: + return False + return None diff --git a/api/app/settings/common.py b/api/app/settings/common.py index 027d029f085d..aec7dcd29b5b 100644 --- a/api/app/settings/common.py +++ b/api/app/settings/common.py @@ -118,6 +118,8 @@ "features.workflows.core", "features.release_pipelines.core", "segments", + "segment_membership", + "clickhouse", "app", "e2etests", "simple_history", @@ -1439,3 +1441,39 @@ PYLON_IDENTITY_VERIFICATION_SECRET = env.str("PYLON_IDENTITY_VERIFICATION_SECRET", None) OSIC_UPDATE_BATCH_SIZE = env.int("OSIC_UPDATE_BATCH_SIZE", default=500) + +# ClickHouse backs the segment_membership backfill and refresh tasks. Set +# CLICKHOUSE_URL (DSN form) or any CLICKHOUSE_HOST + discrete fields to enable. +# Discrete settings override the matching field parsed from the URL. +CLICKHOUSE_URL = env.str("CLICKHOUSE_URL", default=None) +CLICKHOUSE_HOST = env.str("CLICKHOUSE_HOST", default=None) +CLICKHOUSE_PORT = env.int("CLICKHOUSE_PORT", default=None) +CLICKHOUSE_USER = env.str("CLICKHOUSE_USER", default=None) +CLICKHOUSE_PASSWORD = env.str("CLICKHOUSE_PASSWORD", default=None) +CLICKHOUSE_DATABASE = env.str("CLICKHOUSE_DATABASE", default=None) +CLICKHOUSE_SECURE = env.bool("CLICKHOUSE_SECURE", default=None) + +CLICKHOUSE_ENABLED = bool(CLICKHOUSE_URL or CLICKHOUSE_HOST) + +# Always installed: the router fences the `clickhouse` app's migrations off +# the default Postgres database whether or not a CH alias is configured. +DATABASE_ROUTERS.append("app.routers.ClickHouseRouter") + +if CLICKHOUSE_ENABLED: + _clickhouse_db: dict[str, Any] = { + "ENGINE": "clickhouse_backend.backend", + "HOST": CLICKHOUSE_HOST, + "PORT": CLICKHOUSE_PORT, + "USER": CLICKHOUSE_USER, + "PASSWORD": CLICKHOUSE_PASSWORD, + "NAME": CLICKHOUSE_DATABASE, + "OPTIONS": { + "dsn": CLICKHOUSE_URL, + "secure": CLICKHOUSE_SECURE, + "settings": { + # ClickHouse Cloud 25.12 requires this for `JSON`-column DDL. + "allow_experimental_json_type": 1, + }, + }, + } + DATABASES["clickhouse"] = _clickhouse_db # type: ignore[assignment] diff --git a/api/clickhouse/__init__.py b/api/clickhouse/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/api/clickhouse/apps.py b/api/clickhouse/apps.py new file mode 100644 index 000000000000..93ef4c82537d --- /dev/null +++ b/api/clickhouse/apps.py @@ -0,0 +1,6 @@ +from django.apps import AppConfig + + +class ClickHouseConfig(AppConfig): + name = "clickhouse" + label = "clickhouse" diff --git a/api/clickhouse/migrations/0001_create_identities.py b/api/clickhouse/migrations/0001_create_identities.py new file mode 100644 index 000000000000..6643c17d959d --- /dev/null +++ b/api/clickhouse/migrations/0001_create_identities.py @@ -0,0 +1,28 @@ +from django.db import migrations + + +_SCHEMA_DDL = """\ +CREATE TABLE IF NOT EXISTS IDENTITIES ( + environment_id String, + -- (environment_id, identifier) is the natural unique key in Flagsmith's + -- identity model — dedupes ReplacingMergeTree without a synthetic id. + identifier String, + identity_key String, + -- Stored per top-level key as typed subcolumns; SQL NULL for empty traits. + traits JSON, + -- ReplacingMergeTree version column; most-recent insert wins per PK. + inserted_at DateTime DEFAULT now() +) +ENGINE = ReplacingMergeTree(inserted_at) +ORDER BY (environment_id, identifier) +""" + + +class Migration(migrations.Migration): + # ClickHouse has no transactional DDL. + atomic = False + initial = True + + operations = [ + migrations.RunSQL(_SCHEMA_DDL, reverse_sql="DROP TABLE IF EXISTS IDENTITIES"), + ] diff --git a/api/clickhouse/migrations/__init__.py b/api/clickhouse/migrations/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/api/clickhouse/models.py b/api/clickhouse/models.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/api/pyproject.toml b/api/pyproject.toml index c47a1eb5de2a..0ec978a05d7a 100644 --- a/api/pyproject.toml +++ b/api/pyproject.toml @@ -44,6 +44,8 @@ dependencies = [ "drf-writable-nested>=0.6.2,<0.7.0", "django-filter>=2.4.0,<2.5.0", "flagsmith-flag-engine>=10.1.0,<11.0.0", + "flagsmith-sql-flag-engine>=0.1.0,<0.2.0", + "django-clickhouse-backend>=1.4,<2.0", "boto3>=1.35.95,<1.36.0", "slack-sdk>=3.9.0,<3.10.0", "asgiref>=3.8.1,<3.9.0", @@ -71,7 +73,7 @@ dependencies = [ "hubspot-api-client>=12.0.0,<13.0.0", "djangorestframework-dataclasses>=1.3.1,<2.0.0", "pyotp>=2.9.0,<3.0.0", - "flagsmith-common[common-core,flagsmith-schemas,task-processor]>=3.9.0,<4", + "flagsmith-common[common-core,flagsmith-schemas,task-processor]>=3.9.1,<4", "django-stubs>=5.1.3,<6.0.0", "tzdata>=2024.1,<2025.0.0", "djangorestframework-simplejwt>=5.5.1,<6.0.0", @@ -541,6 +543,10 @@ ignore_missing_imports = true module = ["openfeature_flagsmith.*"] ignore_missing_imports = true +[[tool.mypy.overrides]] +module = ["clickhouse_backend.*", "clickhouse_driver.*"] +ignore_missing_imports = true + [[tool.mypy.overrides]] module = ["scim.*"] ignore_missing_imports = true diff --git a/api/scripts/run-docker.sh b/api/scripts/run-docker.sh index 8d014ece218f..ffe8b2207487 100755 --- a/api/scripts/run-docker.sh +++ b/api/scripts/run-docker.sh @@ -45,6 +45,9 @@ run_task_processor() { if [ -n "$TASK_PROCESSOR_DATABASE_URL" ] || [ -n "$TASK_PROCESSOR_DATABASE_NAME" ]; then waitfordb --waitfor 30 --migrations --database task_processor fi + if [ -n "$CLICKHOUSE_URL" ] || [ -n "$CLICKHOUSE_HOST" ]; then + waitfordb --waitfor 30 --migrations --database clickhouse + fi exec flagsmith start \ --bind 0.0.0.0:8000 \ --access-logfile $ACCESS_LOG_LOCATION \ @@ -68,6 +71,12 @@ migrate_task_processor_db(){ fi python manage.py migrate --database task_processor } +migrate_clickhouse_db(){ + if [ -z "$CLICKHOUSE_URL" ] && [ -z "$CLICKHOUSE_HOST" ]; then + return 0 + fi + python manage.py migrate --database clickhouse +} bootstrap(){ python manage.py bootstrap } @@ -81,17 +90,20 @@ if [ "$1" = "migrate" ]; then migrate migrate_analytics_db migrate_task_processor_db + migrate_clickhouse_db elif [ "$1" = "serve" ]; then serve elif [ "$1" = "run-task-processor" ]; then migrate migrate_analytics_db migrate_task_processor_db + migrate_clickhouse_db run_task_processor elif [ "$1" = "migrate-and-serve" ]; then migrate migrate_analytics_db migrate_task_processor_db + migrate_clickhouse_db bootstrap serve else diff --git a/api/segment_membership/__init__.py b/api/segment_membership/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/api/segment_membership/apps.py b/api/segment_membership/apps.py new file mode 100644 index 000000000000..42753fd75971 --- /dev/null +++ b/api/segment_membership/apps.py @@ -0,0 +1,6 @@ +from core.apps import BaseAppConfig + + +class SegmentMembershipConfig(BaseAppConfig): + name = "segment_membership" + default = True diff --git a/api/segment_membership/mappers.py b/api/segment_membership/mappers.py new file mode 100644 index 000000000000..9ee572db4841 --- /dev/null +++ b/api/segment_membership/mappers.py @@ -0,0 +1,43 @@ +from decimal import Decimal + +from flagsmith_schemas import dynamodb + +# (environment_id, identifier, identity_key, traits) +ClickHouseIdentityRow = tuple[str, str, str, dict[str, object] | None] + + +def map_identity_document_to_clickhouse_row( + env_key: str, + identity_doc: dynamodb.Identity, +) -> ClickHouseIdentityRow: + """Project a Dynamo identity document onto an IDENTITIES row tuple + `(environment_id, identifier, identity_key, traits)`.""" + identifier = identity_doc["identifier"] + composite_key = identity_doc["composite_key"] + raw_traits = identity_doc.get("identity_traits") + traits = _flatten_traits(raw_traits) if raw_traits else None + return ( + env_key, + identifier, + composite_key, + traits, + ) + + +def _coerce_trait_value(value: object) -> object: + # boto3 hands us `Decimal` for numbers; narrow so the JSON column + # stores a typed numeric subcolumn instead of failing to serialise. + if isinstance(value, Decimal): + if value == value.to_integral_value(): + return int(value) + return float(value) + return value + + +def _flatten_traits( + identity_traits: list[dynamodb.Trait], +) -> dict[str, object]: + return { + t["trait_key"]: _coerce_trait_value(t.get("trait_value")) + for t in identity_traits + } diff --git a/api/segment_membership/metrics.py b/api/segment_membership/metrics.py new file mode 100644 index 000000000000..b8b697fa9daa --- /dev/null +++ b/api/segment_membership/metrics.py @@ -0,0 +1,24 @@ +import prometheus_client + +# Metrics are global. Per-project / per-env drill-down lives in CH's +# `system.query_log` (via `log_comment`) and in the structlog events. + +flagsmith_segment_membership_backfill_identities_total = prometheus_client.Counter( + "flagsmith_segment_membership_backfill_identities_total", + "Total identities mirrored from Dynamo to ClickHouse by the segment-membership backfill task across all environments.", +) + +flagsmith_segment_membership_backfill_duration_seconds = prometheus_client.Histogram( + "flagsmith_segment_membership_backfill_duration_seconds", + "Duration of a segment-membership backfill for one environment.", +) + +flagsmith_segment_membership_refresh_duration_seconds = prometheus_client.Histogram( + "flagsmith_segment_membership_refresh_duration_seconds", + "Duration of a single segment-membership count-refresh run for one project.", +) + +flagsmith_segment_membership_refresh_failures_total = prometheus_client.Counter( + "flagsmith_segment_membership_refresh_failures_total", + "Total segment-membership refresh runs that failed for any reason.", +) diff --git a/api/segment_membership/migrations/0001_initial.py b/api/segment_membership/migrations/0001_initial.py new file mode 100644 index 000000000000..c4f655439245 --- /dev/null +++ b/api/segment_membership/migrations/0001_initial.py @@ -0,0 +1,57 @@ +# Generated by Django 5.2.13 on 2026-05-08 22:03 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ("environments", "0037_add_uuid_field"), + ("segments", "0030_add_default_to_segment_version"), + ] + + operations = [ + migrations.CreateModel( + name="SegmentMembershipCount", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("count", models.PositiveIntegerField()), + ("last_synced_at", models.DateTimeField()), + ( + "environment", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="+", + to="environments.environment", + ), + ), + ( + "segment", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="membership_counts", + to="segments.segment", + ), + ), + ], + options={ + "constraints": [ + models.UniqueConstraint( + fields=("segment", "environment"), + name="segment_membership_count_unique_segment_environment", + ) + ], + }, + ), + ] diff --git a/api/segment_membership/migrations/__init__.py b/api/segment_membership/migrations/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/api/segment_membership/models.py b/api/segment_membership/models.py new file mode 100644 index 000000000000..a53fe3de0b72 --- /dev/null +++ b/api/segment_membership/models.py @@ -0,0 +1,29 @@ +from django.db import models + +from environments.models import Environment +from segments.models import Segment + + +class SegmentMembershipCount(models.Model): + """Cached identity-match count for one (segment, environment) pair.""" + + segment = models.ForeignKey( + Segment, + on_delete=models.CASCADE, + related_name="membership_counts", + ) + environment = models.ForeignKey( + Environment, + on_delete=models.CASCADE, + related_name="+", + ) + count = models.PositiveIntegerField() + last_synced_at = models.DateTimeField() + + class Meta: + constraints = [ + models.UniqueConstraint( + fields=["segment", "environment"], + name="segment_membership_count_unique_segment_environment", + ), + ] diff --git a/api/segment_membership/services.py b/api/segment_membership/services.py new file mode 100644 index 000000000000..9be17c1baee9 --- /dev/null +++ b/api/segment_membership/services.py @@ -0,0 +1,129 @@ +from contextlib import contextmanager +from typing import Any, Iterator + +import structlog +from django.db import connections +from django.db.backends.utils import CursorWrapper +from flag_engine.context.types import EvaluationContext +from flagsmith_sql_flag_engine import TranslateContext, translate_segment +from flagsmith_sql_flag_engine.dialects import ClickHouseDialect + +from integrations.flagsmith.client import get_openfeature_client +from organisations.models import Organisation +from projects.models import Project +from segment_membership.models import SegmentMembershipCount +from segments.models import Segment +from util.engine_models.context.mappers import map_segment_to_segment_context +from util.mappers.engine import map_segment_to_engine + +logger = structlog.get_logger("segment_membership") + + +def is_membership_enabled(organisation: Organisation) -> bool: + """Resolve the per-org segment-membership inspection flag, default False.""" + return get_openfeature_client().get_boolean_value( + "segment_membership_inspection", + default_value=False, + evaluation_context=organisation.openfeature_evaluation_context, + ) + + +@contextmanager +def open_clickhouse_cursor( + *, log_comment: str | None = None +) -> Iterator[CursorWrapper]: + """Yield a cursor bound to the `clickhouse` database alias. + + `log_comment` lands on every query as a session setting so CH's + `system.query_log` carries per-org / per-project attribution. + """ + with connections["clickhouse"].cursor() as cursor: + if log_comment: + # Underlying clickhouse-driver cursor exposes set_settings(...). + cursor.cursor.set_settings({"log_comment": log_comment}) + yield cursor + + +def get_projects_to_process() -> Iterator[Project]: + """Yield projects with at least one canonical segment whose org has + the segment-membership flag on.""" + project_ids = Segment.live_objects.values_list("project_id", flat=True) + projects_with_live_segments = ( + Project.objects.filter(id__in=project_ids) + .select_related("organisation") + .iterator() + ) + for project in projects_with_live_segments: + if not is_membership_enabled(project.organisation): + continue + yield project + + +def compute_segment_counts_for_project( + project: Project, cursor: CursorWrapper +) -> list[SegmentMembershipCount]: + """Count identity matches per (canonical-segment, environment) for + `project` in one `UNION ALL` query. + + Returns unsaved `SegmentMembershipCount` instances with `count` and + keys populated; the caller stamps `last_synced_at` consistently + across the batch. Untranslatable segments and pairs with zero + matches are absent from the result. `FROM IDENTITIES FINAL` forces + ReplacingMergeTree to dedupe at read time so counts reflect the + most-recent backfill regardless of merge state. + """ + segments = list(Segment.live_objects.filter(project=project)) + env_id_by_key: dict[str, int] = dict( + project.environments.values_list("api_key", "id"), + ) + if not segments or not env_id_by_key: + return [] + + dialect = ClickHouseDialect() + select_clauses: list[str] = [] + for seg in segments: + translate_ctx = TranslateContext( + evaluation_context=EvaluationContext( + environment={"key": "_count", "name": project.name} + ), + dialect=dialect, + ) + predicate = translate_segment( + map_segment_to_segment_context(map_segment_to_engine(seg)), + translate_ctx, + ) + if predicate is None: + logger.error( + "compute.segment.skipped", + project__id=project.id, + segment__id=seg.id, + reason="untranslatable", + ) + continue + select_clauses.append( + f"SELECT {seg.id} AS segment_id, " + f"i.environment_id AS env_key, count() AS c " + f"FROM IDENTITIES AS i FINAL " + f"WHERE i.environment_id IN %(env_keys)s AND ({predicate}) " + f"GROUP BY i.environment_id" + ) + + if not select_clauses: + return [] + + sql = "\nUNION ALL\n".join(select_clauses) + cursor.execute(sql, {"env_keys": tuple(env_id_by_key)}) + rows: list[tuple[Any, ...]] = cursor.fetchall() + membership_counts: list[SegmentMembershipCount] = [] + for row in rows: + env_id = env_id_by_key.get(str(row[1])) + if env_id is None: + continue + membership_counts.append( + SegmentMembershipCount( + segment_id=int(row[0]), + environment_id=env_id, + count=int(row[2]), + ) + ) + return membership_counts diff --git a/api/segment_membership/tasks.py b/api/segment_membership/tasks.py new file mode 100644 index 000000000000..13c056127948 --- /dev/null +++ b/api/segment_membership/tasks.py @@ -0,0 +1,174 @@ +"""Daily backfill of IDENTITIES from Dynamo to ClickHouse, then per-project +refresh of `SegmentMembershipCount` rows. Each backfill fans out the refresh +so the count read always sees the fresh snapshot. Both tasks short-circuit +when `CLICKHOUSE_ENABLED` is False or the org's `segment_membership_inspection` +flag is off. +""" + +from datetime import timedelta +from typing import cast + +import structlog +from django.conf import settings +from django.utils import timezone +from flagsmith_schemas.dynamodb import Identity as DynamoIdentity +from task_processor.decorators import ( + register_recurring_task, + register_task_handler, +) + +from environments.dynamodb.wrappers.identity_wrapper import DynamoIdentityWrapper +from projects.models import Project +from segment_membership.mappers import map_identity_document_to_clickhouse_row +from segment_membership.metrics import ( + flagsmith_segment_membership_backfill_duration_seconds, + flagsmith_segment_membership_backfill_identities_total, + flagsmith_segment_membership_refresh_duration_seconds, + flagsmith_segment_membership_refresh_failures_total, +) +from segment_membership.models import SegmentMembershipCount +from segment_membership.services import ( + compute_segment_counts_for_project, + get_projects_to_process, + is_membership_enabled, + open_clickhouse_cursor, +) +from util.util import batched + +logger = structlog.get_logger("segment_membership") + +# Per-INSERT row count; bounds memory while loading large environments. +_INSERT_BATCH_SIZE = 1000 + +_IDENTITIES_COLUMN_NAMES = ( + "environment_id", + "identifier", + "identity_key", + "traits", +) + +_INSERT_IDENTITIES_SQL = ( + f"INSERT INTO IDENTITIES ({', '.join(_IDENTITIES_COLUMN_NAMES)}) VALUES" +) + + +@register_recurring_task( + run_every=timedelta(days=1), + # 4h fits several large environments back-to-back at SaaS scale. + timeout=timedelta(hours=4), +) +def backfill_identities_to_clickhouse() -> None: + """Insert each relevant environment's current Dynamo state into + IDENTITIES, dispatching one refresh per project as its backfill + completes so the refresh enqueue rate tracks the backfill rate + rather than spiking in one burst at the end. + """ + if not settings.CLICKHOUSE_ENABLED: + logger.info("backfill.skipped", reason="clickhouse_not_configured") + return + + wrapper = DynamoIdentityWrapper() + if not wrapper.is_enabled: + logger.info("backfill.skipped", reason="dynamo_disabled") + return + + for project in get_projects_to_process(): + log_comment = ( + "flagsmith:segment_membership:backfill" + f":org_{project.organisation_id}" + f":project_{project.id}" + ) + with open_clickhouse_cursor(log_comment=log_comment) as cursor: + for env in project.environments.all(): + env_key = env.api_key + row_count = 0 + try: + with flagsmith_segment_membership_backfill_duration_seconds.time(): + for batch in batched( + wrapper.iter_all_items_paginated(env_key), + _INSERT_BATCH_SIZE, + ): + rows = [ + map_identity_document_to_clickhouse_row( + env_key, cast(DynamoIdentity, doc) + ) + for doc in batch + ] + # Django's CursorWrapper stub forbids dicts in + # the params sequence; clickhouse-driver accepts + # them as JSON-column payloads. + cursor.executemany(_INSERT_IDENTITIES_SQL, rows) # type: ignore[arg-type] + row_count += len(rows) + except Exception: + logger.exception( + "backfill.environment.failed", + project__id=project.id, + environment__id=env.id, + ) + continue + flagsmith_segment_membership_backfill_identities_total.inc(row_count) + logger.info( + "backfill.environment.completed", + project__id=project.id, + environment__id=env.id, + rows__count=row_count, + ) + refresh_project_segment_counts.delay(args=(project.id,)) + + +@register_task_handler( + # ~2x the expected legitimate ceiling (a single UNION ALL aggregation + # against IDENTITIES); widen on real data if this starts false-firing. + timeout=timedelta(minutes=10), +) +def refresh_project_segment_counts(project_id: int) -> None: + """Compute per-segment match counts for one project and upsert into + `SegmentMembershipCount`. Re-checks the org flag so a stale fan-out + skips orgs disabled since dispatch.""" + if not settings.CLICKHOUSE_ENABLED: + logger.info( + "refresh.project.skipped", + project__id=project_id, + reason="clickhouse_not_configured", + ) + return + + project = Project.objects.select_related("organisation").get(pk=project_id) + if not is_membership_enabled(project.organisation): + logger.info( + "refresh.project.skipped", + project__id=project_id, + reason="ff_disabled", + ) + return + + log_comment = ( + "flagsmith:segment_membership:refresh" + f":org_{project.organisation_id}" + f":project_{project.id}" + ) + with ( + flagsmith_segment_membership_refresh_duration_seconds.time(), + open_clickhouse_cursor(log_comment=log_comment) as cursor, + ): + try: + membership_counts = compute_segment_counts_for_project(project, cursor) + except Exception: + flagsmith_segment_membership_refresh_failures_total.inc() + logger.exception("refresh.project.failed", project__id=project_id) + return + + now = timezone.now() + for m in membership_counts: + m.last_synced_at = now + SegmentMembershipCount.objects.bulk_create( + membership_counts, + update_conflicts=True, + unique_fields=["segment", "environment"], + update_fields=["count", "last_synced_at"], + ) + logger.info( + "refresh.project.completed", + project__id=project_id, + membership_counts__count=len(membership_counts), + ) diff --git a/api/segments/serializers.py b/api/segments/serializers.py index 6415d26bde73..fbd4d46fd611 100644 --- a/api/segments/serializers.py +++ b/api/segments/serializers.py @@ -10,6 +10,7 @@ from edge_api.utils import is_edge_enabled from metadata.serializers import MetadataSerializer, MetadataSerializerMixin from projects.models import Project +from segment_membership.models import SegmentMembershipCount from segments.models import Condition, Segment, SegmentRule logger = structlog.get_logger(__name__) @@ -17,6 +18,15 @@ DictList = list[dict[str, Any]] +class SegmentMembershipCountSerializer( + serializers.ModelSerializer[SegmentMembershipCount] +): + class Meta: + model = SegmentMembershipCount + fields = ["environment", "count", "last_synced_at"] + read_only_fields = ["environment", "count", "last_synced_at"] + + class ConditionSerializer(serializers.ModelSerializer[Condition]): delete = serializers.BooleanField( write_only=True, @@ -82,6 +92,7 @@ class Meta: class SegmentSerializer(MetadataSerializerMixin, WritableNestedModelSerializer): rules = SegmentRuleSerializer(many=True, required=True, allow_empty=False) metadata = MetadataSerializer(required=False, many=True) + membership_counts = SegmentMembershipCountSerializer(many=True, read_only=True) def __init__(self, *args: Any, **kwargs: Any) -> None: """ @@ -112,7 +123,9 @@ class Meta: "version_of", "rules", "metadata", + "membership_counts", ] + read_only_fields = ["membership_counts"] def validate(self, attrs: dict[str, Any]) -> dict[str, Any]: attrs = super().validate(attrs) diff --git a/api/segments/views.py b/api/segments/views.py index ca8f421eb03b..1e66967a634d 100644 --- a/api/segments/views.py +++ b/api/segments/views.py @@ -108,6 +108,7 @@ def get_queryset(self): # type: ignore[no-untyped-def] # TODO: at the moment, the UI only shows the name and description of the segment in the list view. # we shouldn't return all of the rules and conditions in the list view. queryset = queryset.prefetch_related( + "membership_counts", "rules", "rules__conditions", "rules__rules", diff --git a/api/tests/integration/segments/__init__.py b/api/tests/integration/segments/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/api/tests/integration/segments/test_segment_membership_field.py b/api/tests/integration/segments/test_segment_membership_field.py new file mode 100644 index 000000000000..f1f33eb22ca4 --- /dev/null +++ b/api/tests/integration/segments/test_segment_membership_field.py @@ -0,0 +1,49 @@ +from typing import Any + +from rest_framework import status +from rest_framework.test import APIClient + +from segment_membership.models import SegmentMembershipCount + + +def test_get_segment__no_memberships__returns_empty_list( + admin_client: APIClient, + project: int, + segment: int, +) -> None: + # Given / When + response = admin_client.get(f"/api/v1/projects/{project}/segments/{segment}/") + + # Then + assert response.status_code == status.HTTP_200_OK + body: dict[str, Any] = response.json() + assert body["membership_counts"] == [] + + +def test_get_segment__one_membership_per_environment__returns_per_env_counts( + admin_client: APIClient, + project: int, + segment: int, + environment: int, +) -> None: + # Given + SegmentMembershipCount.objects.create( + segment_id=segment, + environment_id=environment, + count=42, + last_synced_at="2026-05-01T00:00:00Z", + ) + + # When + response = admin_client.get(f"/api/v1/projects/{project}/segments/{segment}/") + + # Then + assert response.status_code == status.HTTP_200_OK + body: dict[str, Any] = response.json() + assert body["membership_counts"] == [ + { + "environment": environment, + "count": 42, + "last_synced_at": "2026-05-01T00:00:00Z", + } + ] diff --git a/api/tests/unit/app/test_unit_app_routers.py b/api/tests/unit/app/test_unit_app_routers.py index f2505c0d5ecc..79b8b94c5990 100644 --- a/api/tests/unit/app/test_unit_app_routers.py +++ b/api/tests/unit/app/test_unit_app_routers.py @@ -106,3 +106,109 @@ def test_analytics_router_allow_migrate__given_db_and_app_label__returns_expecte # Then assert result is expected + + +@pytest.mark.parametrize( + ["given_app_label", "expected_db"], + [ + ("clickhouse", "clickhouse"), + ("another_app", None), + ], +) +def test_clickhouse_router_db_for_read__given_app_label__returns_expected_db( + given_app_label: str, + expected_db: str | None, +) -> None: + # Given + class ClickHouseModel(models.Model): + class Meta: + app_label = given_app_label + + router = routers.ClickHouseRouter() + + # When + db = router.db_for_read(ClickHouseModel) + + # Then + assert db == expected_db + + +@pytest.mark.parametrize( + ["model_app_label", "expected_db"], + [ + ("clickhouse", "clickhouse"), + ("another_app", None), + ], +) +def test_clickhouse_router_db_for_write__given_app_label__returns_expected_db( + model_app_label: str, + expected_db: str | None, +) -> None: + # Given + class MyModel(models.Model): + class Meta: + app_label = model_app_label + + router = routers.ClickHouseRouter() + + # When + db = router.db_for_write(MyModel) + + # Then + assert db == expected_db + + +@pytest.mark.parametrize( + ["model1_app_label", "model2_app_label", "expected"], + [ + ("clickhouse", "clickhouse", False), + ("clickhouse", "another_app", False), + ("another_app", "clickhouse", False), + ("another_app", "yet_another_app", None), + ], +) +def test_clickhouse_router_allow_relation__given_app_labels__returns_expected( + model1_app_label: str, + model2_app_label: str, + expected: bool | None, +) -> None: + # Given + class MyModel1(models.Model): + class Meta: + app_label = model1_app_label + + class MyModel2(models.Model): + class Meta: + app_label = model2_app_label + + router = routers.ClickHouseRouter() + + # When + result = router.allow_relation(MyModel1(), MyModel2()) + + # Then + assert result is expected + + +@pytest.mark.parametrize( + ["db_name", "app_label", "expected"], + [ + ("clickhouse", "clickhouse", True), + ("clickhouse", "another_app", False), + ("default", "clickhouse", False), + ("default", "another_app", None), + ], +) +def test_clickhouse_router_allow_migrate__given_db_and_app_label__returns_expected( + db_name: str, + app_label: str, + expected: bool | None, +) -> None: + # Given + router = routers.ClickHouseRouter() + + # When + result = router.allow_migrate(db_name, app_label) + + # Then + assert result is expected diff --git a/api/tests/unit/segment_membership/__init__.py b/api/tests/unit/segment_membership/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/api/tests/unit/segment_membership/test_unit_segment_membership_mappers.py b/api/tests/unit/segment_membership/test_unit_segment_membership_mappers.py new file mode 100644 index 000000000000..bd3cf464a27e --- /dev/null +++ b/api/tests/unit/segment_membership/test_unit_segment_membership_mappers.py @@ -0,0 +1,97 @@ +from decimal import Decimal + +import pytest +from flagsmith_schemas.dynamodb import Identity as DynamoIdentity + +from segment_membership.mappers import map_identity_document_to_clickhouse_row + +UUID_A = "f47ac10b-58cc-4372-a567-0e02b2c3d479" + + +@pytest.mark.parametrize( + "doc,expected", + [ + pytest.param( + { + "identity_uuid": UUID_A, + "identifier": "alice", + "environment_api_key": "env-key", + "composite_key": "env_x_alice", + "created_date": "2026-05-08T00:00:00Z", + "identity_traits": [ + {"trait_key": "plan", "trait_value": "growth"}, + ], + }, + ("env-key", "alice", "env_x_alice", {"plan": "growth"}), + id="single string trait", + ), + pytest.param( + { + "identity_uuid": UUID_A, + "identifier": "alice", + "environment_api_key": "env-key", + "composite_key": "env_x_alice", + "created_date": "2026-05-08T00:00:00Z", + "identity_traits": [], + }, + ("env-key", "alice", "env_x_alice", None), + id="empty traits collapse to NULL", + ), + pytest.param( + { + "identity_uuid": UUID_A, + "identifier": "alice", + "environment_api_key": "env-key", + "composite_key": "env_x_alice", + "created_date": "2026-05-08T00:00:00Z", + "identity_traits": [ + {"trait_key": "age", "trait_value": Decimal("18")}, + ], + }, + ("env-key", "alice", "env_x_alice", {"age": 18}), + id="whole-number Decimal narrows to int", + ), + pytest.param( + { + "identity_uuid": UUID_A, + "identifier": "alice", + "environment_api_key": "env-key", + "composite_key": "env_x_alice", + "created_date": "2026-05-08T00:00:00Z", + "identity_traits": [ + {"trait_key": "score", "trait_value": Decimal("1.5")}, + ], + }, + ("env-key", "alice", "env_x_alice", {"score": 1.5}), + id="fractional Decimal narrows to float", + ), + pytest.param( + { + "identity_uuid": UUID_A, + "identifier": "alice", + "environment_api_key": "env-key", + "composite_key": "env_x_alice", + "created_date": "2026-05-08T00:00:00Z", + "identity_traits": [ + {"trait_key": "plan", "trait_value": "growth"}, + {"trait_key": "team", "trait_value": "alpha"}, + ], + }, + ( + "env-key", + "alice", + "env_x_alice", + {"plan": "growth", "team": "alpha"}, + ), + id="multiple traits flatten to a single dict", + ), + ], +) +def test_map_identity_document_to_clickhouse_row__cases__return_expected( + doc: DynamoIdentity, + expected: tuple[str, str, str, dict[str, object] | None], +) -> None: + # Given a Dynamo identity document + # When mapped onto an IDENTITIES row + # Then it lines up positionally with the IDENTITIES schema + assert map_identity_document_to_clickhouse_row("env-key", doc) == expected diff --git a/api/tests/unit/segment_membership/test_unit_segment_membership_services.py b/api/tests/unit/segment_membership/test_unit_segment_membership_services.py new file mode 100644 index 000000000000..fe33d3d74554 --- /dev/null +++ b/api/tests/unit/segment_membership/test_unit_segment_membership_services.py @@ -0,0 +1,222 @@ +from unittest.mock import MagicMock + +from pytest_mock import MockerFixture + +from environments.models import Environment +from organisations.models import Organisation +from projects.models import Project +from segment_membership.services import ( + compute_segment_counts_for_project, + get_projects_to_process, + is_membership_enabled, + open_clickhouse_cursor, +) +from segments.models import Segment, SegmentRule +from tests.types import EnableFeaturesFixture + + +def test_is_membership_enabled__flag_off__returns_false( + organisation: Organisation, +) -> None: + # Given / When + # Then + assert is_membership_enabled(organisation) is False + + +def test_is_membership_enabled__flag_on__returns_true( + organisation: Organisation, + enable_features: EnableFeaturesFixture, +) -> None: + # Given + enable_features("segment_membership_inspection") + + # When / Then + assert is_membership_enabled(organisation) is True + + +def test_open_clickhouse_cursor__no_log_comment__yields_cursor( + mocker: MockerFixture, +) -> None: + # Given + cursor = MagicMock() + connections = mocker.patch("segment_membership.services.connections") + connections.__getitem__.return_value.cursor.return_value.__enter__.return_value = ( + cursor + ) + + # When + with open_clickhouse_cursor() as opened: + assert opened is cursor + + # Then + cursor.cursor.set_settings.assert_not_called() + + +def test_open_clickhouse_cursor__with_log_comment__sets_session_attribution( + mocker: MockerFixture, +) -> None: + # Given + cursor = MagicMock() + connections = mocker.patch("segment_membership.services.connections") + connections.__getitem__.return_value.cursor.return_value.__enter__.return_value = ( + cursor + ) + + # When + with open_clickhouse_cursor( + log_comment="flagsmith:segment_membership:refresh:org_1:project_2" + ): + pass + + # Then the comment lands as a clickhouse-driver session setting so every + # query the cursor issues is attributable in CH's query_log. + cursor.cursor.set_settings.assert_called_once_with( + {"log_comment": "flagsmith:segment_membership:refresh:org_1:project_2"} + ) + + +def test_get_projects_to_process__no_canonical_segments__yields_nothing( + project: Project, +) -> None: + # Given / When + # Then + assert list(get_projects_to_process()) == [] + + +def test_get_projects_to_process__ff_disabled__skips_organisation( + project: Project, + segment: Segment, +) -> None: + # Given / When + # Then + assert list(get_projects_to_process()) == [] + + +def test_get_projects_to_process__ff_enabled__yields_project( + project: Project, + segment: Segment, + enable_features: EnableFeaturesFixture, +) -> None: + # Given + enable_features("segment_membership_inspection") + + # When / Then + assert list(get_projects_to_process()) == [project] + + +def test_get_projects_to_process__multiple_segments_per_project__yields_project_once( + project: Project, + segment: Segment, + enable_features: EnableFeaturesFixture, +) -> None: + # Given + enable_features("segment_membership_inspection") + Segment.objects.create(name="another", project=project) + + # When / Then + assert list(get_projects_to_process()) == [project] + + +def test_compute_segment_counts_for_project__no_segments__returns_empty( + project: Project, +) -> None: + # Given + cursor = MagicMock() + + # When + result = compute_segment_counts_for_project(project, cursor) + + # Then + assert result == [] + cursor.execute.assert_not_called() + + +def test_compute_segment_counts_for_project__no_environments__returns_empty( + project: Project, + segment: Segment, +) -> None: + # Given + project.environments.all().delete() + cursor = MagicMock() + + # When + result = compute_segment_counts_for_project(project, cursor) + + # Then + assert result == [] + cursor.execute.assert_not_called() + + +def test_compute_segment_counts_for_project__one_segment__returns_membership_instances( + project: Project, + environment: Environment, + segment: Segment, + segment_rule: SegmentRule, + mocker: MockerFixture, +) -> None: + # Given + mocker.patch( + "segment_membership.services.translate_segment", + return_value="TRUE", + ) + cursor = MagicMock() + cursor.fetchall.return_value = [(segment.id, environment.api_key, 7)] + + # When + result = compute_segment_counts_for_project(project, cursor) + + # Then + [membership] = result + assert membership.segment_id == segment.id + assert membership.environment_id == environment.id + assert membership.count == 7 + assert membership.last_synced_at is None + sql = cursor.execute.call_args.args[0] + assert f"SELECT {segment.id} AS segment_id" in sql + # FINAL forces ReplacingMergeTree dedup at read time. + assert "FROM IDENTITIES AS i FINAL" in sql + assert "GROUP BY i.environment_id" in sql + + +def test_compute_segment_counts_for_project__unknown_env_key_in_row__skips( + project: Project, + environment: Environment, + segment: Segment, + segment_rule: SegmentRule, + mocker: MockerFixture, +) -> None: + # Given + mocker.patch( + "segment_membership.services.translate_segment", + return_value="TRUE", + ) + cursor = MagicMock() + cursor.fetchall.return_value = [(segment.id, "ghost-env", 99)] + + # When + result = compute_segment_counts_for_project(project, cursor) + + # Then + assert result == [] + + +def test_compute_segment_counts_for_project__untranslatable_segment__skips( + project: Project, + environment: Environment, + segment: Segment, + segment_rule: SegmentRule, + mocker: MockerFixture, +) -> None: + # Given + mocker.patch( + "segment_membership.services.translate_segment", + return_value=None, + ) + cursor = MagicMock() + + # When + result = compute_segment_counts_for_project(project, cursor) + + # Then + assert result == [] + cursor.execute.assert_not_called() diff --git a/api/tests/unit/segment_membership/test_unit_segment_membership_tasks.py b/api/tests/unit/segment_membership/test_unit_segment_membership_tasks.py new file mode 100644 index 000000000000..9a57ae884f2a --- /dev/null +++ b/api/tests/unit/segment_membership/test_unit_segment_membership_tasks.py @@ -0,0 +1,297 @@ +from unittest.mock import MagicMock + +from pytest_django.fixtures import SettingsWrapper +from pytest_mock import MockerFixture +from pytest_structlog import StructuredLogCapture + +from environments.models import Environment +from projects.models import Project +from segment_membership import tasks +from segment_membership.models import SegmentMembershipCount +from segment_membership.tasks import ( + backfill_identities_to_clickhouse, + refresh_project_segment_counts, +) +from segments.models import Segment +from tests.types import EnableFeaturesFixture + + +def test_backfill_identities_to_clickhouse__no_clickhouse_creds__skips( + mocker: MockerFixture, + settings: SettingsWrapper, + log: StructuredLogCapture, +) -> None: + # Given + settings.CLICKHOUSE_ENABLED = False + spy = mocker.patch.object(tasks, "open_clickhouse_cursor") + + # When + backfill_identities_to_clickhouse() + + # Then + spy.assert_not_called() + assert any(e["event"] == "backfill.skipped" for e in log.events) + + +def test_backfill_identities_to_clickhouse__dynamo_disabled__skips( + mocker: MockerFixture, + settings: SettingsWrapper, +) -> None: + # Given + settings.CLICKHOUSE_ENABLED = True + spy = mocker.patch.object(tasks, "open_clickhouse_cursor") + mocker.patch.object( + tasks, + "DynamoIdentityWrapper", + return_value=MagicMock(is_enabled=False), + ) + + # When + backfill_identities_to_clickhouse() + + # Then + spy.assert_not_called() + + +def test_backfill_identities_to_clickhouse__happy_path__bulk_inserts( + mocker: MockerFixture, + settings: SettingsWrapper, + project: Project, + environment: Environment, + segment: Segment, + enable_features: EnableFeaturesFixture, + log: StructuredLogCapture, +) -> None: + # Given + enable_features("segment_membership_inspection") + settings.CLICKHOUSE_ENABLED = True + cursor = MagicMock() + open_cursor = mocker.patch.object(tasks, "open_clickhouse_cursor") + open_cursor.return_value.__enter__.return_value = cursor + refresh_dispatch = mocker.patch.object(tasks, "refresh_project_segment_counts") + wrapper = MagicMock(is_enabled=True) + wrapper.iter_all_items_paginated.return_value = iter( + [ + { + "identity_uuid": "f47ac10b-58cc-4372-a567-0e02b2c3d479", + "identifier": "a", + "composite_key": "k1", + "environment_api_key": environment.api_key, + "created_date": "2026-05-08T00:00:00Z", + "identity_traits": [], + }, + { + "identity_uuid": "550e8400-e29b-41d4-a716-446655440000", + "identifier": "b", + "composite_key": "k2", + "environment_api_key": environment.api_key, + "created_date": "2026-05-08T00:00:00Z", + "identity_traits": [], + }, + ] + ) + mocker.patch.object(tasks, "DynamoIdentityWrapper", return_value=wrapper) + + # When + backfill_identities_to_clickhouse() + + # Then + open_cursor.assert_called_with( + log_comment=( + f"flagsmith:segment_membership:backfill" + f":org_{project.organisation_id}" + f":project_{project.id}" + ) + ) + sql, rows_arg = cursor.executemany.call_args.args + assert sql == ( + "INSERT INTO IDENTITIES " + "(environment_id, identifier, identity_key, traits) VALUES" + ) + assert {row[0] for row in rows_arg} == {environment.api_key} + assert {row[1] for row in rows_arg} == {"a", "b"} + assert any( + e["event"] == "backfill.environment.completed" and e["rows__count"] == 2 + for e in log.events + ) + refresh_dispatch.delay.assert_called_once_with(args=(project.id,)) + + +def test_backfill_identities_to_clickhouse__insert_fails__logs_and_continues( + mocker: MockerFixture, + settings: SettingsWrapper, + project: Project, + environment: Environment, + segment: Segment, + enable_features: EnableFeaturesFixture, + log: StructuredLogCapture, +) -> None: + # Given + enable_features("segment_membership_inspection") + settings.CLICKHOUSE_ENABLED = True + cursor = MagicMock() + cursor.executemany.side_effect = RuntimeError("boom") + open_cursor = mocker.patch.object(tasks, "open_clickhouse_cursor") + open_cursor.return_value.__enter__.return_value = cursor + wrapper = MagicMock(is_enabled=True) + wrapper.iter_all_items_paginated.return_value = iter( + [ + { + "identity_uuid": "f47ac10b-58cc-4372-a567-0e02b2c3d479", + "identifier": "a", + "composite_key": "k1", + "environment_api_key": environment.api_key, + "created_date": "2026-05-08T00:00:00Z", + "identity_traits": [], + } + ] + ) + mocker.patch.object(tasks, "DynamoIdentityWrapper", return_value=wrapper) + + # When + backfill_identities_to_clickhouse() + + # Then + assert any(e["event"] == "backfill.environment.failed" for e in log.events) + + +def test_backfill_identities_to_clickhouse__multiple_projects__fans_out_refresh_per_project( + mocker: MockerFixture, + settings: SettingsWrapper, + project: Project, + project_b: Project, + segment: Segment, + enable_features: EnableFeaturesFixture, +) -> None: + # Given + enable_features("segment_membership_inspection") + Segment.objects.create(name="seg-b", project=project_b) + settings.CLICKHOUSE_ENABLED = True + cursor = MagicMock() + open_cursor = mocker.patch.object(tasks, "open_clickhouse_cursor") + open_cursor.return_value.__enter__.return_value = cursor + refresh_dispatch = mocker.patch.object(tasks, "refresh_project_segment_counts") + wrapper = MagicMock(is_enabled=True) + wrapper.iter_all_items_paginated.return_value = iter([]) + mocker.patch.object(tasks, "DynamoIdentityWrapper", return_value=wrapper) + + # When + backfill_identities_to_clickhouse() + + # Then + dispatched_ids = { + call.kwargs["args"][0] for call in refresh_dispatch.delay.call_args_list + } + assert dispatched_ids == {project.id, project_b.id} + + +def test_refresh_project_segment_counts__no_clickhouse_creds__skips( + mocker: MockerFixture, + settings: SettingsWrapper, + project: Project, + log: StructuredLogCapture, +) -> None: + # Given + settings.CLICKHOUSE_ENABLED = False + spy = mocker.patch.object(tasks, "open_clickhouse_cursor") + + # When + refresh_project_segment_counts(project.id) + + # Then + spy.assert_not_called() + assert any( + e["event"] == "refresh.project.skipped" + and e["reason"] == "clickhouse_not_configured" + for e in log.events + ) + + +def test_refresh_project_segment_counts__ff_disabled__skips( + mocker: MockerFixture, + settings: SettingsWrapper, + project: Project, + log: StructuredLogCapture, +) -> None: + # Given + settings.CLICKHOUSE_ENABLED = True + spy = mocker.patch.object(tasks, "open_clickhouse_cursor") + + # When + refresh_project_segment_counts(project.id) + + # Then + spy.assert_not_called() + assert any( + e["event"] == "refresh.project.skipped" and e["reason"] == "ff_disabled" + for e in log.events + ) + + +def test_refresh_project_segment_counts__compute_fails__logs( + mocker: MockerFixture, + settings: SettingsWrapper, + project: Project, + segment: Segment, + enable_features: EnableFeaturesFixture, + log: StructuredLogCapture, +) -> None: + # Given + enable_features("segment_membership_inspection") + settings.CLICKHOUSE_ENABLED = True + cursor = MagicMock() + open_cursor = mocker.patch.object(tasks, "open_clickhouse_cursor") + open_cursor.return_value.__enter__.return_value = cursor + mocker.patch.object( + tasks, "compute_segment_counts_for_project", side_effect=RuntimeError("boom") + ) + + # When + refresh_project_segment_counts(project.id) + + # Then + assert any(e["event"] == "refresh.project.failed" for e in log.events) + + +def test_refresh_project_segment_counts__counts_returned__upserts_per_env_rows( + mocker: MockerFixture, + settings: SettingsWrapper, + project: Project, + environment: Environment, + segment: Segment, + enable_features: EnableFeaturesFixture, +) -> None: + # Given + enable_features("segment_membership_inspection") + settings.CLICKHOUSE_ENABLED = True + cursor = MagicMock() + open_cursor = mocker.patch.object(tasks, "open_clickhouse_cursor") + open_cursor.return_value.__enter__.return_value = cursor + mocker.patch.object( + tasks, + "compute_segment_counts_for_project", + return_value=[ + SegmentMembershipCount( + segment_id=segment.id, + environment_id=environment.id, + count=42, + ), + ], + ) + + # When + refresh_project_segment_counts(project.id) + + # Then + membership = SegmentMembershipCount.objects.get( + segment=segment, environment=environment + ) + assert membership.count == 42 + assert membership.last_synced_at is not None + open_cursor.assert_called_once_with( + log_comment=( + f"flagsmith:segment_membership:refresh" + f":org_{project.organisation_id}" + f":project_{project.id}" + ) + ) diff --git a/api/tests/unit/segments/test_unit_segments_views.py b/api/tests/unit/segments/test_unit_segments_views.py index 80cb26e3679a..c015d2e348f1 100644 --- a/api/tests/unit/segments/test_unit_segments_views.py +++ b/api/tests/unit/segments/test_unit_segments_views.py @@ -594,8 +594,8 @@ def test_get_segment_by_uuid__existing_segment__returns_segment_data( # type: i @pytest.mark.parametrize( "client, num_queries", [ - (lazy_fixture("admin_master_api_key_client"), 12), - (lazy_fixture("admin_client"), 14), + (lazy_fixture("admin_master_api_key_client"), 13), + (lazy_fixture("admin_client"), 15), ], ) def test_list_segments__without_rbac__expected_num_queries( @@ -651,8 +651,8 @@ def test_list_segments__system_segment_exists__excludes_system_segment( @pytest.mark.parametrize( "client, num_queries", [ - (lazy_fixture("admin_master_api_key_client"), 12), - (lazy_fixture("admin_client"), 15), + (lazy_fixture("admin_master_api_key_client"), 13), + (lazy_fixture("admin_client"), 16), ], ) def test_list_segments__with_rbac__expected_num_queries( diff --git a/api/tests/unit/util/test_util.py b/api/tests/unit/util/test_util.py index 4f35ca8c098a..4aa75833918c 100644 --- a/api/tests/unit/util/test_util.py +++ b/api/tests/unit/util/test_util.py @@ -1,6 +1,6 @@ import pytest -from util.util import iter_chunked_concat, iter_paired_chunks +from util.util import batched, iter_chunked_concat, iter_paired_chunks def test_iter_paired_chunks__both_empty__returns_empty_list() -> None: @@ -121,3 +121,24 @@ def test_iter_chunked_concat__various_inputs__returns_expected_chunks( # Then assert list(result) == expected_result + + +def test_batched__empty_iterable__yields_nothing() -> None: + # Given an empty iterable + # When batched + # Then no batches are yielded + assert list(batched([], 3)) == [] + + +def test_batched__exact_multiple__yields_full_batches() -> None: + # Given an iterable whose length is a multiple of the batch size + # When batched + # Then every batch is full + assert list(batched(range(6), 2)) == [[0, 1], [2, 3], [4, 5]] + + +def test_batched__remainder__yields_smaller_final_batch() -> None: + # Given an iterable whose length isn't a multiple of the batch size + # When batched + # Then the final batch carries the remainder + assert list(batched([1, 2, 3, 4, 5], 2)) == [[1, 2], [3, 4], [5]] diff --git a/api/util/util.py b/api/util/util.py index 37cded499214..157fb11e3170 100644 --- a/api/util/util.py +++ b/api/util/util.py @@ -93,3 +93,13 @@ def truncate( separated by a delimiter. """ return delimiter.join([value[:ends_len], value[-ends_len:]]) + + +def batched(iterable: Iterable[T], size: int) -> Generator[list[T], None, None]: + """Yield consecutive batches of `size` items from `iterable`. The + final batch may be smaller. + + Backport from Python 3.12.""" + iterator = iter(iterable) + while batch := list(islice(iterator, size)): + yield batch diff --git a/api/uv.lock b/api/uv.lock index 6708a9717e40..d06dd83f271b 100644 --- a/api/uv.lock +++ b/api/uv.lock @@ -613,6 +613,58 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1a/70/e63223f8116931d365993d4a6b7ef653a4d920b41d03de7c59499962821f/click-8.1.6-py3-none-any.whl", hash = "sha256:fa244bb30b3b5ee2cae3da8f55c9e5e0c0e86093306301fb418eb9dc40fbded5", size = 97909, upload-time = "2023-07-18T20:05:12.481Z" }, ] +[[package]] +name = "clickhouse-driver" +version = "0.2.10" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pytz" }, + { name = "tzlocal" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/46/9e/d8e40b29b6269a84552441a553fc64dff28f2d7e2d92e81c6be84fe12b4c/clickhouse_driver-0.2.10.tar.gz", hash = "sha256:925fc6ecda1e5314e3f03bcb493955c068b070cdba221fb8ce27329ee8a7f71b", size = 409448, upload-time = "2025-11-10T22:49:58.764Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/13/9d/bb9ee6df5a8fb60b56f9ca76cb9b22beb6d47d45ce561dca4ef9a63c1d4a/clickhouse_driver-0.2.10-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:946daf834f021d69acbebee7abde87bd4ad5aa46fa7b7643805abc7ff487a91f", size = 212466, upload-time = "2025-11-10T22:47:33.591Z" }, + { url = "https://files.pythonhosted.org/packages/ec/e0/1ae285f4d5bb61bb62016deb38dc175a2b8cbe578dffdad5e1a5a02a176c/clickhouse_driver-0.2.10-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:3572e74cd65828f72284bf607de259c059178b44b19a93cb67766b7e7458cf8e", size = 208477, upload-time = "2025-11-10T22:47:34.925Z" }, + { url = "https://files.pythonhosted.org/packages/28/04/e2fb47a4aaf9653c9ed872e1505e997d42f834b0891351ef54169e100c5c/clickhouse_driver-0.2.10-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:767af2b2d2e02fb7abd8fc9619f8aa2de65be010d3024029d68bc9b1be564466", size = 1006131, upload-time = "2025-11-10T22:47:36.386Z" }, + { url = "https://files.pythonhosted.org/packages/e3/52/626cf3a908dde51638213a6c414e86bc66c47e384cb379c4a0533930a329/clickhouse_driver-0.2.10-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:b00005a89f0e40ec0bc313f7e131d958aa17e6af5c3260dbb5c7daf5370c5443", size = 1059838, upload-time = "2025-11-10T22:47:38.266Z" }, + { url = "https://files.pythonhosted.org/packages/12/57/a5917930760e4032e98017916bd6770308e146d44c58e450da6fa87f2d4b/clickhouse_driver-0.2.10-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a62ad120dab6bcc68b6413b7ef0dbaef75ab5ca985d490a9e1ec13d93bf33dc3", size = 1069504, upload-time = "2025-11-10T22:47:40.681Z" }, + { url = "https://files.pythonhosted.org/packages/f5/08/4419ce43b27b6349fd14af0d8f5d8594d270b9bb24cbaca575bacfec630e/clickhouse_driver-0.2.10-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a9f65e71c99f0c8a64afa348977793967f897c4f731984ed54fed4eca8d375a0", size = 998284, upload-time = "2025-11-10T22:47:42.713Z" }, + { url = "https://files.pythonhosted.org/packages/36/10/edbe55be3554e2cea7c68ed1761aaa2bea0153474d81a467a0ac862b3478/clickhouse_driver-0.2.10-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c120b182ea7e9713b119ba2b518dd75503c18f26cb001a6e436326765fddd123", size = 971989, upload-time = "2025-11-10T22:47:44.642Z" }, + { url = "https://files.pythonhosted.org/packages/2f/18/d0b883af04067c70e99c22dbab1f085062ae764a063f22d4e144e833048d/clickhouse_driver-0.2.10-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:d3da6f6d05f14780f3183f8b7b23ed9826d1e0f2f73c2471037d5335b474782e", size = 1022107, upload-time = "2025-11-10T22:47:46.148Z" }, + { url = "https://files.pythonhosted.org/packages/fe/40/11446c52c5330123354f2f88151c620e1b38ca6d5131b2cc71786ec3c067/clickhouse_driver-0.2.10-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:9617c6ef154e58c6693be6e1169000957238f83d21fe20d57bb492412cc6128d", size = 1015750, upload-time = "2025-11-10T22:47:47.702Z" }, + { url = "https://files.pythonhosted.org/packages/36/9b/32abe3c76fe8494ad1642febbe4c59dfd46477e27c401d2ac8cc8a0a6117/clickhouse_driver-0.2.10-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:b8c2dbee2083295c6d9789a10cb0c967c4e123e6315e3192e4ad935cd55967c3", size = 975901, upload-time = "2025-11-10T22:47:49.136Z" }, + { url = "https://files.pythonhosted.org/packages/5f/50/0cbbe783bf14381add11a241b4ba35349f5512d1765aabde63d7175c1f0b/clickhouse_driver-0.2.10-cp311-cp311-win32.whl", hash = "sha256:293f47bd36a5b69ede39638ea8f7c1eb3fa7e55f35778a3feb9a3615749100c4", size = 190097, upload-time = "2025-11-10T22:47:50.463Z" }, + { url = "https://files.pythonhosted.org/packages/1c/12/7bb65617527b2b9a2b4ca8cc0d1ea2d4bdbd6d05a4d9d141cef0083342db/clickhouse_driver-0.2.10-cp311-cp311-win_amd64.whl", hash = "sha256:c9aeebbb0c159173de37deef494b28a4423a767cd7c4b2d596556f7a072f90a6", size = 203591, upload-time = "2025-11-10T22:47:51.562Z" }, + { url = "https://files.pythonhosted.org/packages/a9/54/0a82bcfc66bbab7a36c37b4bd8b8d088d9001da93661e278b7f6c87a3791/clickhouse_driver-0.2.10-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:1851f5f8b84ead0be6373ef001e8f09d48d45ec0628377e199a223e70417cd40", size = 213107, upload-time = "2025-11-10T22:47:53.314Z" }, + { url = "https://files.pythonhosted.org/packages/32/7b/8e526f6ffb9983c0c6d082e358df4b20fe1a9e95f453e704bc7a25ef4aab/clickhouse_driver-0.2.10-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:188775d38ff7cb36e7045441aabf3a6a8751127d8b37b6eb1b1518494eaac5bd", size = 207193, upload-time = "2025-11-10T22:47:55.146Z" }, + { url = "https://files.pythonhosted.org/packages/65/96/40f274896abf287c378575f025c602fa4e834278930dd63574ff548815c4/clickhouse_driver-0.2.10-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0ff5cba860df61845d6ae12f31d4a70ff4ae3be4e6a8a876e68af8aa4b0e45bc", size = 1046187, upload-time = "2025-11-10T22:47:58.246Z" }, + { url = "https://files.pythonhosted.org/packages/0c/80/7b6e110c3b803fa8b3f8cdba0e08553a62c5f64e5ad57e56de3ea95cd9e1/clickhouse_driver-0.2.10-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:fad865009d96de44d548f1691ed92adee971f72c001cf4466b3ba2ac7d9db47b", size = 1088806, upload-time = "2025-11-10T22:47:59.834Z" }, + { url = "https://files.pythonhosted.org/packages/c5/d6/7f77bd00fc01df9db2e573de21bbc1f66083549d004864b892877dee8a76/clickhouse_driver-0.2.10-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:cf0fe791e7c2adc0ab41d4770953c00f8a88bdd7e3ee83bb849a661a6c93d4ef", size = 1109839, upload-time = "2025-11-10T22:48:01.405Z" }, + { url = "https://files.pythonhosted.org/packages/55/f7/57a80ff9cc44a333021e2caf8d35fc23da6ec7b602bbc3bf8dfac0253a6e/clickhouse_driver-0.2.10-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c5744daafdd0ff7520c6ae95a78211a0ff5c2cfb3513a20f5602d2bc7eed580d", size = 1049773, upload-time = "2025-11-10T22:48:03.089Z" }, + { url = "https://files.pythonhosted.org/packages/f6/3e/fcf8e9cb9edc717ce6c467a9ec7c96b4495d5f8ec4859175952149fbdaa8/clickhouse_driver-0.2.10-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:f02f6c9f71ae5c06e3b760d3d9f4f758b32acf6f71504b6d90bacca9abbfec18", size = 1006817, upload-time = "2025-11-10T22:48:05.038Z" }, + { url = "https://files.pythonhosted.org/packages/95/ab/1bc25a385012c03595b91311d8341205a5790375207d80425e2285055d42/clickhouse_driver-0.2.10-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:6df571410f149e16e0a0e5529f1c2a9e41bb62b9357a3c8b0bd0647d6bb0fd1e", size = 1051047, upload-time = "2025-11-10T22:48:07.115Z" }, + { url = "https://files.pythonhosted.org/packages/fd/e1/9dd7331d08495beacf4291a6fbe5514fd0f6f8d53014121a8d70d8bd6c1e/clickhouse_driver-0.2.10-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:1e891162226a44fa169bdc996efd49b22bcf59372c35118ec5785e936fe97178", size = 1052014, upload-time = "2025-11-10T22:48:08.608Z" }, + { url = "https://files.pythonhosted.org/packages/ee/e9/af10e0ddbbd90c4ead933effff1b8914bc687bd52a70d244404db4c91529/clickhouse_driver-0.2.10-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:3a261947ba0cf0034d044c30563ad151d1cf8156a5ff419b017c423b4235e0ac", size = 1020937, upload-time = "2025-11-10T22:48:10.993Z" }, + { url = "https://files.pythonhosted.org/packages/f8/78/dbb93037939a7f9424451d10feeb0daa0d5a76163ef2e62407f4d18705ea/clickhouse_driver-0.2.10-cp312-cp312-win32.whl", hash = "sha256:d94f4db3c338ed41b28379931756852586920c5cbac1f07fcc8b7f109b3e69db", size = 190729, upload-time = "2025-11-10T22:48:12.742Z" }, + { url = "https://files.pythonhosted.org/packages/b6/86/74bddbfaa3c116da34d8c762f728c59d9842a65f202b0b746f5d9a597869/clickhouse_driver-0.2.10-cp312-cp312-win_amd64.whl", hash = "sha256:3ce051f55bdb33396f76a77702936bfa973dfd98f7cf00a72817a1e01c9b7406", size = 204141, upload-time = "2025-11-10T22:48:14.452Z" }, + { url = "https://files.pythonhosted.org/packages/76/7f/ee5d539611b782a04845545ba391208634124f67c4d57a32d4a46eadfb58/clickhouse_driver-0.2.10-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e76c1189a6273c79d906ee8274b9d3c7bf8df597ee6199cb75449ee2702deb8a", size = 210645, upload-time = "2025-11-10T22:48:15.995Z" }, + { url = "https://files.pythonhosted.org/packages/34/92/ee5a2d7a812b65d9690e46222218f33064c4bd44f3535b1ba564fb4b528b/clickhouse_driver-0.2.10-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:8be64c77d58d4a33b3c957cdb7c5a4deeac56bf93f4188dbfb5c5454eb04c985", size = 205158, upload-time = "2025-11-10T22:48:17.745Z" }, + { url = "https://files.pythonhosted.org/packages/03/00/6c532a0aea89e3d09dd4150b1df0b92e787a306b8711d54d003d18fd1ddd/clickhouse_driver-0.2.10-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:23abafd0c883ccc1baea527c1d05a6bc0c59aae6c29ae65e1b84d498b265f8c0", size = 1033476, upload-time = "2025-11-10T22:48:19.239Z" }, + { url = "https://files.pythonhosted.org/packages/ea/9b/137ea1ff9539da77cd022331ec4fa079cbefbd4ebbcb5c51bdd7dcd0bca0/clickhouse_driver-0.2.10-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:77ffe2063469c637c5e57bf0713ca1b617b612d55a8392799f97e34c353e6908", size = 1079495, upload-time = "2025-11-10T22:48:20.744Z" }, + { url = "https://files.pythonhosted.org/packages/ec/1c/e13766af7e4e174c6f17b1fbc5a078b28584f53adc91f103caacc73f569b/clickhouse_driver-0.2.10-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:df2d77779fcc1ddb68614b75bf45b8db61cf63f42a03d5624ce6922a305e609f", size = 1100658, upload-time = "2025-11-10T22:48:22.277Z" }, + { url = "https://files.pythonhosted.org/packages/41/e5/0686ad3ef1b594c16e8b13394c73ee4860fd025d70211a360f797dd7a28a/clickhouse_driver-0.2.10-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:85e46e31e4b14626571819e669341a3017376ce935d25b2cc0bfea9343b1b562", size = 1034175, upload-time = "2025-11-10T22:48:24.117Z" }, + { url = "https://files.pythonhosted.org/packages/d8/32/fea4e971297b50e5af3318fd90d400269ae1c74ad4d83a9453b89f578d3c/clickhouse_driver-0.2.10-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d7435d1ff2bc577aeedf8f01d94b5777af382484f8973a9c5018d5afd0dd175c", size = 995963, upload-time = "2025-11-10T22:48:25.824Z" }, + { url = "https://files.pythonhosted.org/packages/02/c4/d42f2b69ab5903e5bc9119b179f55c9aef79fe667f77cab4d8ae90492dcd/clickhouse_driver-0.2.10-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:b60c7e3321214eec4568811bcd953836671fa078c57f6607f236414447636de2", size = 1044626, upload-time = "2025-11-10T22:48:27.927Z" }, + { url = "https://files.pythonhosted.org/packages/78/36/043b6b2d967396172a60f10bf26de2c83248857f9a1e75b481f02218d1d7/clickhouse_driver-0.2.10-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:13fdf6571e20ac79992605ad65058296ac0f2437c1e7428a98dd6d173753119e", size = 1045772, upload-time = "2025-11-10T22:48:29.439Z" }, + { url = "https://files.pythonhosted.org/packages/0c/cf/bc5c807cbe68ce9eeac6a1997b937c81774ca86b2ab593c6efb9121a9f08/clickhouse_driver-0.2.10-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:06b6b683af086f9049d0c5e7e660fb76013439efa640e6c8ff6673622c3838fa", size = 1006716, upload-time = "2025-11-10T22:48:31.086Z" }, + { url = "https://files.pythonhosted.org/packages/d9/21/9b0e4814aa6c78d717c22cf51e3a5bd73b4f09c4078fc3f274f8a17c3658/clickhouse_driver-0.2.10-cp313-cp313-win32.whl", hash = "sha256:8fc51b0991dc9b89ea2379a0bc0995a9ed24428aad834f6af7841cc94488b82f", size = 189789, upload-time = "2025-11-10T22:48:33.039Z" }, + { url = "https://files.pythonhosted.org/packages/85/7d/46195e09d4b43f59bc9053f81f6d0c0f25e44c8ef94218fe3c972067cdf4/clickhouse_driver-0.2.10-cp313-cp313-win_amd64.whl", hash = "sha256:fbfe244d2f06824e175492a54033d8282f0ec5644db8e7e3805f7a9d076f5fcf", size = 203020, upload-time = "2025-11-10T22:48:34.874Z" }, + { url = "https://files.pythonhosted.org/packages/d7/64/6d8ea99c03e99aaf8dfe87961fd2e5baedc72d0a289cab1c5138acbe2d1b/clickhouse_driver-0.2.10-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:cb0bbad7b79ecb41f5f348528b6b0c46850d741669dd453f5fa633a8f3eff855", size = 190729, upload-time = "2025-11-10T22:49:45.635Z" }, + { url = "https://files.pythonhosted.org/packages/40/7d/9abdd95b0da0dcf6dc644336459f132575bfbdee1a4ba377195c2032c03a/clickhouse_driver-0.2.10-pp311-pypy311_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:68e96e04282d126486b3820391a8ebf1d7c32b61e5fcbd701aeeda79017349e5", size = 216933, upload-time = "2025-11-10T22:49:47.474Z" }, + { url = "https://files.pythonhosted.org/packages/5a/a4/33d4b6f1650847280265756e4d54f94730cdac082ab3f9e6518ba97502bf/clickhouse_driver-0.2.10-pp311-pypy311_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0fd9f72fcfe86e0fef0681cd124325714e92e96ad1fd675dfbeaccdfb7bd2f64", size = 219670, upload-time = "2025-11-10T22:49:49.401Z" }, + { url = "https://files.pythonhosted.org/packages/d6/c0/320b7cb5e2b6ab73e10ca6f1c5d0a2f04b42dc3acb689533d102d9f3f5e4/clickhouse_driver-0.2.10-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:e24582110f3d532b35e106583981bea0b7a3b10167c8a890a7b6ea5c7c6ce6d6", size = 192945, upload-time = "2025-11-10T22:49:51.168Z" }, +] + [[package]] name = "colorama" version = "0.4.6" @@ -881,6 +933,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/34/c8/83620d6a281487dcd97cbe28b0a3684702b3c251159a46091e25501de46d/django_axes-8.1.0-py3-none-any.whl", hash = "sha256:50117de17d189497a01d0544a9977b5136764c513085cf15138084de9dd81993", size = 73731, upload-time = "2025-12-19T19:44:04.216Z" }, ] +[[package]] +name = "django-clickhouse-backend" +version = "1.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "clickhouse-driver" }, + { name = "django" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c1/db/55e54cc0793e391ae0f7797f010771ce31348e89629d4db7dec2aebb66f8/django_clickhouse_backend-1.6.tar.gz", hash = "sha256:fd4840b8789f571838fbc8a4c323e9600fed0ee34113fa761ef7a4f231298e10", size = 76500, upload-time = "2026-01-12T06:26:10.073Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0b/56/995c790c736c2016fe0ae6418a84b65b201bd3134d81e50868d36a43f640/django_clickhouse_backend-1.6-py3-none-any.whl", hash = "sha256:6e1ddba09d7c2bcbb3dd2a9c80606c75c5b42c6f9d220104751857c925a9a5e0", size = 88911, upload-time = "2026-01-12T06:26:08.744Z" }, +] + [[package]] name = "django-cockroachdb" version = "4.2" @@ -1387,6 +1452,7 @@ dependencies = [ { name = "django" }, { name = "django-admin-sso" }, { name = "django-axes" }, + { name = "django-clickhouse-backend" }, { name = "django-cockroachdb" }, { name = "django-cors-headers" }, { name = "django-environ" }, @@ -1414,6 +1480,7 @@ dependencies = [ { name = "flagsmith" }, { name = "flagsmith-common", extra = ["common-core", "flagsmith-schemas", "task-processor"] }, { name = "flagsmith-flag-engine" }, + { name = "flagsmith-sql-flag-engine" }, { name = "google-api-python-client" }, { name = "google-re2" }, { name = "gunicorn" }, @@ -1526,6 +1593,7 @@ requires-dist = [ { name = "django", specifier = ">=5,<6" }, { name = "django-admin-sso", specifier = ">=5.2.0,<5.3.0" }, { name = "django-axes", specifier = ">=8.1.0,<9.0.0" }, + { name = "django-clickhouse-backend", specifier = ">=1.4,<2.0" }, { name = "django-cockroachdb", specifier = ">=4.2,<4.3.0" }, { name = "django-cors-headers", specifier = ">=3.5.0,<3.6.0" }, { name = "django-debug-toolbar", marker = "extra == 'dev'" }, @@ -1558,11 +1626,12 @@ requires-dist = [ { name = "email-validator", marker = "extra == 'dev'", specifier = ">=2.0.0" }, { name = "environs", specifier = ">=14.1.1,<15.0.0" }, { name = "flagsmith", specifier = ">=5.3.0,<6.0.0" }, - { name = "flagsmith-common", extras = ["common-core", "flagsmith-schemas", "task-processor"], specifier = ">=3.9.0,<4" }, + { name = "flagsmith-common", extras = ["common-core", "flagsmith-schemas", "task-processor"], specifier = ">=3.9.1,<4" }, { name = "flagsmith-common", extras = ["test-tools"], marker = "extra == 'dev'" }, { name = "flagsmith-flag-engine", specifier = ">=10.1.0,<11.0.0" }, { name = "flagsmith-ldap", marker = "extra == 'ldap'", git = "https://github.com/flagsmith/flagsmith-ldap?tag=v0.1.2" }, { name = "flagsmith-private", marker = "extra == 'private'", specifier = ">=0.6.0,<1", index = "https://flagsmith-production-084060095745.d.codeartifact.eu-west-2.amazonaws.com/pypi/flagsmith-pypi-production/simple/" }, + { name = "flagsmith-sql-flag-engine", specifier = ">=0.1.0,<0.2.0" }, { name = "google-api-python-client", specifier = ">=1.12.5,<1.13.0" }, { name = "google-re2", specifier = ">=1.0,<2.0.0" }, { name = "gunicorn", specifier = ">=23.0.0,<23.1.0" }, @@ -1630,11 +1699,11 @@ provides-extras = ["auth-controller", "private", "ldap", "workflows", "licensing [[package]] name = "flagsmith-common" -version = "3.9.0" +version = "3.9.1" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/f3/a8/c65b2989644c1a0acf45e63692b112be2b5f13a37753aba26460553cdc0d/flagsmith_common-3.9.0.tar.gz", hash = "sha256:b47b141d366a6714285a0768e08e24adbc9849400294d6fc4e6030087928d8e6", size = 59007, upload-time = "2026-05-01T11:02:08.999Z" } +sdist = { url = "https://files.pythonhosted.org/packages/77/02/6db44d9089832b0267f9b8bac73cf57eeb3769364e6a523112273ee5cbea/flagsmith_common-3.9.1.tar.gz", hash = "sha256:2b78015b290c571d20e2ba59ee621346cf7ec8340bfc01acf3620af3725d9318", size = 59166, upload-time = "2026-05-11T16:46:29.797Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/e2/7e/ddf4be1f0cbd3c53a78c44e7aa78226949f8e290e5e9cc6fb873b00bbb70/flagsmith_common-3.9.0-py3-none-any.whl", hash = "sha256:782183d05b891ed5f19bebad2f281a2ebd6f69728c3d3f524c4bebc25a654cf6", size = 96586, upload-time = "2026-05-01T11:02:07.19Z" }, + { url = "https://files.pythonhosted.org/packages/06/fd/67f602c3859eba1baf30fc1916d0997cb3541b71acea5964d47c2b4b5a3e/flagsmith_common-3.9.1-py3-none-any.whl", hash = "sha256:32bd530a32ecd0ff7a6e868341b60d7e778d56ebf0a7ed0623ecdfcabd87fef8", size = 96771, upload-time = "2026-05-11T16:46:28.271Z" }, ] [package.optional-dependencies] @@ -1713,6 +1782,19 @@ wheels = [ { url = "https://flagsmith-production-084060095745.d.codeartifact.eu-west-2.amazonaws.com/pypi/flagsmith-pypi-production/simple/flagsmith-private/0.6.0/flagsmith_private-0.6.0-py3-none-any.whl", hash = "sha256:5016964c60dc75177a5ea0d290a50a23a6b2283a654ef95c27f8325c486f0cbe" }, ] +[[package]] +name = "flagsmith-sql-flag-engine" +version = "0.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "flagsmith-flag-engine" }, + { name = "jsonpath-rfc9535" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/26/6f/9037e90f3b0cb16a25617422f6e1261eedbc3fbdb038c95a363c7a8aacaa/flagsmith_sql_flag_engine-0.1.0.tar.gz", hash = "sha256:bf6377b2d73edbe90f3bcfb8821301b58f0b9eca8a396882b39aea376cbb8c8e", size = 17853, upload-time = "2026-05-20T13:02:17.905Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/df/68/7339fbfe64c3d1bfd35d942a8df68c712f81a266b7fb2a3e98283b84c08e/flagsmith_sql_flag_engine-0.1.0-py3-none-any.whl", hash = "sha256:4c5a0671ba73ec8daab7c21592c1df0dd56c839fdf8af1387322a93038b62fbd", size = 21617, upload-time = "2026-05-20T13:02:16.331Z" }, +] + [[package]] name = "freezegun" version = "1.5.5" @@ -4036,6 +4118,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/65/58/f9c9e6be752e9fcb8b6a0ee9fb87e6e7a1f6bcab2cdc73f02bb7ba91ada0/tzdata-2024.1-py2.py3-none-any.whl", hash = "sha256:9068bc196136463f5245e51efda838afa15aaeca9903f49050dfa2679db4d252", size = 345370, upload-time = "2024-02-11T23:22:38.223Z" }, ] +[[package]] +name = "tzlocal" +version = "5.3.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "tzdata" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/8b/2e/c14812d3d4d9cd1773c6be938f89e5735a1f11a9f184ac3639b93cef35d5/tzlocal-5.3.1.tar.gz", hash = "sha256:cceffc7edecefea1f595541dbd6e990cb1ea3d19bf01b2809f362a03dd7921fd", size = 30761, upload-time = "2025-03-05T21:17:41.549Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c2/14/e2a54fabd4f08cd7af1c07030603c3356b74da07f7cc056e600436edfa17/tzlocal-5.3.1-py3-none-any.whl", hash = "sha256:eb1a66c3ef5847adf7a834f1be0800581b683b5608e74f86ecbcef8ab91bb85d", size = 18026, upload-time = "2025-03-05T21:17:39.857Z" }, +] + [[package]] name = "uritemplate" version = "3.0.1" diff --git a/docs/docs/deployment-self-hosting/observability/_events-catalogue.md b/docs/docs/deployment-self-hosting/observability/_events-catalogue.md index 1d126c0430d7..e64dc5dda4f0 100644 --- a/docs/docs/deployment-self-hosting/observability/_events-catalogue.md +++ b/docs/docs/deployment-self-hosting/observability/_events-catalogue.md @@ -327,10 +327,75 @@ Logged at `warning` from: Attributes: +### `segment_membership.backfill.environment.completed` + +Logged at `info` from: + - `api/segment_membership/tasks.py:110` + +Attributes: + - `environment.id` + - `project.id` + - `rows.count` + +### `segment_membership.backfill.environment.failed` + +Logged at `exception` from: + - `api/segment_membership/tasks.py:103` + +Attributes: + - `environment.id` + - `project.id` + +### `segment_membership.backfill.skipped` + +Logged at `info` from: + - `api/segment_membership/tasks.py:67` + - `api/segment_membership/tasks.py:72` + +Attributes: + - `reason` + +### `segment_membership.compute.segment.skipped` + +Logged at `error` from: + - `api/segment_membership/services.py:96` + +Attributes: + - `project.id` + - `reason` + - `segment.id` + +### `segment_membership.refresh.project.completed` + +Logged at `info` from: + - `api/segment_membership/tasks.py:170` + +Attributes: + - `membership_counts.count` + - `project.id` + +### `segment_membership.refresh.project.failed` + +Logged at `exception` from: + - `api/segment_membership/tasks.py:158` + +Attributes: + - `project.id` + +### `segment_membership.refresh.project.skipped` + +Logged at `info` from: + - `api/segment_membership/tasks.py:129` + - `api/segment_membership/tasks.py:138` + +Attributes: + - `project.id` + - `reason` + ### `segments.serializers.segment_revision_created` Logged at `info` from: - - `api/segments/serializers.py:142` + - `api/segments/serializers.py:155` Attributes: - `revision_id` diff --git a/docs/docs/deployment-self-hosting/observability/_metrics-catalogue.md b/docs/docs/deployment-self-hosting/observability/_metrics-catalogue.md index 6cae297b29e4..b931a958595e 100644 --- a/docs/docs/deployment-self-hosting/observability/_metrics-catalogue.md +++ b/docs/docs/deployment-self-hosting/observability/_metrics-catalogue.md @@ -70,6 +70,38 @@ Labels: - `method` - `response_status` +### `flagsmith_segment_membership_backfill_duration_seconds` + +Histogram. + +Duration of a segment-membership backfill for one environment. + +Labels: + +### `flagsmith_segment_membership_backfill_identities` + +Counter. + +Total identities mirrored from Dynamo to ClickHouse by the segment-membership backfill task across all environments. + +Labels: + +### `flagsmith_segment_membership_refresh_duration_seconds` + +Histogram. + +Duration of a single segment-membership count-refresh run for one project. + +Labels: + +### `flagsmith_segment_membership_refresh_failures` + +Counter. + +Total segment-membership refresh runs that failed for any reason. + +Labels: + ### `flagsmith_task_processor_enqueued_tasks` Counter. diff --git a/infrastructure/aws/staging/ecs-task-definition-admin-api.json b/infrastructure/aws/staging/ecs-task-definition-admin-api.json index 8544293d9f0d..b3c584dac7c8 100644 --- a/infrastructure/aws/staging/ecs-task-definition-admin-api.json +++ b/infrastructure/aws/staging/ecs-task-definition-admin-api.json @@ -295,6 +295,10 @@ { "name": "FLAGSMITH_ON_FLAGSMITH_SERVER_KEY", "valueFrom": "arn:aws:secretsmanager:eu-west-2:302456015006:secret:ECS-API-heAdoB:FLAGSMITH_ON_FLAGSMITH_SERVER_KEY::" + }, + { + "name": "CLICKHOUSE_URL", + "valueFrom": "arn:aws:secretsmanager:eu-west-2:302456015006:secret:clickhouse-url-ns26gC" } ], "logConfiguration": {