From 86188920a665df59e3359d10a0db6419c51859dd Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Thu, 23 Apr 2026 13:27:41 -0400 Subject: [PATCH 01/15] preset validation phase 1 --- bbot/core/config/files.py | 27 +- bbot/core/config/merge.py | 83 +++++ bbot/core/config/models.py | 352 ++++++++++++++++++ bbot/core/core.py | 69 ++-- bbot/core/helpers/misc.py | 27 +- bbot/core/modules.py | 116 ++++-- bbot/modules/legba.py | 7 +- bbot/modules/output/http.py | 4 +- bbot/scanner/__init__.py | 3 +- bbot/scanner/preset/__init__.py | 3 +- bbot/scanner/preset/args.py | 45 ++- bbot/scanner/preset/environ.py | 25 +- bbot/scanner/preset/preset.py | 21 +- bbot/scanner/preset/validate.py | 245 ++++++++++++ bbot/scanner/scanner.py | 2 +- bbot/test/bbot_fixtures.py | 9 +- bbot/test/conftest.py | 5 +- bbot/test/test_step_1/test_config.py | 30 +- bbot/test/test_step_1/test_presets.py | 44 +-- bbot/test/test_step_1/test_validate_preset.py | 61 +++ bbot/test/test_step_2/module_tests/base.py | 6 +- pyproject.toml | 3 +- uv.lock | 58 +-- 23 files changed, 1032 insertions(+), 213 deletions(-) create mode 100644 bbot/core/config/merge.py create mode 100644 bbot/core/config/models.py create mode 100644 bbot/scanner/preset/validate.py create mode 100644 bbot/test/test_step_1/test_validate_preset.py diff --git a/bbot/core/config/files.py b/bbot/core/config/files.py index 2be7bbaa1a..b056fdd500 100644 --- a/bbot/core/config/files.py +++ b/bbot/core/config/files.py @@ -1,7 +1,8 @@ import sys +import yaml from pathlib import Path -from omegaconf import OmegaConf +from .merge import deep_update from ...logger import log_to_stderr from ...errors import ConfigLoadError @@ -18,24 +19,32 @@ class BBOTConfigFiles: def __init__(self, core): self.core = core - def _get_config(self, filename, name="config"): + def _get_config(self, filename, name="config") -> dict: filename = Path(filename).resolve() + if not filename.exists(): + return {} try: - conf = OmegaConf.load(str(filename)) + with open(filename) as f: + conf = yaml.safe_load(f) or {} + if not isinstance(conf, dict): + raise ConfigLoadError( + f"Error parsing config at {filename}: expected a YAML mapping at the top level, " + f"got {type(conf).__name__}" + ) cli_silent = any(x in sys.argv for x in ("-s", "--silent")) if __name__ == "__main__" and not cli_silent: log_to_stderr(f"Loaded {name} from {filename}") return conf + except ConfigLoadError: + raise except Exception as e: - if filename.exists(): - raise ConfigLoadError(f"Error parsing config at {filename}:\n\n{e}") - return OmegaConf.create() + raise ConfigLoadError(f"Error parsing config at {filename}:\n\n{e}") - def get_custom_config(self): - return OmegaConf.merge( + def get_custom_config(self) -> dict: + return deep_update( self._get_config(self.config_filename, name="config"), self._get_config(self.secrets_filename, name="secrets"), ) - def get_default_config(self): + def get_default_config(self) -> dict: return self._get_config(self.defaults_filename, name="defaults") diff --git a/bbot/core/config/merge.py b/bbot/core/config/merge.py new file mode 100644 index 0000000000..735f803429 --- /dev/null +++ b/bbot/core/config/merge.py @@ -0,0 +1,83 @@ +""" +Deep-merge helpers replacing omegaconf's merge semantics. + +`deep_update(a, b)` returns a new dict that is `a` with `b` merged in: nested +dicts are merged recursively, leaf values (and lists) from `b` replace those in +`a`. This matches `OmegaConf.merge(a, b)` for BBOT's preset layering use case. +""" + +from __future__ import annotations + +from typing import Any + + +def deep_update(base: dict[str, Any], *updates: dict[str, Any]) -> dict[str, Any]: + """ + Deep-merge one or more update dicts into a copy of `base`. Last wins on + leaf conflicts; lists are replaced wholesale (not concatenated). + """ + result: dict[str, Any] = dict(base) if base else {} + for update in updates: + if not update: + continue + for k, v in update.items(): + if k in result and isinstance(result[k], dict) and isinstance(v, dict): + result[k] = deep_update(result[k], v) + else: + result[k] = v + return result + + +def dotted_get(data: dict[str, Any], path: str, default: Any = None) -> Any: + """ + Look up a dotted path in a nested dict. + + >>> dotted_get({"a": {"b": {"c": 1}}}, "a.b.c") + 1 + >>> dotted_get({"a": 1}, "a.b.c", default="x") + 'x' + """ + cursor: Any = data + for part in path.split("."): + if not isinstance(cursor, dict) or part not in cursor: + return default + cursor = cursor[part] + return cursor + + +def dotted_set(data: dict[str, Any], path: str, value: Any) -> None: + """ + Set a dotted path in a nested dict, creating intermediate dicts as needed. + + >>> d = {} + >>> dotted_set(d, "a.b.c", 1) + >>> d + {'a': {'b': {'c': 1}}} + """ + parts = path.split(".") + cursor = data + for part in parts[:-1]: + if part not in cursor or not isinstance(cursor[part], dict): + cursor[part] = {} + cursor = cursor[part] + cursor[parts[-1]] = value + + +def iter_dotted_paths(data: dict[str, Any], prefix: str = "") -> list[str]: + """ + Yield every dotted leaf path in a nested dict. + + >>> iter_dotted_paths({"a": 1, "b": {"c": 2}}) + ['a', 'b.c'] + """ + paths: list[str] = [] + for k, v in data.items(): + path = f"{prefix}.{k}" if prefix else k + if isinstance(v, dict) and v: + paths.extend(iter_dotted_paths(v, path)) + else: + paths.append(path) + return paths + + +__all__ = ["deep_update", "dotted_get", "dotted_set", "iter_dotted_paths"] diff --git a/bbot/core/config/models.py b/bbot/core/config/models.py new file mode 100644 index 0000000000..63a64df7e2 --- /dev/null +++ b/bbot/core/config/models.py @@ -0,0 +1,352 @@ +""" +Pydantic schema for BBOT's global config and preset files. + +The top-level `BBOTConfig` mirrors the structure of `defaults.yml`. Per-module +configs are validated separately at bake time against each module's own +`class Config(BaseModuleConfig)` (see `BaseModuleConfig` below). +""" + +from __future__ import annotations + +from typing import Any, Optional + +from pydantic import BaseModel, ConfigDict, Field +from pydantic_settings import BaseSettings, SettingsConfigDict + + +STRICT = ConfigDict(extra="forbid") + + +class ScopeConfig(BaseModel): + model_config = STRICT + + strict: bool = False + report_distance: int = 0 + search_distance: int = 0 + + +class DnsConfig(BaseModel): + model_config = STRICT + + disable: bool = False + minimal: bool = False + threads: int = 10 + cache_size: int = 100000 + brute_threads: int = 1000 + brute_nameservers: str = ( + "https://raw.githubusercontent.com/blacklanternsecurity/public-dns-servers/master/nameservers.txt" + ) + search_distance: int = 1 + runaway_limit: int = 5 + timeout: int = 5 + retries: int = 1 + wildcard_disable: bool = False + wildcard_ignore: list[str] = Field(default_factory=list) + wildcard_tests: int = 10 + abort_threshold: int = 10 + filter_ptrs: bool = True + debug: bool = False + omit_queries: list[str] = Field( + default_factory=lambda: [ + "SRV:mail.protection.outlook.com", + "CNAME:mail.protection.outlook.com", + "TXT:mail.protection.outlook.com", + ] + ) + + +class WebConfig(BaseModel): + model_config = STRICT + + http_proxy: Optional[str] = None + user_agent: str = ( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " + "(KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36 Edg/119.0.2151.97" + ) + user_agent_suffix: Optional[str] = None + spider_distance: int = 0 + spider_depth: int = 1 + spider_links_per_page: int = 25 + http_timeout: int = 10 + httpx_timeout: int = 5 + http_headers: dict[str, str] = Field(default_factory=dict) + http_cookies: dict[str, str] = Field(default_factory=dict) + api_retries: int = 2 + http_retries: int = 1 + httpx_retries: int = 1 + sleep_interval_429: int = Field(30, alias="429_sleep_interval") + max_sleep_interval_429: int = Field(60, alias="429_max_sleep_interval") + debug: bool = False + http_max_redirects: int = 5 + ssl_verify: bool = False + + +class EngineConfig(BaseModel): + model_config = STRICT + + debug: bool = False + + +class DepsToolConfig(BaseModel): + """Per-tool dep config, e.g. deps.ffuf.version""" + + model_config = STRICT + + version: Optional[str] = None + + +class DepsConfig(BaseModel): + model_config = STRICT + + behavior: str = "abort_on_failure" + ffuf: DepsToolConfig = Field(default_factory=lambda: DepsToolConfig(version="2.1.0")) + + +class BaseModuleConfig(BaseModel): + """ + Shared base for every module's `class Config(BaseModuleConfig)`. + Carries the three universal module options that are applied to every + module regardless of declaration. + """ + + model_config = STRICT + + batch_size: int = 10 + module_threads: int = 5 + module_timeout: int = 3600 + + +class BBOTConfig(BaseSettings): + """ + Root BBOT config. Mirrors `bbot/defaults.yml`. + + Unknown top-level keys raise ValidationError. This is what catches typos + like `scpoe:` or `moudules:` in user configs. + """ + + model_config = SettingsConfigDict( + extra="forbid", + env_prefix="BBOT_", + env_nested_delimiter="__", + populate_by_name=True, + ) + + # Basic options + home: str = "~/.bbot" + keep_scans: int = 20 + status_frequency: int = 15 + file_blobs: bool = False + folder_blobs: bool = False + + # Scope / DNS / Web / Engine / Deps + scope: ScopeConfig = Field(default_factory=ScopeConfig) + dns: DnsConfig = Field(default_factory=DnsConfig) + web: WebConfig = Field(default_factory=WebConfig) + engine: EngineConfig = Field(default_factory=EngineConfig) + deps: DepsConfig = Field(default_factory=DepsConfig) + + # Module loader paths + module_dirs: list[str] = Field(default_factory=list) + + # Module runtime + module_handle_event_timeout: int = 3600 + module_handle_batch_timeout: int = 7200 + + # Internal module toggles (these are hardcoded because they're first-class + # features of the scan pipeline; the set changes rarely) + speculate: bool = True + excavate: bool = True + aggregate: bool = True + dnsresolve: bool = True + cloudcheck: bool = True + + # URL handling + url_querystring_remove: bool = True + url_querystring_collapse: bool = True + url_extension_blacklist: list[str] = Field( + default_factory=lambda: [ + "png", + "jpg", + "bmp", + "ico", + "jpeg", + "gif", + "svg", + "webp", + "css", + "woff", + "woff2", + "ttf", + "eot", + "sass", + "scss", + "mp3", + "m4a", + "wav", + "flac", + "mp4", + "mkv", + "avi", + "wmv", + "mov", + "flv", + "webm", + ] + ) + url_extension_special: list[str] = Field(default_factory=lambda: ["js"]) + url_extension_static: list[str] = Field( + default_factory=lambda: [ + "pdf", + "doc", + "docx", + "xls", + "xlsx", + "ppt", + "pptx", + "txt", + "csv", + "xml", + "yaml", + "ini", + "log", + "conf", + "cfg", + "env", + "md", + "rtf", + "tiff", + "bmp", + "jpg", + "jpeg", + "png", + "gif", + "svg", + "ico", + "mp3", + "wav", + "flac", + "mp4", + "mov", + "avi", + "mkv", + "webm", + "zip", + "tar", + "gz", + "bz2", + "7z", + "rar", + ] + ) + + # Parameter handling + parameter_blacklist: list[str] = Field( + default_factory=lambda: [ + "__VIEWSTATE", + "__EVENTARGUMENT", + "__EVENTVALIDATION", + "__EVENTTARGET", + "__VIEWSTATEGENERATOR", + "__SCROLLPOSITIONY", + "__SCROLLPOSITIONX", + "ASP.NET_SessionId", + ".AspNetCore.Session", + "PHPSESSID", + "__cf_bm", + "f5_cspm", + ] + ) + parameter_blacklist_prefixes: list[str] = Field( + default_factory=lambda: [ + "TS01", + "BIGipServer", + "f5avr", + "incap_", + "visid_incap_", + "AWSALB", + "utm_", + "ApplicationGatewayAffinity", + "JSESSIONID", + "ARRAffinity", + ] + ) + + # Event output filter + omit_event_types: list[str] = Field( + default_factory=lambda: [ + "HTTP_RESPONSE", + "RAW_TEXT", + "URL_UNVERIFIED", + "DNS_NAME_UNRESOLVED", + "FILESYSTEM", + "WEB_PARAMETER", + "RAW_DNS_RECORD", + ] + ) + + # Interactsh + interactsh_server: Optional[str] = None + interactsh_token: Optional[str] = None + interactsh_disable: bool = False + + # Per-module configs — validated separately, per-module, at bake time. + # Stored here as raw dicts so the root validator accepts any module + # registered at preload time. + modules: dict[str, dict[str, Any]] = Field(default_factory=dict) + + +class PresetSchema(BaseModel): + """ + Schema for the top-level keys in a preset YAML file. Catches typos like + `modlues:` or `flgas:` at load time. + + `target`/`targets` and `include`/`presets` are aliases; both accepted. + `config` is validated separately as `BBOTConfig`. + """ + + model_config = ConfigDict( + extra="forbid", + populate_by_name=True, + ) + + target: Optional[list[str]] = Field(default=None) + targets: Optional[list[str]] = Field(default=None) + seeds: Optional[list[str]] = None + blacklist: Optional[list[str]] = None + + modules: Optional[list[str]] = None + output_modules: Optional[list[str]] = None + exclude_modules: Optional[list[str]] = None + flags: Optional[list[str]] = None + require_flags: Optional[list[str]] = None + exclude_flags: Optional[list[str]] = None + + config: Optional[dict[str, Any]] = None + module_dirs: Optional[list[str]] = None + + include: Optional[list[str]] = None + presets: Optional[list[str]] = None + + scan_name: Optional[str] = None + output_dir: Optional[str] = None + name: Optional[str] = None + description: Optional[str] = None + + conditions: Optional[list[str]] = None + + verbose: bool = False + debug: bool = False + silent: bool = False + + +__all__ = [ + "BBOTConfig", + "BaseModuleConfig", + "DepsConfig", + "DepsToolConfig", + "DnsConfig", + "EngineConfig", + "PresetSchema", + "ScopeConfig", + "WebConfig", +] diff --git a/bbot/core/core.py b/bbot/core/core.py index 5814052771..3084b16da6 100644 --- a/bbot/core/core.py +++ b/bbot/core/core.py @@ -1,15 +1,14 @@ import os import logging -from copy import copy +from copy import copy, deepcopy from pathlib import Path -from contextlib import suppress -from omegaconf import OmegaConf from bbot.errors import BBOTError +from .config.merge import deep_update from .multiprocess import SHARED_INTERPRETER_STATE -DEFAULT_CONFIG = None +DEFAULT_CONFIG: dict | None = None class BBOTCore: @@ -35,8 +34,8 @@ def __init__(self): self._logger = None self._files_config = None - self._config = None - self._custom_config = None + self._config: dict | None = None + self._custom_config: dict | None = None # bare minimum == logging self.logger @@ -85,22 +84,20 @@ def scans_dir(self): return self.home / "scans" @property - def config(self): + def config(self) -> dict: """ - .config is just .default_config + .custom_config merged together + .config is just .default_config + .custom_config merged together. - any new values should be added to custom_config. + Any new values should be added to custom_config. """ if self._config is None: - self._config = OmegaConf.merge(self.default_config, self.custom_config) - # set read-only flag (change .custom_config instead) - OmegaConf.set_readonly(self._config, True) + self._config = deep_update(self.default_config, self.custom_config) return self._config @property - def default_config(self): + def default_config(self) -> dict: """ - The default BBOT config (from `defaults.yml`). Read-only. + The default BBOT config (from `defaults.yml`). """ global DEFAULT_CONFIG if DEFAULT_CONFIG is None: @@ -111,16 +108,14 @@ def default_config(self): return DEFAULT_CONFIG @default_config.setter - def default_config(self, value): + def default_config(self, value: dict): # we temporarily clear out the config so it can be refreshed if/when default_config changes global DEFAULT_CONFIG self._config = None - DEFAULT_CONFIG = value - # set read-only flag (change .custom_config instead) - OmegaConf.set_readonly(DEFAULT_CONFIG, True) + DEFAULT_CONFIG = dict(value) if value else {} @property - def custom_config(self): + def custom_config(self) -> dict: """ Custom BBOT config (from `~/.config/bbot/bbot.yml`) """ @@ -131,21 +126,16 @@ def custom_config(self): return self._custom_config @custom_config.setter - def custom_config(self, value): - # we temporarily clear out the config so it can be refreshed if/when custom_config changes + def custom_config(self, value: dict): self._config = None - # ensure the modules key is always a dictionary - modules_entry = value.get("modules", None) - if modules_entry is not None and not OmegaConf.is_dict(modules_entry): - value["modules"] = {} - self._custom_config = value + self._custom_config = dict(value) if value else {} def no_secrets_config(self, config): + """Return a copy of the config with secret-looking keys removed.""" from .helpers.misc import clean_dict - with suppress(ValueError): - config = OmegaConf.to_object(config) - + if not isinstance(config, dict): + config = deepcopy(config) return clean_dict( config, *self.secrets_strings, @@ -154,11 +144,11 @@ def no_secrets_config(self, config): ) def secrets_only_config(self, config): + """Return a copy of the config containing only secret-looking keys.""" from .helpers.misc import filter_dict - with suppress(ValueError): - config = OmegaConf.to_object(config) - + if not isinstance(config, dict): + config = deepcopy(config) return filter_dict( config, *self.secrets_strings, @@ -167,23 +157,20 @@ def secrets_only_config(self, config): ) def merge_custom(self, config): - """ - Merge a config into the custom config. - """ - self.custom_config = OmegaConf.merge(self.custom_config, OmegaConf.create(config)) + """Merge a config dict into the custom config.""" + self.custom_config = deep_update(self.custom_config, dict(config) if config else {}) def merge_default(self, config): - """ - Merge a config into the default config. - """ - self.default_config = OmegaConf.merge(self.default_config, OmegaConf.create(config)) + """Merge a config dict into the default config.""" + self.default_config = deep_update(self.default_config, dict(config) if config else {}) def copy(self): """ Return a semi-shallow copy of self. (`custom_config` is copied, but `default_config` stays the same) """ core_copy = copy(self) - core_copy._custom_config = self._custom_config.copy() + core_copy._custom_config = deepcopy(self._custom_config) if self._custom_config else {} + core_copy._config = None return core_copy @property diff --git a/bbot/core/helpers/misc.py b/bbot/core/helpers/misc.py index 8e5bb23a16..a6a80599fb 100644 --- a/bbot/core/helpers/misc.py +++ b/bbot/core/helpers/misc.py @@ -2786,36 +2786,23 @@ def truncate_filename(file_path, max_length=255): def get_keys_in_dot_syntax(config): - """Retrieve all keys in an OmegaConf configuration in dot notation. - - This function converts an OmegaConf configuration into a list of keys - represented in dot notation. + """Retrieve all leaf keys in a nested dict in dot notation. Args: - config (DictConfig): The OmegaConf configuration object. + config (dict): A nested dict. Returns: - List[str]: A list of keys in dot notation. + List[str]: A list of leaf keys in dot notation. Examples: - >>> config = OmegaConf.create({ - ... "web": { - ... "test": True - ... }, - ... "db": { - ... "host": "localhost", - ... "port": 5432 - ... } - ... }) - >>> get_keys_in_dot_syntax(config) + >>> get_keys_in_dot_syntax({"web": {"test": True}, "db": {"host": "localhost", "port": 5432}}) ['web.test', 'db.host', 'db.port'] """ - from omegaconf import OmegaConf - - container = OmegaConf.to_container(config, resolve=True) keys = [] def recursive_keys(d, parent_key=""): + if not isinstance(d, dict): + return for k, v in d.items(): full_key = f"{parent_key}.{k}" if parent_key else k if isinstance(v, dict): @@ -2823,7 +2810,7 @@ def recursive_keys(d, parent_key=""): else: keys.append(full_key) - recursive_keys(container) + recursive_keys(config) return keys diff --git a/bbot/core/modules.py b/bbot/core/modules.py index 1100919423..9b1e2045b7 100644 --- a/bbot/core/modules.py +++ b/bbot/core/modules.py @@ -1,15 +1,14 @@ import re import ast import sys +import yaml import atexit import pickle import logging import importlib -import omegaconf import traceback from copy import copy from pathlib import Path -from omegaconf import OmegaConf from contextlib import suppress from bbot.core import CORE @@ -34,6 +33,77 @@ bbot_code_dir = Path(__file__).parent.parent +_UNEVALUATED = object() + + +def _eval_ast_default(node): + """ + Extract a literal default value from an AST node. Returns _UNEVALUATED if + the node can't be resolved statically (e.g. `default_factory=lambda: ...` + or a non-constant expression). Preload uses this for display-time defaults + only; actual validation happens at bake time against the real pydantic + Config class. + """ + if node is None: + return _UNEVALUATED + try: + return ast.literal_eval(node) + except (ValueError, TypeError, SyntaxError): + # Recognize a few common default_factory values. + if isinstance(node, ast.Name): + return {"list": [], "dict": {}, "set": set(), "tuple": (), "str": "", "int": 0, "float": 0.0}.get( + node.id, _UNEVALUATED + ) + return _UNEVALUATED + + +def _extract_pydantic_config(config_class: ast.ClassDef) -> tuple[dict, dict]: + """ + Walk a `class Config(BaseModuleConfig):` block and extract + `(field_defaults, field_descriptions)` dicts mirroring the legacy + `options`/`options_desc` shape. + + Used by the preloader so `bbot -l` and module listing work without + importing the module (deps may be missing on the host). + """ + defaults: dict = {} + descriptions: dict = {} + for node in config_class.body: + # `model_config = ConfigDict(...)` etc. are plain assigns, not typed — skip. + if not isinstance(node, ast.AnnAssign) or not isinstance(node.target, ast.Name): + continue + name = node.target.id + if name.startswith("_"): + continue + + default = _UNEVALUATED + description = "" + + value = node.value + if isinstance(value, ast.Call) and isinstance(value.func, ast.Name) and value.func.id == "Field": + # Field(default, description="...", default_factory=..., ...) + # - first positional arg is the default, if given + if value.args: + default = _eval_ast_default(value.args[0]) + for kw in value.keywords: + if kw.arg == "default": + default = _eval_ast_default(kw.value) + elif kw.arg == "default_factory": + default = _eval_ast_default(kw.value) + elif kw.arg == "description": + with suppress(ValueError, TypeError, SyntaxError): + description = ast.literal_eval(kw.value) + elif value is not None: + default = _eval_ast_default(value) + + if default is _UNEVALUATED: + # couldn't statically determine; fall back to None so listing still works + default = None + defaults[name] = default + descriptions[name] = description + return defaults, descriptions + + class ModuleLoader: """ Main class responsible for preloading BBOT modules. @@ -196,18 +266,12 @@ def preload(self, module_dirs=None): self.flag_choices.update(set(flags)) self.__preloaded[module_name] = preloaded - config = OmegaConf.create(preloaded.get("config", {})) - self._configs[module_name] = config + self._configs[module_name] = dict(preloaded.get("config", {})) self._module_dirs_preloaded.add(module_dir) # update default config with module defaults - module_config = omegaconf.OmegaConf.create( - { - "modules": self.configs(), - } - ) - self.core.merge_default(module_config) + self.core.merge_default({"modules": self.configs()}) return new_modules @@ -254,12 +318,9 @@ def preloaded(self, type=None): return preloaded def configs(self, type=None): - configs = {} if type is not None: - configs = {k: v for k, v in self._configs.items() if self.check_type(k, type)} - else: - configs = dict(self._configs) - return OmegaConf.create(configs) + return {k: dict(v) for k, v in self._configs.items() if self.check_type(k, type)} + return {k: dict(v) for k, v in self._configs.items()} def find_and_replace(self, **kwargs): self.__preloaded = search_format_dict(self.__preloaded, **kwargs) @@ -355,15 +416,22 @@ def preload_module(self, module_file): # look for classes if type(root_element) == ast.ClassDef: for class_attr in root_element.body: + # nested `class Config(BaseModuleConfig): ...` — the new pydantic-based schema + if type(class_attr) == ast.ClassDef and class_attr.name == "Config": + config_fields, config_descs = _extract_pydantic_config(class_attr) + config.update(config_fields) + options_desc.update(config_descs) + continue + if not type(class_attr) == ast.Assign: continue # class attributes that are dictionaries if type(class_attr.value) == ast.Dict: - # module options + # module options (legacy — pre-pydantic shim) if any(target.id == "options" for target in class_attr.targets): config.update(ast.literal_eval(class_attr.value)) - # module options + # module options descriptions (legacy) elif any(target.id == "options_desc" for target in class_attr.targets): options_desc.update(ast.literal_eval(class_attr.value)) # module metadata @@ -692,25 +760,25 @@ def ensure_config_files(self): + "# Please be sure to uncomment when inserting API keys, etc.\n" ) - config_obj = OmegaConf.to_object(self.core.default_config) + config_obj = dict(self.core.default_config) # ensure bbot.yml if not files.config_filename.exists(): log_to_stderr(f"Creating BBOT config at {files.config_filename}") no_secrets_config = self.core.no_secrets_config(config_obj) - yaml = OmegaConf.to_yaml(no_secrets_config) - yaml = comment_notice + "\n".join(f"# {line}" for line in yaml.splitlines()) + yaml_str = yaml.dump(no_secrets_config, sort_keys=False) + yaml_str = comment_notice + "\n".join(f"# {line}" for line in yaml_str.splitlines()) with open(str(files.config_filename), "w") as f: - f.write(yaml) + f.write(yaml_str) # ensure secrets.yml if not files.secrets_filename.exists(): log_to_stderr(f"Creating BBOT secrets at {files.secrets_filename}") secrets_only_config = self.core.secrets_only_config(config_obj) - yaml = OmegaConf.to_yaml(secrets_only_config) - yaml = comment_notice + "\n".join(f"# {line}" for line in yaml.splitlines()) + yaml_str = yaml.dump(secrets_only_config, sort_keys=False) + yaml_str = comment_notice + "\n".join(f"# {line}" for line in yaml_str.splitlines()) with open(str(files.secrets_filename), "w") as f: - f.write(yaml) + f.write(yaml_str) files.secrets_filename.chmod(0o600) diff --git a/bbot/modules/legba.py b/bbot/modules/legba.py index 27b8a6b4bd..66b73cde42 100644 --- a/bbot/modules/legba.py +++ b/bbot/modules/legba.py @@ -218,6 +218,11 @@ async def construct_command(self, host, port, protocol): "1", ] else: - cmd += ["--rate-limit", self.config.rate_limit, "--concurrency", self.config.concurrency] + cmd += [ + "--rate-limit", + str(self.config.get("rate_limit")), + "--concurrency", + str(self.config.get("concurrency")), + ] return cmd, output_path diff --git a/bbot/modules/output/http.py b/bbot/modules/output/http.py index 28fa917fc7..e2ad87d898 100644 --- a/bbot/modules/output/http.py +++ b/bbot/modules/output/http.py @@ -1,5 +1,3 @@ -from omegaconf import OmegaConf - from bbot.models.pydantic import Event from bbot.modules.output.base import BaseOutputModule @@ -34,7 +32,7 @@ async def setup(self): self.url = self.config.get("url", "") self.method = self.config.get("method", "POST") self.timeout = self.config.get("timeout", 10) - self.headers = OmegaConf.to_object(self.config.get("headers", OmegaConf.create())) + self.headers = dict(self.config.get("headers") or {}) bearer = self.config.get("bearer", "") if bearer: self.headers["Authorization"] = f"Bearer {bearer}" diff --git a/bbot/scanner/__init__.py b/bbot/scanner/__init__.py index 17338fd73e..2f5970db48 100644 --- a/bbot/scanner/__init__.py +++ b/bbot/scanner/__init__.py @@ -1,4 +1,5 @@ from .preset import Preset +from .preset import PresetValidationError, validate_preset, validate_preset_file from .scanner import Scanner -__all__ = ["Preset", "Scanner"] +__all__ = ["Preset", "PresetValidationError", "Scanner", "validate_preset", "validate_preset_file"] diff --git a/bbot/scanner/preset/__init__.py b/bbot/scanner/preset/__init__.py index 063f3659d2..e80a0f5683 100644 --- a/bbot/scanner/preset/__init__.py +++ b/bbot/scanner/preset/__init__.py @@ -1,3 +1,4 @@ from .preset import Preset +from .validate import PresetValidationError, validate_preset, validate_preset_file -__all__ = ["Preset"] +__all__ = ["Preset", "PresetValidationError", "validate_preset", "validate_preset_file"] diff --git a/bbot/scanner/preset/args.py b/bbot/scanner/preset/args.py index f99305e5d8..ae51491d8f 100644 --- a/bbot/scanner/preset/args.py +++ b/bbot/scanner/preset/args.py @@ -1,11 +1,49 @@ import re +import yaml import logging import argparse -from omegaconf import OmegaConf from bbot.errors import * +from bbot.core.config.merge import dotted_get, dotted_set from bbot.core.helpers.misc import chain_lists, get_closest_match, get_keys_in_dot_syntax + +def _parse_cli_value(raw: str): + """ + Parse the RHS of a `-c a.b.c=value` argument. + + YAML safe_load handles `true`/`false`/`null`/ints/floats and quoted strings + the way users expect when they write `web.spider_distance=2` or + `modules.stdout.event_fields='[type, data]'`. + """ + try: + return yaml.safe_load(raw) + except yaml.YAMLError: + return raw + + +def parse_dotted_cli(entries): + """ + Parse one or more `a.b.c=value` strings into a nested dict. + + Examples: + >>> parse_dotted_cli(["modules.shodan.api_key=1234"]) + {'modules': {'shodan': {'api_key': 1234}}} + >>> parse_dotted_cli(["web.spider_distance=2"]) + {'web': {'spider_distance': 2}} + """ + result: dict = {} + for entry in entries: + if "=" not in entry: + raise ValueError(f'Expected "key=value" (got {entry!r})') + path, _, raw = entry.partition("=") + path = path.strip() + if not path: + raise ValueError(f'Empty key in "{entry}"') + dotted_set(result, path, _parse_cli_value(raw.strip())) + return result + + log = logging.getLogger("bbot.presets.args") @@ -197,8 +235,7 @@ def preset_from_args(self): # CLI config options (dot-syntax) for config_arg in self.parsed.config: try: - # if that fails, try to parse as key=value syntax - args_preset.core.merge_custom(OmegaConf.from_cli([config_arg])) + args_preset.core.merge_custom(parse_dotted_cli([config_arg])) except Exception as e: raise BBOTArgumentError(f'Error parsing command-line config option: "{config_arg}": {e}') @@ -459,7 +496,7 @@ def validate(self): all_options = set(get_keys_in_dot_syntax(self.preset.core.default_config)) for c in self.parsed.config: c = c.split("=")[0].strip() - v = OmegaConf.select(self.preset.core.default_config, c, default=sentinel) + v = dotted_get(self.preset.core.default_config, c, default=sentinel) # if option isn't in the default config if v is sentinel: # skip if it's excluded from validation diff --git a/bbot/scanner/preset/environ.py b/bbot/scanner/preset/environ.py index a222dd1bb3..b7689e3788 100644 --- a/bbot/scanner/preset/environ.py +++ b/bbot/scanner/preset/environ.py @@ -1,6 +1,5 @@ import os import sys -import omegaconf from pathlib import Path from bbot.core.helpers.misc import ( @@ -33,11 +32,6 @@ def increase_limit(new_limit): increase_limit(65535) -# Custom custom omegaconf resolver to get environment variables -def env_resolver(env_name, default=None): - return os.getenv(env_name, default) - - def add_to_path(v, k="PATH", environ=None): """ Add an entry to a colon-separated PATH variable. @@ -65,26 +59,23 @@ def add_to_path(v, k="PATH", environ=None): add_to_path(local_bin_dir) -# Register the new resolver -# this allows you to substitute environment variables in your config like "${env:PATH}"" -omegaconf.OmegaConf.register_new_resolver("env", env_resolver) - - class BBOTEnviron: def __init__(self, preset): self.preset = preset def flatten_config(self, config, base="bbot"): """ - Flatten a JSON-like config into a list of environment variables: - {"modules": [{"httpx": {"timeout": 5}}]} --> "BBOT_MODULES_HTTPX_TIMEOUT=5" + Flatten a JSON-like config into a sequence of (KEY, value) env var pairs: + {"modules": {"httpx": {"timeout": 5}}} --> ("BBOT_MODULES_HTTPX_TIMEOUT", "5") + + Lists are skipped (they don't translate cleanly to env var values). """ - if type(config) == omegaconf.dictconfig.DictConfig: + if isinstance(config, dict): for k, v in config.items(): new_base = f"{base}_{k}" - if type(v) == omegaconf.dictconfig.DictConfig: + if isinstance(v, dict): yield from self.flatten_config(v, base=new_base) - elif type(v) != omegaconf.listconfig.ListConfig: + elif not isinstance(v, list): yield (new_base.upper(), str(v)) def prepare(self): @@ -129,7 +120,7 @@ def prepare(self): import urllib3 urllib3.disable_warnings() - ssl_verify = self.preset.config.get("ssl_verify", False) + ssl_verify = self.preset.config.get("web", {}).get("ssl_verify", False) global REQUESTS_PATCHED if not ssl_verify and not REQUESTS_PATCHED: diff --git a/bbot/scanner/preset/preset.py b/bbot/scanner/preset/preset.py index c6ab3a8220..3779011017 100644 --- a/bbot/scanner/preset/preset.py +++ b/bbot/scanner/preset/preset.py @@ -1,11 +1,9 @@ import os import yaml import logging -import omegaconf import traceback from copy import copy from pathlib import Path -from contextlib import suppress from .path import PRESET_PATH @@ -93,7 +91,7 @@ class Preset(metaclass=BasePreset): require_flags (set): Require modules to have these flags. When set, automatically removes offending modules. exclude_flags (set): Exclude modules that have any of these flags. When set, automatically removes offending modules. module_dirs (set): Custom directories from which to load modules (alias to `self.module_loader.module_dirs`). When set, automatically preloads contained modules. - config (omegaconf.dictconfig.DictConfig): BBOT config (alias to `core.config`) + config (dict): BBOT config (alias to `core.config`) core (BBOTCore): Local copy of BBOTCore object. verbose (bool): Whether log level is currently set to verbose. When set, updates log level for all BBOT log handlers. debug (bool): Whether log level is currently set to debug. When set, updates log level for all BBOT log handlers. @@ -249,7 +247,7 @@ def __init__( # bbot core config self.core = CORE.copy() if config is None: - config = omegaconf.OmegaConf.create({}) + config = {} # merge custom configs if specified by the user self.core.merge_custom(config) @@ -570,16 +568,14 @@ def apply_log_level(self, apply_core=False): if apply_core: self.core.logger.log_level = "CRITICAL" for key in ("verbose", "debug"): - with suppress(omegaconf.errors.ConfigKeyError): - del self.core.custom_config[key] + self.core.custom_config.pop(key, None) else: # then debug if self.debug: self.verbose = False if apply_core: self.core.logger.log_level = "DEBUG" - with suppress(omegaconf.errors.ConfigKeyError): - del self.core.custom_config["verbose"] + self.core.custom_config.pop("verbose", None) else: # finally verbose if self.verbose and apply_core: @@ -764,7 +760,7 @@ def from_yaml_file(cls, filename, _exclude=None, _log=False): except FileNotFoundError: raise PresetNotFoundError(f'Could not find preset at "{filename}" - file does not exist') preset = cls.from_dict( - omegaconf.OmegaConf.create(yaml_str), + yaml.safe_load(yaml_str) or {}, name=filename.stem, _exclude=_exclude, _log=_log, @@ -789,7 +785,7 @@ def from_yaml_string(cls, yaml_preset): >>> - portscan''' >>> preset = Preset.from_yaml_string(yaml_string) """ - return cls.from_dict(omegaconf.OmegaConf.create(yaml_preset)) + return cls.from_dict(yaml.safe_load(yaml_preset) or {}) def to_dict(self, include_target=False, full_config=False, redact_secrets=False): """ @@ -814,10 +810,9 @@ def to_dict(self, include_target=False, full_config=False, redact_secrets=False) # config if full_config: - config = self.core.config + config = dict(self.core.config) else: - config = self.core.custom_config - config = omegaconf.OmegaConf.to_object(config) + config = dict(self.core.custom_config) if redact_secrets: config = self.core.no_secrets_config(config) if config: diff --git a/bbot/scanner/preset/validate.py b/bbot/scanner/preset/validate.py new file mode 100644 index 0000000000..f1162ae795 --- /dev/null +++ b/bbot/scanner/preset/validate.py @@ -0,0 +1,245 @@ +""" +Public `validate_preset()` API for BBOT presets. + +Given a preset-as-dict (i.e. what comes out of a YAML file before it's +instantiated), return a list of human-readable errors in three layers: + +1. Top-level preset keys (e.g. `modlues:` typo) +2. Global config (e.g. `scope.strct`, `web.sslverify`) +3. Per-module config (e.g. `modules.nuclei.tgas`) + +Errors are aggregated — a user with multiple typos gets all the errors at once +rather than having to fix one at a time. The caller decides whether to raise +or just print them. +""" + +from __future__ import annotations + +import importlib +import logging +from dataclasses import dataclass +from pathlib import Path +from typing import Iterable, Optional + +from pydantic import ValidationError + +from bbot.core.config.models import BBOTConfig, BaseModuleConfig, PresetSchema + + +log = logging.getLogger("bbot.presets.validate") + + +@dataclass +class PresetValidationError: + """A single validation problem. Stringifies to a human-readable message.""" + + where: str # "preset", "config", "module:" + path: str # dotted location, e.g. "scope.strict" or "" + message: str # pydantic-derived message, reformatted + + def __str__(self) -> str: + loc = f"{self.where}:{self.path}" if self.path else self.where + return f"[{loc}] {self.message}" + + +def _format_pydantic_errors(exc: ValidationError, where: str) -> list[PresetValidationError]: + out: list[PresetValidationError] = [] + for err in exc.errors(): + path = ".".join(str(p) for p in err["loc"]) + kind = err["type"] + input_value = err.get("input") + if kind == "extra_forbidden": + msg = f"Unknown option: {path!r}" + if input_value is not None and isinstance(input_value, (str, int, bool, float)): + msg += f" (value: {input_value!r})" + elif kind in {"int_parsing", "int_type"}: + msg = f"Expected an integer, got {type(input_value).__name__}: {input_value!r}" + elif kind in {"bool_parsing", "bool_type"}: + msg = f"Expected a boolean, got {type(input_value).__name__}: {input_value!r}" + elif kind in {"string_type"}: + msg = f"Expected a string, got {type(input_value).__name__}: {input_value!r}" + elif kind == "list_type": + msg = f"Expected a list, got {type(input_value).__name__}" + elif kind == "missing": + msg = "Required option is missing" + else: + msg = err["msg"] + out.append(PresetValidationError(where=where, path=path, message=msg)) + return out + + +def _get_module_config_class(module_name: str, module_loader) -> Optional[type[BaseModuleConfig]]: + """ + Return the module's `Config` class, or None if the module doesn't declare one + (legacy `options`/`options_desc` dict modules). + """ + preloaded = module_loader.preloaded().get(module_name) + if not preloaded: + return None + module_path = Path(preloaded["path"]) + namespace = preloaded["namespace"] + full_namespace = f"{namespace}.{module_name}" + + # Re-import by path. Uses the existing importlib pattern from ModuleLoader.load_module. + spec = importlib.util.spec_from_file_location(full_namespace, str(module_path)) + if spec is None or spec.loader is None: + return None + mod = importlib.util.module_from_spec(spec) + try: + spec.loader.exec_module(mod) + except Exception as e: + log.debug(f"Could not import {module_name} for validation: {e}") + return None + + # Find the module class (same heuristic as ModuleLoader.load_module) + for attr_name in vars(mod): + value = getattr(mod, attr_name) + if not hasattr(value, "watched_events") or not hasattr(value, "produced_events"): + continue + if not isinstance(getattr(value, "watched_events"), list): + continue + if getattr(value, "__name__", "").lower() != module_name.lower(): + continue + cfg = getattr(value, "Config", None) + if cfg is not None and isinstance(cfg, type) and issubclass(cfg, BaseModuleConfig): + return cfg + return None + return None + + +def _modules_referenced(preset_dict: dict, config_dict: dict) -> set[str]: + """Every module mentioned anywhere in the preset — enabled, configured, or both.""" + names: set[str] = set() + for key in ("modules", "output_modules", "exclude_modules"): + names.update(preset_dict.get(key) or []) + modules_config = config_dict.get("modules") or {} + if isinstance(modules_config, dict): + names.update(modules_config.keys()) + return names + + +def validate_preset( + preset_dict: dict, + module_loader=None, + *, + validate_modules: bool = True, + known_modules: Optional[Iterable[str]] = None, +) -> list[PresetValidationError]: + """ + Validate a preset dict against BBOT's schemas. + + Returns a list of `PresetValidationError` objects. An empty list means the + preset is valid. The function aggregates errors across all three layers + (preset, config, per-module) so a user with multiple typos sees them all. + + Args: + preset_dict: Preset as a plain dict (e.g. from `yaml.safe_load`). + module_loader: Optional module loader for per-module validation. Falls + back to the global MODULE_LOADER if not provided. If neither is + available, per-module validation is skipped. + validate_modules: If False, only validate preset top-level keys and the + global config tree (skip per-module Config validation). + known_modules: Optional set of known module names. If provided, module + names not in this set are reported as errors. Defaults to the + module loader's known modules. + + Examples: + >>> errors = validate_preset({"modlues": ["nuclei"]}) + >>> print(errors[0]) + [preset:modlues] Unknown option: 'modlues' (value: ['nuclei']) + """ + errors: list[PresetValidationError] = [] + + if not isinstance(preset_dict, dict): + return [PresetValidationError("preset", "", f"Expected a dict, got {type(preset_dict).__name__}")] + + # 1. Top-level preset keys + try: + PresetSchema.model_validate(preset_dict) + except ValidationError as e: + errors.extend(_format_pydantic_errors(e, where="preset")) + + # 2. Global config tree + config_dict = preset_dict.get("config") or {} + if not isinstance(config_dict, dict): + errors.append( + PresetValidationError("config", "", f"`config` must be a dict, got {type(config_dict).__name__}") + ) + config_dict = {} + else: + # Exclude the `modules` section from root-level BBOTConfig validation; + # per-module schemas are validated separately below. + config_for_root = {k: v for k, v in config_dict.items() if k != "modules"} + try: + BBOTConfig.model_validate(config_for_root) + except ValidationError as e: + errors.extend(_format_pydantic_errors(e, where="config")) + + # 3. Per-module config + if validate_modules: + if module_loader is None: + try: + from bbot.core.modules import MODULE_LOADER + + module_loader = MODULE_LOADER + except Exception: + module_loader = None + + referenced = _modules_referenced(preset_dict, config_dict) + + if module_loader is not None: + if known_modules is None: + known_modules = set(module_loader.all_module_choices) + else: + known_modules = set(known_modules) + + modules_config = config_dict.get("modules") or {} + for name in sorted(referenced): + if name not in known_modules: + errors.append(PresetValidationError("preset", "modules", f'Unknown module: "{name}"')) + continue + + raw = modules_config.get(name) or {} + if not isinstance(raw, dict): + errors.append( + PresetValidationError( + f"module:{name}", + "", + f"module config must be a dict, got {type(raw).__name__}", + ) + ) + continue + + cfg_cls = _get_module_config_class(name, module_loader) + if cfg_cls is None: + # legacy module — no Config class yet. Skip strict validation. + continue + try: + cfg_cls.model_validate(raw) + except ValidationError as e: + errors.extend(_format_pydantic_errors(e, where=f"module:{name}")) + + return errors + + +def validate_preset_file(path: str | Path, **kwargs) -> list[PresetValidationError]: + """Convenience wrapper for validating a YAML preset file on disk.""" + import yaml + + with open(path) as f: + data = yaml.safe_load(f) or {} + if not isinstance(data, dict): + return [PresetValidationError("preset", "", f"Expected a YAML mapping, got {type(data).__name__}")] + return validate_preset(data, **kwargs) + + +__all__ = ["PresetValidationError", "validate_preset", "validate_preset_file"] + + +def _assert_preset_valid(preset_dict: dict, **kwargs) -> None: + """Raise if a preset dict has any validation errors. Tests use this.""" + from bbot.errors import ValidationError as BBOTValidationError + + errs = validate_preset(preset_dict, **kwargs) + if errs: + raise BBOTValidationError("\n".join(str(e) for e in errs)) diff --git a/bbot/scanner/scanner.py b/bbot/scanner/scanner.py index 76554a75d5..659bde18c4 100644 --- a/bbot/scanner/scanner.py +++ b/bbot/scanner/scanner.py @@ -76,7 +76,7 @@ class Scanner: _status_code (int): The numerical representation of the current scan status, stored for internal use. It is mapped according to the values in `_status_codes`. target (Target): Target of scan (alias to `self.preset.target`). preset (Preset): The main scan Preset in its baked form. - config (omegaconf.dictconfig.DictConfig): BBOT config (alias to `self.preset.config`). + config (dict): BBOT config (alias to `self.preset.config`). seeds (Target): Scan seeds (by default this is the same as `target`) (alias to `self.preset.seeds`). blacklist (Target): Scan blacklist (this takes ultimate precedence) (alias to `self.preset.blacklist`). helpers (ConfigAwareHelper): Helper containing various reusable functions, regexes, etc. (alias to `self.preset.helpers`). diff --git a/bbot/test/bbot_fixtures.py b/bbot/test/bbot_fixtures.py index 1deca2be08..afdb24c016 100644 --- a/bbot/test/bbot_fixtures.py +++ b/bbot/test/bbot_fixtures.py @@ -7,12 +7,12 @@ import tldextract import pytest_httpserver from pathlib import Path -from omegaconf import OmegaConf # noqa from werkzeug.wrappers import Request from bbot.errors import * # noqa: F401 from bbot.core import CORE +from bbot.core.config.merge import deep_update from bbot.scanner import Preset from bbot.core.helpers.misc import mkdir, rand_string @@ -49,13 +49,14 @@ def tempapkfile(): @pytest.fixture def clean_default_config(monkeypatch): - clean_config = OmegaConf.merge( - CORE.files_config.get_default_config(), {"modules": DEFAULT_PRESET.module_loader.configs()} + clean_config = deep_update( + CORE.files_config.get_default_config(), + {"modules": DEFAULT_PRESET.module_loader.configs()}, ) with monkeypatch.context() as m: m.setattr("bbot.core.core.DEFAULT_CONFIG", clean_config) # Also clear CORE's custom_config to ensure Preset.copy() gets a clean core - m.setattr(CORE, "_custom_config", OmegaConf.create({})) + m.setattr(CORE, "_custom_config", {}) yield diff --git a/bbot/test/conftest.py b/bbot/test/conftest.py index ac4ef7b8b2..80b10a4fbb 100644 --- a/bbot/test/conftest.py +++ b/bbot/test/conftest.py @@ -1,13 +1,13 @@ import os import ssl import time +import yaml import pytest import shutil import asyncio import logging from pathlib import Path from contextlib import suppress -from omegaconf import OmegaConf from pytest_httpserver import HTTPServer from bbot.core import CORE @@ -23,7 +23,8 @@ debug_handler.setFormatter(debug_format) root_logger.addHandler(debug_handler) -test_config = OmegaConf.load(Path(__file__).parent / "test.conf") +with open(Path(__file__).parent / "test.conf") as _f: + test_config = yaml.safe_load(_f) or {} os.environ["BBOT_DEBUG"] = "True" CORE.logger.log_level = logging.DEBUG diff --git a/bbot/test/test_step_1/test_config.py b/bbot/test/test_step_1/test_config.py index b040bf5dc0..6454a8f599 100644 --- a/bbot/test/test_step_1/test_config.py +++ b/bbot/test/test_step_1/test_config.py @@ -3,23 +3,21 @@ @pytest.mark.asyncio async def test_config(bbot_scanner): - config = OmegaConf.create( - { - "plumbus": "asdf", - "speculate": True, - "modules": { - "ipneighbor": {"test_option": "ipneighbor"}, - "python": {"test_option": "asdf"}, - "speculate": {"test_option": "speculate"}, - }, - } - ) + config = { + "plumbus": "asdf", + "speculate": True, + "modules": { + "ipneighbor": {"test_option": "ipneighbor"}, + "python": {"test_option": "asdf"}, + "speculate": {"test_option": "speculate"}, + }, + } scan1 = bbot_scanner("127.0.0.1", modules=["ipneighbor"], config=config) await scan1._prep() - assert scan1.config.web.user_agent == "BBOT Test User-Agent" - assert scan1.config.plumbus == "asdf" - assert scan1.modules["ipneighbor"].config.test_option == "ipneighbor" - assert scan1.modules["python"].config.test_option == "asdf" - assert scan1.modules["speculate"].config.test_option == "speculate" + assert scan1.config["web"]["user_agent"] == "BBOT Test User-Agent" + assert scan1.config["plumbus"] == "asdf" + assert scan1.modules["ipneighbor"].config["test_option"] == "ipneighbor" + assert scan1.modules["python"].config["test_option"] == "asdf" + assert scan1.modules["speculate"].config["test_option"] == "speculate" await scan1._cleanup() diff --git a/bbot/test/test_step_1/test_presets.py b/bbot/test/test_step_1/test_presets.py index ff8e4d0214..571f869115 100644 --- a/bbot/test/test_step_1/test_presets.py +++ b/bbot/test/test_step_1/test_presets.py @@ -25,31 +25,21 @@ def test_preset_descriptions(): def test_core(): from bbot.core import CORE - import omegaconf - assert "testasdf" not in CORE.default_config assert "testasdf" not in CORE.custom_config assert "testasdf" not in CORE.config core_copy = CORE.copy() - # make sure our default config is read-only - with pytest.raises(omegaconf.errors.ReadonlyConfigError): - core_copy.default_config["testasdf"] = "test" - # same for merged config - with pytest.raises(omegaconf.errors.ReadonlyConfigError): - core_copy.config["testasdf"] = "test" - - assert "testasdf" not in core_copy.default_config - assert "testasdf" not in core_copy.custom_config - assert "testasdf" not in core_copy.config - + # custom_config is mutable per-instance; default/merged config should not leak back core_copy.custom_config["testasdf"] = "test" - assert "testasdf" not in core_copy.default_config + assert "testasdf" not in CORE.custom_config + assert "testasdf" not in CORE.config assert "testasdf" in core_copy.custom_config + # force a re-merge by reading .config assert "testasdf" in core_copy.config # test config merging - config_to_merge = omegaconf.OmegaConf.create({"test123": {"test321": [3, 2, 1], "test456": [4, 5, 6]}}) + config_to_merge = {"test123": {"test321": [3, 2, 1], "test456": [4, 5, 6]}} core_copy.merge_custom(config_to_merge) assert "test123" not in core_copy.default_config assert "test123" in core_copy.custom_config @@ -58,7 +48,9 @@ def test_core(): assert "test321" in core_copy.config["test123"] # test deletion - del core_copy.custom_config.test123.test321 + del core_copy.custom_config["test123"]["test321"] + # force re-merge + core_copy._config = None assert "test123" in core_copy.custom_config assert "test123" in core_copy.config assert "test321" not in core_copy.custom_config["test123"] @@ -902,19 +894,19 @@ def test_preset_include(): # with include= preset = Preset(include=[str(custom_preset_dir_1 / "preset1")]) - assert preset.config.modules.testpreset1.test == "asdf" - assert preset.config.modules.testpreset2.test == "fdsa" - assert preset.config.modules.testpreset3.test == "qwerty" - assert preset.config.modules.testpreset4.test == "zxcv" - assert preset.config.modules.testpreset5.test == "hjkl" + assert preset.config["modules"]["testpreset1"]["test"] == "asdf" + assert preset.config["modules"]["testpreset2"]["test"] == "fdsa" + assert preset.config["modules"]["testpreset3"]["test"] == "qwerty" + assert preset.config["modules"]["testpreset4"]["test"] == "zxcv" + assert preset.config["modules"]["testpreset5"]["test"] == "hjkl" # same thing but with presets= (an alias to include) preset = Preset(presets=[str(custom_preset_dir_1 / "preset1")]) - assert preset.config.modules.testpreset1.test == "asdf" - assert preset.config.modules.testpreset2.test == "fdsa" - assert preset.config.modules.testpreset3.test == "qwerty" - assert preset.config.modules.testpreset4.test == "zxcv" - assert preset.config.modules.testpreset5.test == "hjkl" + assert preset.config["modules"]["testpreset1"]["test"] == "asdf" + assert preset.config["modules"]["testpreset2"]["test"] == "fdsa" + assert preset.config["modules"]["testpreset3"]["test"] == "qwerty" + assert preset.config["modules"]["testpreset4"]["test"] == "zxcv" + assert preset.config["modules"]["testpreset5"]["test"] == "hjkl" # can't use both include= and presets= at the same time with pytest.raises(ValueError): diff --git a/bbot/test/test_step_1/test_validate_preset.py b/bbot/test/test_step_1/test_validate_preset.py new file mode 100644 index 0000000000..4c697893fa --- /dev/null +++ b/bbot/test/test_step_1/test_validate_preset.py @@ -0,0 +1,61 @@ +from ..bbot_fixtures import * # noqa: F401 + +from bbot.scanner import validate_preset + + +def test_validate_preset_valid(): + errs = validate_preset({"modules": ["sslcert"], "config": {"scope": {"strict": True}}}) + assert errs == [] + + +def test_validate_preset_typo_top_level(): + errs = validate_preset({"modlues": ["nuclei"]}, validate_modules=False) + assert len(errs) == 1 + assert errs[0].where == "preset" + assert "modlues" in errs[0].message + + +def test_validate_preset_typo_in_config(): + errs = validate_preset({"config": {"scope": {"strct": True}}}, validate_modules=False) + assert len(errs) == 1 + assert errs[0].where == "config" + assert "strct" in errs[0].message + assert errs[0].path == "scope.strct" + + +def test_validate_preset_wrong_type(): + errs = validate_preset({"config": {"web": {"http_timeout": "not-a-number"}}}, validate_modules=False) + assert len(errs) == 1 + assert errs[0].where == "config" + assert errs[0].path == "web.http_timeout" + assert "integer" in errs[0].message + + +def test_validate_preset_unknown_module(): + errs = validate_preset({"modules": ["nucleii"]}) + assert any('Unknown module: "nucleii"' in str(e) for e in errs) + + +def test_validate_preset_multiple_errors(): + """A preset with several typos should produce errors for all of them, not just the first.""" + errs = validate_preset( + { + "modlues": ["x"], # typo in top-level key + "config": { + "scope": {"strct": True}, # typo in config section + "web": {"http_timeout": "bad"}, # wrong type + }, + }, + validate_modules=False, + ) + assert len(errs) >= 3 + messages = " ".join(str(e) for e in errs) + assert "modlues" in messages + assert "strct" in messages + assert "http_timeout" in messages + + +def test_validate_preset_non_dict(): + errs = validate_preset(["not a dict"]) + assert len(errs) == 1 + assert "dict" in errs[0].message diff --git a/bbot/test/test_step_2/module_tests/base.py b/bbot/test/test_step_2/module_tests/base.py index 8bea4e2700..0a30fc0054 100644 --- a/bbot/test/test_step_2/module_tests/base.py +++ b/bbot/test/test_step_2/module_tests/base.py @@ -2,10 +2,10 @@ import asyncio import logging import pytest_asyncio -from omegaconf import OmegaConf from ...bbot_fixtures import * from bbot.scanner import Scanner +from bbot.core.config.merge import deep_update from bbot.core.helpers.misc import rand_string log = logging.getLogger("bbot.test.modules") @@ -28,7 +28,7 @@ def __init__( self, module_test_base, httpx_mock, httpserver, httpserver_ssl, monkeypatch, request, caplog, capsys ): self.name = module_test_base.name - self.config = OmegaConf.merge(CORE.config, OmegaConf.create(module_test_base.config_overrides)) + self.config = deep_update(dict(CORE.config), dict(module_test_base.config_overrides)) self.caplog = caplog self.capsys = capsys @@ -51,7 +51,7 @@ def __init__( if module_type == "output": output_modules.append(module) elif module_type == "internal" and not module == "dnsresolve": - self.config = OmegaConf.merge(self.config, {module: True}) + self.config = deep_update(self.config, {module: True}) seeds = module_test_base.seeds or None diff --git a/pyproject.toml b/pyproject.toml index 3e2870e696..eee3165761 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,7 +16,6 @@ classifiers = [ ] dependencies = [ "pip", - "omegaconf>=2.3.0,<3", "psutil>=5.9.4,<8.0.0", "wordninja>=2.0.0,<3", "ansible-runner>=2.3.2,<3", @@ -43,6 +42,8 @@ dependencies = [ "httpx>=0.28.1,<1", "puremagic>=1.28,<2", "pydantic>=2.12.2,<3", + "pydantic-settings>=2.6.0,<3", + "pyyaml>=6.0.0,<7", "radixtarget>=4.0.1,<5", "asndb>=1.0.4", "orjson>=3.10.12,<4", diff --git a/uv.lock b/uv.lock index b3704d0c43..6560425fc4 100644 --- a/uv.lock +++ b/uv.lock @@ -97,12 +97,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a9/da/19512e72e9cf2b8e7e6345264baa6c7ac1bb0ab128eb19c73a58407c4566/ansible_runner-2.4.2-py3-none-any.whl", hash = "sha256:0bde6cb39224770ff49ccdc6027288f6a98f4ed2ea0c64688b31217033221893", size = 79758, upload-time = "2025-10-14T19:10:48.994Z" }, ] -[[package]] -name = "antlr4-python3-runtime" -version = "4.9.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/3e/38/7859ff46355f76f8d19459005ca000b6e7012f2f1ca597746cbcd1fbfe5e/antlr4-python3-runtime-4.9.3.tar.gz", hash = "sha256:f224469b4168294902bb1efa80a8bf7855f24c99aef99cbefc1bcd3cce77881b", size = 117034, upload-time = "2021-11-06T17:52:23.524Z" } - [[package]] name = "anyio" version = "4.12.1" @@ -205,14 +199,15 @@ dependencies = [ { name = "jinja2" }, { name = "lxml" }, { name = "mmh3" }, - { name = "omegaconf" }, { name = "orjson" }, { name = "pip" }, { name = "psutil" }, { name = "puremagic" }, { name = "pycryptodome" }, { name = "pydantic" }, + { name = "pydantic-settings" }, { name = "pyjwt" }, + { name = "pyyaml" }, { name = "pyzmq" }, { name = "radixtarget" }, { name = "regex" }, @@ -267,7 +262,7 @@ requires-dist = [ { name = "asndb", specifier = ">=1.0.4" }, { name = "beautifulsoup4", specifier = ">=4.12.2,<5" }, { name = "blastdns", specifier = ">=1.9.0,<2" }, - { name = "cachetools", specifier = ">=5.3.2,<7.0.0" }, + { name = "cachetools", specifier = ">=5.3.2,<8.0.0" }, { name = "cloudcheck", specifier = ">=9.2.0,<10" }, { name = "deepdiff", specifier = ">=8.0.0,<10" }, { name = "dnspython", specifier = ">=2.7.0,<2.9.0" }, @@ -276,14 +271,15 @@ requires-dist = [ { name = "jinja2", specifier = ">=3.1.3,<4" }, { name = "lxml", specifier = ">=4.9.2,<7.0.0" }, { name = "mmh3", specifier = ">=4.1,<6.0" }, - { name = "omegaconf", specifier = ">=2.3.0,<3" }, { name = "orjson", specifier = ">=3.10.12,<4" }, { name = "pip" }, { name = "psutil", specifier = ">=5.9.4,<8.0.0" }, { name = "puremagic", specifier = ">=1.28,<2" }, { name = "pycryptodome", specifier = ">=3.17,<4" }, { name = "pydantic", specifier = ">=2.12.2,<3" }, + { name = "pydantic-settings", specifier = ">=2.6.0,<3" }, { name = "pyjwt", specifier = ">=2.7.0,<3" }, + { name = "pyyaml", specifier = ">=6.0.0,<7" }, { name = "pyzmq", specifier = ">=26.0.3,<28.0.0" }, { name = "radixtarget", specifier = ">=4.0.1,<5" }, { name = "regex", specifier = ">=2024.4.16,<2027.0.0" }, @@ -305,7 +301,7 @@ dev = [ { name = "fastapi", specifier = ">=0.115.5,<0.137.0" }, { name = "pre-commit", specifier = ">=3.4,<5.0" }, { name = "pytest", specifier = ">=8.3.1,<9" }, - { name = "pytest-asyncio", specifier = "==1.2.0" }, + { name = "pytest-asyncio", specifier = "==1.3.0" }, { name = "pytest-benchmark", specifier = ">=4,<6" }, { name = "pytest-cov", specifier = ">=5,<8" }, { name = "pytest-env", specifier = ">=0.8.2,<1.7.0" }, @@ -1673,19 +1669,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/88/b2/d0896bdcdc8d28a7fc5717c305f1a861c26e18c05047949fb371034d98bd/nodeenv-1.10.0-py2.py3-none-any.whl", hash = "sha256:5bb13e3eed2923615535339b3c620e76779af4cb4c6a90deccc9e36b274d3827", size = 23438, upload-time = "2025-12-20T14:08:52.782Z" }, ] -[[package]] -name = "omegaconf" -version = "2.3.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "antlr4-python3-runtime" }, - { name = "pyyaml" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/09/48/6388f1bb9da707110532cb70ec4d2822858ddfb44f1cdf1233c20a80ea4b/omegaconf-2.3.0.tar.gz", hash = "sha256:d5d4b6d29955cc50ad50c46dc269bcd92c6e00f5f90d23ab5fee7bfca4ba4cc7", size = 3298120, upload-time = "2022-12-08T20:59:22.753Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e3/94/1843518e420fa3ed6919835845df698c7e27e183cb997394e4a670973a65/omegaconf-2.3.0-py3-none-any.whl", hash = "sha256:7b4df175cdb08ba400f45cae3bdcae7ba8365db4d165fc65fd04b050ab63b46b", size = 79500, upload-time = "2022-12-08T20:59:19.686Z" }, -] - [[package]] name = "orderly-set" version = "5.5.0" @@ -2090,6 +2073,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/36/c7/cfc8e811f061c841d7990b0201912c3556bfeb99cdcb7ed24adc8d6f8704/pydantic_core-2.41.5-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:56121965f7a4dc965bff783d70b907ddf3d57f6eba29b6d2e5dabfaf07799c51", size = 2145302, upload-time = "2025-11-04T13:43:46.64Z" }, ] +[[package]] +name = "pydantic-settings" +version = "2.14.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pydantic" }, + { name = "python-dotenv" }, + { name = "typing-inspection" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/42/98/c8345dccdc31de4228c039a98f6467a941e39558da41c1744fbe29fa5666/pydantic_settings-2.14.0.tar.gz", hash = "sha256:24285fd4b0e0c06507dd9fdfd331ee23794305352aaec8fc4eb92d4047aeb67d", size = 235709, upload-time = "2026-04-20T13:37:40.293Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/01/dd/bebff3040138f00ae8a102d426b27349b9a49acc310fcae7f92112d867e3/pydantic_settings-2.14.0-py3-none-any.whl", hash = "sha256:fc8d5d692eb7092e43c8647c1c35a3ecd00e040fcf02ed86f4cb5458ca62182e", size = 60940, upload-time = "2026-04-20T13:37:38.586Z" }, +] + [[package]] name = "pygments" version = "2.19.2" @@ -2150,16 +2147,16 @@ wheels = [ [[package]] name = "pytest-asyncio" -version = "1.2.0" +version = "1.3.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "backports-asyncio-runner", marker = "python_full_version < '3.11'" }, { name = "pytest" }, { name = "typing-extensions", marker = "python_full_version < '3.13'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/42/86/9e3c5f48f7b7b638b216e4b9e645f54d199d7abbbab7a64a13b4e12ba10f/pytest_asyncio-1.2.0.tar.gz", hash = "sha256:c609a64a2a8768462d0c99811ddb8bd2583c33fd33cf7f21af1c142e824ffb57", size = 50119, upload-time = "2025-09-12T07:33:53.816Z" } +sdist = { url = "https://files.pythonhosted.org/packages/90/2c/8af215c0f776415f3590cac4f9086ccefd6fd463befeae41cd4d3f193e5a/pytest_asyncio-1.3.0.tar.gz", hash = "sha256:d7f52f36d231b80ee124cd216ffb19369aa168fc10095013c6b014a34d3ee9e5", size = 50087, upload-time = "2025-11-10T16:07:47.256Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/04/93/2fa34714b7a4ae72f2f8dad66ba17dd9a2c793220719e736dda28b7aec27/pytest_asyncio-1.2.0-py3-none-any.whl", hash = "sha256:8e17ae5e46d8e7efe51ab6494dd2010f4ca8dae51652aa3c8d55acf50bfb2e99", size = 15095, upload-time = "2025-09-12T07:33:52.639Z" }, + { url = "https://files.pythonhosted.org/packages/e5/35/f8b19922b6a25bc0880171a2f1a003eaeb93657475193ab516fd87cac9da/pytest_asyncio-1.3.0-py3-none-any.whl", hash = "sha256:611e26147c7f77640e6d0a92a38ed17c3e9848063698d5c93d5aa7aa11cebff5", size = 15075, upload-time = "2025-11-10T16:07:45.537Z" }, ] [[package]] @@ -2276,6 +2273,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892, upload-time = "2024-03-01T18:36:18.57Z" }, ] +[[package]] +name = "python-dotenv" +version = "1.2.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/82/ed/0301aeeac3e5353ef3d94b6ec08bbcabd04a72018415dcb29e588514bba8/python_dotenv-1.2.2.tar.gz", hash = "sha256:2c371a91fbd7ba082c2c1dc1f8bf89ca22564a087c2c287cd9b662adde799cf3", size = 50135, upload-time = "2026-03-01T16:00:26.196Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0b/d7/1959b9648791274998a9c3526f6d0ec8fd2233e4d4acce81bbae76b44b2a/python_dotenv-1.2.2-py3-none-any.whl", hash = "sha256:1d8214789a24de455a8b8bd8ae6fe3c6b69a5e3d64aa8a8e5d68e694bbcb285a", size = 22101, upload-time = "2026-03-01T16:00:25.09Z" }, +] + [[package]] name = "python-whois" version = "0.9.6" From ca4e74221b90b2d2031927eb9b560cffe1434e29 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Fri, 24 Apr 2026 12:46:00 -0400 Subject: [PATCH 02/15] don't duplicate defaults in code --- bbot/core/config/models.py | 316 ++++++++++++------------------------- 1 file changed, 102 insertions(+), 214 deletions(-) diff --git a/bbot/core/config/models.py b/bbot/core/config/models.py index 63a64df7e2..c793067d74 100644 --- a/bbot/core/config/models.py +++ b/bbot/core/config/models.py @@ -1,9 +1,14 @@ """ Pydantic schema for BBOT's global config and preset files. -The top-level `BBOTConfig` mirrors the structure of `defaults.yml`. Per-module -configs are validated separately at bake time against each module's own -`class Config(BaseModuleConfig)` (see `BaseModuleConfig` below). +These models describe the *shape* of valid BBOT configuration — field names +and their expected types — so that `validate_preset()` can catch typos +(`scpoe:`, `http_timoeut:`) and type errors at the boundary. + +Defaults live in `bbot/defaults.yml` — the single source of truth. This file +intentionally does **not** repeat those values; every field is optional, and +an absent field passes validation. At runtime, `BBOTConfigFiles` loads the +merged dict straight from YAML, and these models only ever validate shape. """ from __future__ import annotations @@ -20,75 +25,65 @@ class ScopeConfig(BaseModel): model_config = STRICT - strict: bool = False - report_distance: int = 0 - search_distance: int = 0 + strict: Optional[bool] = None + report_distance: Optional[int] = None + search_distance: Optional[int] = None class DnsConfig(BaseModel): model_config = STRICT - disable: bool = False - minimal: bool = False - threads: int = 10 - cache_size: int = 100000 - brute_threads: int = 1000 - brute_nameservers: str = ( - "https://raw.githubusercontent.com/blacklanternsecurity/public-dns-servers/master/nameservers.txt" - ) - search_distance: int = 1 - runaway_limit: int = 5 - timeout: int = 5 - retries: int = 1 - wildcard_disable: bool = False - wildcard_ignore: list[str] = Field(default_factory=list) - wildcard_tests: int = 10 - abort_threshold: int = 10 - filter_ptrs: bool = True - debug: bool = False - omit_queries: list[str] = Field( - default_factory=lambda: [ - "SRV:mail.protection.outlook.com", - "CNAME:mail.protection.outlook.com", - "TXT:mail.protection.outlook.com", - ] - ) + disable: Optional[bool] = None + minimal: Optional[bool] = None + threads: Optional[int] = None + cache_size: Optional[int] = None + brute_threads: Optional[int] = None + brute_nameservers: Optional[str] = None + search_distance: Optional[int] = None + runaway_limit: Optional[int] = None + timeout: Optional[int] = None + retries: Optional[int] = None + wildcard_disable: Optional[bool] = None + wildcard_ignore: Optional[list[str]] = None + wildcard_tests: Optional[int] = None + abort_threshold: Optional[int] = None + filter_ptrs: Optional[bool] = None + debug: Optional[bool] = None + omit_queries: Optional[list[str]] = None class WebConfig(BaseModel): model_config = STRICT http_proxy: Optional[str] = None - user_agent: str = ( - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " - "(KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36 Edg/119.0.2151.97" - ) + user_agent: Optional[str] = None user_agent_suffix: Optional[str] = None - spider_distance: int = 0 - spider_depth: int = 1 - spider_links_per_page: int = 25 - http_timeout: int = 10 - httpx_timeout: int = 5 - http_headers: dict[str, str] = Field(default_factory=dict) - http_cookies: dict[str, str] = Field(default_factory=dict) - api_retries: int = 2 - http_retries: int = 1 - httpx_retries: int = 1 - sleep_interval_429: int = Field(30, alias="429_sleep_interval") - max_sleep_interval_429: int = Field(60, alias="429_max_sleep_interval") - debug: bool = False - http_max_redirects: int = 5 - ssl_verify: bool = False + spider_distance: Optional[int] = None + spider_depth: Optional[int] = None + spider_links_per_page: Optional[int] = None + http_timeout: Optional[int] = None + httpx_timeout: Optional[int] = None + http_headers: Optional[dict[str, str]] = None + http_cookies: Optional[dict[str, str]] = None + api_retries: Optional[int] = None + http_retries: Optional[int] = None + httpx_retries: Optional[int] = None + # The `429_*` keys start with a digit, so we expose them via aliases. + sleep_interval_429: Optional[int] = Field(default=None, alias="429_sleep_interval") + max_sleep_interval_429: Optional[int] = Field(default=None, alias="429_max_sleep_interval") + debug: Optional[bool] = None + http_max_redirects: Optional[int] = None + ssl_verify: Optional[bool] = None class EngineConfig(BaseModel): model_config = STRICT - debug: bool = False + debug: Optional[bool] = None class DepsToolConfig(BaseModel): - """Per-tool dep config, e.g. deps.ffuf.version""" + """Per-tool dep config (e.g. `deps.ffuf.version`).""" model_config = STRICT @@ -98,30 +93,34 @@ class DepsToolConfig(BaseModel): class DepsConfig(BaseModel): model_config = STRICT - behavior: str = "abort_on_failure" - ffuf: DepsToolConfig = Field(default_factory=lambda: DepsToolConfig(version="2.1.0")) + behavior: Optional[str] = None + ffuf: Optional[DepsToolConfig] = None class BaseModuleConfig(BaseModel): """ Shared base for every module's `class Config(BaseModuleConfig)`. - Carries the three universal module options that are applied to every - module regardless of declaration. + + Declares the three universal module options that are applied to every + module regardless of declaration. The actual default values live in + `bbot/defaults.yml`; this class only validates shape. """ model_config = STRICT - batch_size: int = 10 - module_threads: int = 5 - module_timeout: int = 3600 + batch_size: Optional[int] = None + module_threads: Optional[int] = None + module_timeout: Optional[int] = None class BBOTConfig(BaseSettings): """ - Root BBOT config. Mirrors `bbot/defaults.yml`. + Root BBOT config schema. Unknown top-level keys are rejected so that + typos like `scpoe:` or `moudules:` become loud errors instead of silent + no-ops. - Unknown top-level keys raise ValidationError. This is what catches typos - like `scpoe:` or `moudules:` in user configs. + This is a validation schema only — it has no default values. The real + defaults live in `bbot/defaults.yml`. """ model_config = SettingsConfigDict( @@ -132,167 +131,56 @@ class BBOTConfig(BaseSettings): ) # Basic options - home: str = "~/.bbot" - keep_scans: int = 20 - status_frequency: int = 15 - file_blobs: bool = False - folder_blobs: bool = False - - # Scope / DNS / Web / Engine / Deps - scope: ScopeConfig = Field(default_factory=ScopeConfig) - dns: DnsConfig = Field(default_factory=DnsConfig) - web: WebConfig = Field(default_factory=WebConfig) - engine: EngineConfig = Field(default_factory=EngineConfig) - deps: DepsConfig = Field(default_factory=DepsConfig) + home: Optional[str] = None + keep_scans: Optional[int] = None + status_frequency: Optional[int] = None + file_blobs: Optional[bool] = None + folder_blobs: Optional[bool] = None + + # Nested sections + scope: Optional[ScopeConfig] = None + dns: Optional[DnsConfig] = None + web: Optional[WebConfig] = None + engine: Optional[EngineConfig] = None + deps: Optional[DepsConfig] = None # Module loader paths - module_dirs: list[str] = Field(default_factory=list) + module_dirs: Optional[list[str]] = None # Module runtime - module_handle_event_timeout: int = 3600 - module_handle_batch_timeout: int = 7200 + module_handle_event_timeout: Optional[int] = None + module_handle_batch_timeout: Optional[int] = None - # Internal module toggles (these are hardcoded because they're first-class - # features of the scan pipeline; the set changes rarely) - speculate: bool = True - excavate: bool = True - aggregate: bool = True - dnsresolve: bool = True - cloudcheck: bool = True + # Internal module toggles (hardcoded because they're first-class scan + # pipeline features; the set changes rarely) + speculate: Optional[bool] = None + excavate: Optional[bool] = None + aggregate: Optional[bool] = None + dnsresolve: Optional[bool] = None + cloudcheck: Optional[bool] = None # URL handling - url_querystring_remove: bool = True - url_querystring_collapse: bool = True - url_extension_blacklist: list[str] = Field( - default_factory=lambda: [ - "png", - "jpg", - "bmp", - "ico", - "jpeg", - "gif", - "svg", - "webp", - "css", - "woff", - "woff2", - "ttf", - "eot", - "sass", - "scss", - "mp3", - "m4a", - "wav", - "flac", - "mp4", - "mkv", - "avi", - "wmv", - "mov", - "flv", - "webm", - ] - ) - url_extension_special: list[str] = Field(default_factory=lambda: ["js"]) - url_extension_static: list[str] = Field( - default_factory=lambda: [ - "pdf", - "doc", - "docx", - "xls", - "xlsx", - "ppt", - "pptx", - "txt", - "csv", - "xml", - "yaml", - "ini", - "log", - "conf", - "cfg", - "env", - "md", - "rtf", - "tiff", - "bmp", - "jpg", - "jpeg", - "png", - "gif", - "svg", - "ico", - "mp3", - "wav", - "flac", - "mp4", - "mov", - "avi", - "mkv", - "webm", - "zip", - "tar", - "gz", - "bz2", - "7z", - "rar", - ] - ) + url_querystring_remove: Optional[bool] = None + url_querystring_collapse: Optional[bool] = None + url_extension_blacklist: Optional[list[str]] = None + url_extension_special: Optional[list[str]] = None + url_extension_static: Optional[list[str]] = None # Parameter handling - parameter_blacklist: list[str] = Field( - default_factory=lambda: [ - "__VIEWSTATE", - "__EVENTARGUMENT", - "__EVENTVALIDATION", - "__EVENTTARGET", - "__VIEWSTATEGENERATOR", - "__SCROLLPOSITIONY", - "__SCROLLPOSITIONX", - "ASP.NET_SessionId", - ".AspNetCore.Session", - "PHPSESSID", - "__cf_bm", - "f5_cspm", - ] - ) - parameter_blacklist_prefixes: list[str] = Field( - default_factory=lambda: [ - "TS01", - "BIGipServer", - "f5avr", - "incap_", - "visid_incap_", - "AWSALB", - "utm_", - "ApplicationGatewayAffinity", - "JSESSIONID", - "ARRAffinity", - ] - ) + parameter_blacklist: Optional[list[str]] = None + parameter_blacklist_prefixes: Optional[list[str]] = None # Event output filter - omit_event_types: list[str] = Field( - default_factory=lambda: [ - "HTTP_RESPONSE", - "RAW_TEXT", - "URL_UNVERIFIED", - "DNS_NAME_UNRESOLVED", - "FILESYSTEM", - "WEB_PARAMETER", - "RAW_DNS_RECORD", - ] - ) + omit_event_types: Optional[list[str]] = None # Interactsh interactsh_server: Optional[str] = None interactsh_token: Optional[str] = None - interactsh_disable: bool = False + interactsh_disable: Optional[bool] = None - # Per-module configs — validated separately, per-module, at bake time. - # Stored here as raw dicts so the root validator accepts any module - # registered at preload time. - modules: dict[str, dict[str, Any]] = Field(default_factory=dict) + # Per-module configs — validated separately, per-module, against each + # module's own `class Config(BaseModuleConfig)`. + modules: Optional[dict[str, dict[str, Any]]] = None class PresetSchema(BaseModel): @@ -300,8 +188,8 @@ class PresetSchema(BaseModel): Schema for the top-level keys in a preset YAML file. Catches typos like `modlues:` or `flgas:` at load time. - `target`/`targets` and `include`/`presets` are aliases; both accepted. - `config` is validated separately as `BBOTConfig`. + `target`/`targets` and `include`/`presets` are aliases; both are accepted. + The `config` key is validated separately as `BBOTConfig`. """ model_config = ConfigDict( @@ -309,8 +197,8 @@ class PresetSchema(BaseModel): populate_by_name=True, ) - target: Optional[list[str]] = Field(default=None) - targets: Optional[list[str]] = Field(default=None) + target: Optional[list[str]] = None + targets: Optional[list[str]] = None seeds: Optional[list[str]] = None blacklist: Optional[list[str]] = None @@ -334,9 +222,9 @@ class PresetSchema(BaseModel): conditions: Optional[list[str]] = None - verbose: bool = False - debug: bool = False - silent: bool = False + verbose: Optional[bool] = None + debug: Optional[bool] = None + silent: Optional[bool] = None __all__ = [ From 7578d1d0477d8264d1a74d0ab7445a89b90425b4 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Mon, 27 Apr 2026 13:54:50 -0400 Subject: [PATCH 03/15] preset validation wip --- bbot/core/config/models.py | 4 +- bbot/core/modules.py | 150 ++++++++- bbot/modules/anubisdb.py | 8 +- bbot/modules/apkpure.py | 8 +- bbot/modules/baddns.py | 24 +- bbot/modules/baddns_direct.py | 12 +- bbot/modules/baddns_zone.py | 12 +- bbot/modules/badsecrets.py | 9 +- bbot/modules/bevigil.py | 7 +- bbot/modules/bucket_amazon.py | 8 +- bbot/modules/bucket_digitalocean.py | 8 +- bbot/modules/bucket_file_enum.py | 8 +- bbot/modules/bucket_firebase.py | 8 +- bbot/modules/bucket_google.py | 8 +- bbot/modules/bucket_microsoft.py | 8 +- bbot/modules/bufferoverrun.py | 7 +- bbot/modules/builtwith.py | 7 +- bbot/modules/c99.py | 6 +- bbot/modules/censys_dns.py | 10 +- bbot/modules/censys_ip.py | 12 +- bbot/modules/chaos.py | 6 +- bbot/modules/credshed.py | 12 +- bbot/modules/dehashed.py | 6 +- bbot/modules/dnsbimi.py | 16 +- bbot/modules/dnsbrute.py | 13 +- bbot/modules/dnsbrute_mutations.py | 10 +- bbot/modules/dnscaa.py | 19 +- bbot/modules/dnscommonsrv.py | 6 +- bbot/modules/dnstlsrpt.py | 16 +- bbot/modules/docker_pull.py | 10 +- bbot/modules/ffuf.py | 27 +- bbot/modules/ffuf_shortnames.py | 37 +-- bbot/modules/filedownload.py | 77 +---- bbot/modules/fingerprintx.py | 9 +- bbot/modules/fullhunt.py | 6 +- bbot/modules/git_clone.py | 10 +- bbot/modules/gitdumper.py | 16 +- bbot/modules/github_codesearch.py | 7 +- bbot/modules/github_org.py | 12 +- bbot/modules/github_usersearch.py | 6 +- bbot/modules/github_workflows.py | 12 +- bbot/modules/gitlab_com.py | 6 +- bbot/modules/gitlab_onprem.py | 6 +- bbot/modules/gowitness.py | 34 +- bbot/modules/graphql_introspection.py | 13 +- bbot/modules/httpx.py | 25 +- bbot/modules/hunterio.py | 6 +- bbot/modules/iis_shortnames.py | 12 +- bbot/modules/internal/excavate.py | 16 +- bbot/modules/internal/speculate.py | 12 +- bbot/modules/ip2location.py | 10 +- bbot/modules/ipneighbor.py | 6 +- bbot/modules/ipstack.py | 6 +- bbot/modules/jadx.py | 10 +- bbot/modules/kreuzberg.py | 56 +--- bbot/modules/leakix.py | 6 +- bbot/modules/legba.py | 37 +-- bbot/modules/lightfuzz/lightfuzz.py | 25 +- bbot/modules/medusa.py | 22 +- bbot/modules/ntlm.py | 6 +- bbot/modules/nuclei.py | 59 ++-- bbot/modules/oauth.py | 6 +- bbot/modules/otx.py | 6 +- bbot/modules/output/asset_inventory.py | 14 +- bbot/modules/output/csv.py | 6 +- bbot/modules/output/discord.py | 14 +- bbot/modules/output/elastic.py | 19 +- bbot/modules/output/emails.py | 6 +- bbot/modules/output/http.py | 28 +- bbot/modules/output/json.py | 8 +- bbot/modules/output/kafka.py | 13 +- bbot/modules/output/mongo.py | 22 +- bbot/modules/output/mysql.py | 22 +- bbot/modules/output/nats.py | 13 +- bbot/modules/output/neo4j.py | 12 +- bbot/modules/output/postgres.py | 22 +- bbot/modules/output/rabbitmq.py | 13 +- bbot/modules/output/slack.py | 14 +- bbot/modules/output/splunk.py | 22 +- bbot/modules/output/sqlite.py | 10 +- bbot/modules/output/stdout.py | 16 +- bbot/modules/output/subdomains.py | 7 +- bbot/modules/output/teams.py | 14 +- bbot/modules/output/txt.py | 6 +- bbot/modules/output/web_parameters.py | 10 +- bbot/modules/output/web_report.py | 10 +- bbot/modules/output/websocket.py | 14 +- bbot/modules/output/zeromq.py | 10 +- bbot/modules/paramminer_cookies.py | 21 +- bbot/modules/paramminer_getparams.py | 16 +- bbot/modules/paramminer_headers.py | 16 +- bbot/modules/passivetotal.py | 6 +- bbot/modules/pgp.py | 13 +- bbot/modules/portfilter.py | 13 +- bbot/modules/portscan.py | 41 +-- bbot/modules/postman.py | 6 +- bbot/modules/postman_download.py | 10 +- bbot/modules/retirejs.py | 16 +- bbot/modules/robots.py | 12 +- bbot/modules/securitytrails.py | 6 +- bbot/modules/securitytxt.py | 13 +- bbot/modules/shodan_dns.py | 6 +- bbot/modules/shodan_enterprise.py | 10 +- bbot/modules/shodan_idb.py | 9 +- bbot/modules/sslcert.py | 7 +- bbot/modules/subdomainradar.py | 12 +- bbot/modules/telerik.py | 10 +- bbot/modules/trajan.py | 31 +- bbot/modules/trickest.py | 10 +- bbot/modules/trufflehog.py | 22 +- bbot/modules/url_manipulation.py | 8 +- bbot/modules/urlscan.py | 6 +- bbot/modules/virustotal.py | 6 +- bbot/modules/wafw00f.py | 6 +- bbot/modules/wayback.py | 10 +- bbot/modules/wpscan.py | 28 +- bbot/scanner/preset/validate.py | 291 +++++++----------- bbot/test/test_step_1/test_validate_preset.py | 31 +- 118 files changed, 963 insertions(+), 1116 deletions(-) diff --git a/bbot/core/config/models.py b/bbot/core/config/models.py index c793067d74..815e66d8c6 100644 --- a/bbot/core/config/models.py +++ b/bbot/core/config/models.py @@ -13,7 +13,7 @@ from __future__ import annotations -from typing import Any, Optional +from typing import Any, Literal, Optional from pydantic import BaseModel, ConfigDict, Field from pydantic_settings import BaseSettings, SettingsConfigDict @@ -93,7 +93,7 @@ class DepsToolConfig(BaseModel): class DepsConfig(BaseModel): model_config = STRICT - behavior: Optional[str] = None + behavior: Optional[Literal["abort_on_failure", "retry_failed", "ignore_failed", "disable", "force_install"]] = None ffuf: Optional[DepsToolConfig] = None diff --git a/bbot/core/modules.py b/bbot/core/modules.py index 9b1e2045b7..adeca4862f 100644 --- a/bbot/core/modules.py +++ b/bbot/core/modules.py @@ -57,14 +57,119 @@ def _eval_ast_default(node): return _UNEVALUATED +def _exec_config_class(source: str, module_name: str): + """ + Execute a `class Config(BaseModuleConfig):` snippet in a controlled + namespace and return the resulting class. The namespace provides exactly + what a Config block is allowed to reference: the typing primitives, the + pydantic `Field` factory, and `BaseModuleConfig`. + + This replaces parsing annotations as strings: pydantic handles every + valid type expression (`Optional[str]`, `Literal["a", "b"]`, + `list[Union[int, str]]`, …) without any hand-rolled resolver. + """ + from typing import Any, Dict, List, Literal, Optional, Set, Tuple, Union + from pydantic import Field + from bbot.core.config.models import BaseModuleConfig + + namespace: dict = { + "Any": Any, + "Dict": Dict, + "List": List, + "Literal": Literal, + "Optional": Optional, + "Set": Set, + "Tuple": Tuple, + "Union": Union, + "Field": Field, + "BaseModuleConfig": BaseModuleConfig, + } + try: + exec(source, namespace) + except Exception as e: + raise BBOTError( + f'module "{module_name}" has an invalid Config class ({type(e).__name__}: {e}). ' + "Config blocks may only reference: Optional, Union, Literal, Any, List, Dict, Tuple, Set, " + "Field, BaseModuleConfig, and Python builtins." + ) from e + cfg = namespace.get("Config") + if cfg is None: + raise BBOTError(f'module "{module_name}": Config snippet did not define a class named "Config"') + return cfg + + +def _build_validation_schema(preloaded: dict): + """ + Build the composite preset validation schema. + + Structure: + FullPresetSchema + ├── (all PresetSchema fields — target, modules, flags, …) + └── config: FullBBOTConfig + ├── (all BBOTConfig fields — scope, dns, web, …) + └── modules: ModulesSchema + ├── nuclei: NucleiModuleConfig + ├── httpx: HttpxModuleConfig + ├── sslcert: SslcertModuleConfig + └── … one field per known module + + A single `FullPresetSchema.model_validate(preset_dict)` call then catches + every class of error in one pass: + - top-level preset typos (extra='forbid' on PresetSchema) + - global config typos / wrong types (extra='forbid' on BBOTConfig) + - unknown module names (extra='forbid' on ModulesSchema) + - wrong module option names / wrong types (extra='forbid' per module) + """ + import warnings + from typing import Optional + from pydantic import ConfigDict, Field, create_model + from bbot.core.config.models import BaseModuleConfig, BBOTConfig, PresetSchema + + module_fields = {} + for name, data in preloaded.items(): + source = data.get("config_source") + if source: + cfg_model = _exec_config_class(source, name) + else: + # Module declares no Config — only the universal options apply. + cfg_model = BaseModuleConfig + module_fields[name] = (Optional[cfg_model], Field(default=None)) + + # Some module names (e.g. `json`) shadow BaseModel's deprecated method + # names and trigger a UserWarning. The field still validates correctly; + # silence the noise. + with warnings.catch_warnings(): + warnings.filterwarnings( + "ignore", + message=r'Field name ".+" in ".+" shadows an attribute in parent "BaseModel"', + category=UserWarning, + ) + ModulesSchema = create_model( + "ModulesSchema", + __config__=ConfigDict(extra="forbid"), + **module_fields, + ) + FullBBOTConfig = create_model( + "FullBBOTConfig", + __base__=BBOTConfig, + modules=(Optional[ModulesSchema], Field(default=None)), + ) + FullPresetSchema = create_model( + "FullPresetSchema", + __base__=PresetSchema, + config=(Optional[FullBBOTConfig], Field(default=None)), + ) + return FullPresetSchema + + def _extract_pydantic_config(config_class: ast.ClassDef) -> tuple[dict, dict]: """ Walk a `class Config(BaseModuleConfig):` block and extract - `(field_defaults, field_descriptions)` dicts mirroring the legacy - `options`/`options_desc` shape. + `(defaults, descriptions)` dicts — cheap metadata used for `bbot -l` and + similar listing paths, without importing or exec-ing anything. - Used by the preloader so `bbot -l` and module listing work without - importing the module (deps may be missing on the host). + The actual typed pydantic class is built later via `_exec_config_class` + on the captured source text. """ defaults: dict = {} descriptions: dict = {} @@ -134,6 +239,9 @@ def __init__(self): self.internal_module_choices = set() self._preload_cache = None + # Composite preset-validation schema, built from preloaded modules. + # Invalidated whenever a new module is preloaded. + self._validation_schema = None self._module_dirs = set() self._module_dirs_preloaded = set() @@ -273,6 +381,11 @@ def preload(self, module_dirs=None): # update default config with module defaults self.core.merge_default({"modules": self.configs()}) + # invalidate the composite validation schema; it'll rebuild lazily + # on next access now that the set of modules has changed + if new_modules: + self._validation_schema = None + return new_modules @property @@ -322,6 +435,18 @@ def configs(self, type=None): return {k: dict(v) for k, v in self._configs.items() if self.check_type(k, type)} return {k: dict(v) for k, v in self._configs.items()} + @property + def validation_schema(self): + """ + The composite pydantic schema for validating a full preset dict. + + Built lazily from preloaded module metadata and cached. Rebuilt when + `preload()` discovers new modules (e.g. after `add_module_dir`). + """ + if self._validation_schema is None: + self._validation_schema = _build_validation_schema(self._preloaded) + return self._validation_schema + def find_and_replace(self, **kwargs): self.__preloaded = search_format_dict(self.__preloaded, **kwargs) self._shared_deps = search_format_dict(self._shared_deps, **kwargs) @@ -390,6 +515,7 @@ def preload_module(self, module_file): ansible_tasks = [] config = {} options_desc = {} + config_source: str | None = None disable_auto_module_deps = False with open(module_file) as f: python_code = f.read() @@ -416,26 +542,21 @@ def preload_module(self, module_file): # look for classes if type(root_element) == ast.ClassDef: for class_attr in root_element.body: - # nested `class Config(BaseModuleConfig): ...` — the new pydantic-based schema + # nested `class Config(BaseModuleConfig): ...` — the module's config schema if type(class_attr) == ast.ClassDef and class_attr.name == "Config": config_fields, config_descs = _extract_pydantic_config(class_attr) config.update(config_fields) options_desc.update(config_descs) + # capture the class source verbatim; schema build re-execs it + config_source = ast.get_source_segment(python_code, class_attr) continue if not type(class_attr) == ast.Assign: continue - # class attributes that are dictionaries + # module metadata if type(class_attr.value) == ast.Dict: - # module options (legacy — pre-pydantic shim) - if any(target.id == "options" for target in class_attr.targets): - config.update(ast.literal_eval(class_attr.value)) - # module options descriptions (legacy) - elif any(target.id == "options_desc" for target in class_attr.targets): - options_desc.update(ast.literal_eval(class_attr.value)) - # module metadata - elif any(target.id == "meta" for target in class_attr.targets): + if any(target.id == "meta" for target in class_attr.targets): meta = ast.literal_eval(class_attr.value) # class attributes that are lists @@ -509,6 +630,7 @@ def preload_module(self, module_file): "meta": meta, "config": config, "options_desc": options_desc, + "config_source": config_source, "hash": module_hash, "deps": { "modules": sorted(deps_modules), diff --git a/bbot/modules/anubisdb.py b/bbot/modules/anubisdb.py index f1621a1f2a..0d667e15a0 100644 --- a/bbot/modules/anubisdb.py +++ b/bbot/modules/anubisdb.py @@ -1,4 +1,6 @@ from bbot.modules.templates.subdomain_enum import subdomain_enum +from pydantic import Field +from bbot.core.config.models import BaseModuleConfig class anubisdb(subdomain_enum): @@ -10,10 +12,8 @@ class anubisdb(subdomain_enum): "created_date": "2022-10-04", "author": "@TheTechromancer", } - options = {"limit": 1000} - options_desc = { - "limit": "Limit the number of subdomains returned per query (increasing this may slow the scan due to garbage results from this API)" - } + class Config(BaseModuleConfig): + limit: int = Field(1000, description='Limit the number of subdomains returned per query (increasing this may slow the scan due to garbage results from this API)') base_url = "https://jldc.me/anubis/subdomains" dns_abort_depth = 5 diff --git a/bbot/modules/apkpure.py b/bbot/modules/apkpure.py index 8364a918e1..1e38d3ea5d 100644 --- a/bbot/modules/apkpure.py +++ b/bbot/modules/apkpure.py @@ -1,6 +1,8 @@ import re from pathlib import Path from bbot.modules.base import BaseModule +from pydantic import Field +from bbot.core.config.models import BaseModuleConfig class apkpure(BaseModule): @@ -12,10 +14,8 @@ class apkpure(BaseModule): "created_date": "2024-10-11", "author": "@domwhewell-sage", } - options = {"output_folder": ""} - options_desc = { - "output_folder": "Folder to download APKs to. If not specified, downloaded APKs will be deleted when the scan completes, to minimize disk usage." - } + class Config(BaseModuleConfig): + output_folder: str = Field('', description='Folder to download APKs to. If not specified, downloaded APKs will be deleted when the scan completes, to minimize disk usage.') async def setup(self): output_folder = self.config.get("output_folder", "") diff --git a/bbot/modules/baddns.py b/bbot/modules/baddns.py index 515b54b221..5cbddf37ed 100644 --- a/bbot/modules/baddns.py +++ b/bbot/modules/baddns.py @@ -1,8 +1,11 @@ +from typing import Literal from baddns.base import get_all_modules from baddns.lib.loader import load_signatures from .base import BaseModule import logging +from pydantic import Field +from bbot.core.config.models import BaseModuleConfig SEVERITY_LEVELS = ("INFO", "LOW", "MEDIUM", "HIGH", "CRITICAL") CONFIDENCE_LEVELS = ("UNKNOWN", "LOW", "MEDIUM", "HIGH", "CONFIRMED") @@ -45,13 +48,20 @@ class baddns(BaseModule): "created_date": "2024-01-18", "author": "@liquidsec", } - options = {"custom_nameservers": [], "min_severity": "LOW", "min_confidence": "MEDIUM", "enabled_submodules": []} - options_desc = { - "custom_nameservers": "Force BadDNS to use a list of custom nameservers", - "min_severity": "Minimum severity to emit (INFO, LOW, MEDIUM, HIGH, CRITICAL)", - "min_confidence": "Minimum confidence to emit (UNKNOWN, LOW, MEDIUM, HIGH, CONFIRMED)", - "enabled_submodules": "A list of submodules to enable. Empty list (default) enables CNAME, TXT and MX Only", - } + class Config(BaseModuleConfig): + custom_nameservers: list[str] = Field( + default_factory=list, description="Force BadDNS to use a list of custom nameservers" + ) + min_severity: Literal["INFO", "LOW", "MEDIUM", "HIGH", "CRITICAL"] = Field( + "LOW", description="Minimum severity to emit" + ) + min_confidence: Literal["UNKNOWN", "LOW", "MEDIUM", "HIGH", "CONFIRMED"] = Field( + "MEDIUM", description="Minimum confidence to emit" + ) + enabled_submodules: list[str] = Field( + default_factory=list, + description="A list of submodules to enable. Empty list (default) enables CNAME, TXT and MX Only", + ) module_threads = 8 deps_pip = ["baddns~=2.1.0"] diff --git a/bbot/modules/baddns_direct.py b/bbot/modules/baddns_direct.py index 243f224a11..2bde5917b0 100644 --- a/bbot/modules/baddns_direct.py +++ b/bbot/modules/baddns_direct.py @@ -1,4 +1,6 @@ from .baddns import baddns as baddns_module +from pydantic import Field +from bbot.core.config.models import BaseModuleConfig class baddns_direct(baddns_module): @@ -10,12 +12,10 @@ class baddns_direct(baddns_module): "created_date": "2024-01-29", "author": "@liquidsec", } - options = {"custom_nameservers": [], "min_severity": "LOW", "min_confidence": "MEDIUM"} - options_desc = { - "custom_nameservers": "Force BadDNS to use a list of custom nameservers", - "min_severity": "Minimum severity to emit (INFO, LOW, MEDIUM, HIGH, CRITICAL)", - "min_confidence": "Minimum confidence to emit (UNKNOWN, LOW, MEDIUM, HIGH, CONFIRMED)", - } + class Config(BaseModuleConfig): + custom_nameservers: list = Field([], description='Force BadDNS to use a list of custom nameservers') + min_severity: str = Field('LOW', description='Minimum severity to emit (INFO, LOW, MEDIUM, HIGH, CRITICAL)') + min_confidence: str = Field('MEDIUM', description='Minimum confidence to emit (UNKNOWN, LOW, MEDIUM, HIGH, CONFIRMED)') module_threads = 8 deps_pip = ["baddns~=2.1.0"] diff --git a/bbot/modules/baddns_zone.py b/bbot/modules/baddns_zone.py index e13140c70a..5193b9ef81 100644 --- a/bbot/modules/baddns_zone.py +++ b/bbot/modules/baddns_zone.py @@ -1,4 +1,6 @@ from .baddns import baddns as baddns_module +from pydantic import Field +from bbot.core.config.models import BaseModuleConfig class baddns_zone(baddns_module): @@ -10,12 +12,10 @@ class baddns_zone(baddns_module): "created_date": "2024-01-29", "author": "@liquidsec", } - options = {"custom_nameservers": [], "min_severity": "INFO", "min_confidence": "MEDIUM"} - options_desc = { - "custom_nameservers": "Force BadDNS to use a list of custom nameservers", - "min_severity": "Minimum severity to emit (INFO, LOW, MEDIUM, HIGH, CRITICAL)", - "min_confidence": "Minimum confidence to emit (UNKNOWN, LOW, MEDIUM, HIGH, CONFIRMED)", - } + class Config(BaseModuleConfig): + custom_nameservers: list = Field([], description='Force BadDNS to use a list of custom nameservers') + min_severity: str = Field('INFO', description='Minimum severity to emit (INFO, LOW, MEDIUM, HIGH, CRITICAL)') + min_confidence: str = Field('MEDIUM', description='Minimum confidence to emit (UNKNOWN, LOW, MEDIUM, HIGH, CONFIRMED)') module_threads = 8 deps_pip = ["baddns~=2.1.0"] diff --git a/bbot/modules/badsecrets.py b/bbot/modules/badsecrets.py index a5f8049f9e..b398661604 100644 --- a/bbot/modules/badsecrets.py +++ b/bbot/modules/badsecrets.py @@ -2,6 +2,9 @@ from pathlib import Path from .base import BaseModule from badsecrets.base import carve_all_modules +from pydantic import Field +from bbot.core.config.models import BaseModuleConfig +from typing import Optional class badsecrets(BaseModule): @@ -13,10 +16,8 @@ class badsecrets(BaseModule): "created_date": "2022-11-19", "author": "@liquidsec", } - options = {"custom_secrets": None} - options_desc = { - "custom_secrets": "Include custom secrets loaded from a local file", - } + class Config(BaseModuleConfig): + custom_secrets: Optional[str] = Field(None, description='Include custom secrets loaded from a local file') deps_pip = ["badsecrets~=1.0.0"] async def setup(self): diff --git a/bbot/modules/bevigil.py b/bbot/modules/bevigil.py index 82d9e00c90..43de0fe4f9 100644 --- a/bbot/modules/bevigil.py +++ b/bbot/modules/bevigil.py @@ -1,4 +1,6 @@ from bbot.modules.templates.subdomain_enum import subdomain_enum_apikey +from pydantic import Field +from bbot.core.config.models import BaseModuleConfig class bevigil(subdomain_enum_apikey): @@ -15,8 +17,9 @@ class bevigil(subdomain_enum_apikey): "author": "@alt-glitch", "auth_required": True, } - options = {"api_key": "", "urls": False} - options_desc = {"api_key": "BeVigil OSINT API Key", "urls": "Emit URLs in addition to DNS_NAMEs"} + class Config(BaseModuleConfig): + api_key: str = Field('', description='BeVigil OSINT API Key') + urls: bool = Field(False, description='Emit URLs in addition to DNS_NAMEs') base_url = "https://osint.bevigil.com/api" diff --git a/bbot/modules/bucket_amazon.py b/bbot/modules/bucket_amazon.py index 1b9d7fd788..10cd2a3708 100644 --- a/bbot/modules/bucket_amazon.py +++ b/bbot/modules/bucket_amazon.py @@ -1,4 +1,6 @@ from bbot.modules.templates.bucket import bucket_template +from pydantic import Field +from bbot.core.config.models import BaseModuleConfig class bucket_amazon(bucket_template): @@ -10,10 +12,8 @@ class bucket_amazon(bucket_template): "created_date": "2022-11-04", "author": "@TheTechromancer", } - options = {"permutations": False} - options_desc = { - "permutations": "Whether to try permutations", - } + class Config(BaseModuleConfig): + permutations: bool = Field(False, description='Whether to try permutations') scope_distance_modifier = 3 cloudcheck_provider_name = "Amazon" diff --git a/bbot/modules/bucket_digitalocean.py b/bbot/modules/bucket_digitalocean.py index cd0e90a90c..a4ae6e7128 100644 --- a/bbot/modules/bucket_digitalocean.py +++ b/bbot/modules/bucket_digitalocean.py @@ -1,4 +1,6 @@ from bbot.modules.templates.bucket import bucket_template +from pydantic import Field +from bbot.core.config.models import BaseModuleConfig class bucket_digitalocean(bucket_template): @@ -10,10 +12,8 @@ class bucket_digitalocean(bucket_template): "created_date": "2022-11-08", "author": "@TheTechromancer", } - options = {"permutations": False} - options_desc = { - "permutations": "Whether to try permutations", - } + class Config(BaseModuleConfig): + permutations: bool = Field(False, description='Whether to try permutations') cloudcheck_provider_name = "DigitalOcean" delimiters = ("", "-") diff --git a/bbot/modules/bucket_file_enum.py b/bbot/modules/bucket_file_enum.py index 8849970e4b..ac96a94219 100644 --- a/bbot/modules/bucket_file_enum.py +++ b/bbot/modules/bucket_file_enum.py @@ -1,5 +1,7 @@ from bbot.modules.base import BaseModule import xml.etree.ElementTree as ET +from pydantic import Field +from bbot.core.config.models import BaseModuleConfig class bucket_file_enum(BaseModule): @@ -17,10 +19,8 @@ class bucket_file_enum(BaseModule): "author": "@TheTechromancer", } flags = ["safe", "passive", "cloud-enum"] - options = { - "file_limit": 50, - } - options_desc = {"file_limit": "Limit the number of files downloaded per bucket"} + class Config(BaseModuleConfig): + file_limit: int = Field(50, description='Limit the number of files downloaded per bucket') scope_distance_modifier = 2 async def setup(self): diff --git a/bbot/modules/bucket_firebase.py b/bbot/modules/bucket_firebase.py index 4859911a68..81fab8bfde 100644 --- a/bbot/modules/bucket_firebase.py +++ b/bbot/modules/bucket_firebase.py @@ -1,4 +1,6 @@ from bbot.modules.templates.bucket import bucket_template +from pydantic import Field +from bbot.core.config.models import BaseModuleConfig class bucket_firebase(bucket_template): @@ -10,10 +12,8 @@ class bucket_firebase(bucket_template): "created_date": "2023-03-20", "author": "@TheTechromancer", } - options = {"permutations": False} - options_desc = { - "permutations": "Whether to try permutations", - } + class Config(BaseModuleConfig): + permutations: bool = Field(False, description='Whether to try permutations') cloudcheck_provider_name = "Google" delimiters = ("", "-") diff --git a/bbot/modules/bucket_google.py b/bbot/modules/bucket_google.py index 338db450c4..5610c87773 100644 --- a/bbot/modules/bucket_google.py +++ b/bbot/modules/bucket_google.py @@ -1,4 +1,6 @@ from bbot.modules.templates.bucket import bucket_template +from pydantic import Field +from bbot.core.config.models import BaseModuleConfig class bucket_google(bucket_template): @@ -14,10 +16,8 @@ class bucket_google(bucket_template): "created_date": "2022-11-04", "author": "@TheTechromancer", } - options = {"permutations": False} - options_desc = { - "permutations": "Whether to try permutations", - } + class Config(BaseModuleConfig): + permutations: bool = Field(False, description='Whether to try permutations') cloudcheck_provider_name = "Google" delimiters = ("", "-", ".", "_") diff --git a/bbot/modules/bucket_microsoft.py b/bbot/modules/bucket_microsoft.py index 9dc5924c0a..b392f0307a 100644 --- a/bbot/modules/bucket_microsoft.py +++ b/bbot/modules/bucket_microsoft.py @@ -1,4 +1,6 @@ from bbot.modules.templates.bucket import bucket_template +from pydantic import Field +from bbot.core.config.models import BaseModuleConfig class bucket_microsoft(bucket_template): @@ -10,10 +12,8 @@ class bucket_microsoft(bucket_template): "created_date": "2022-11-04", "author": "@TheTechromancer", } - options = {"permutations": False} - options_desc = { - "permutations": "Whether to try permutations", - } + class Config(BaseModuleConfig): + permutations: bool = Field(False, description='Whether to try permutations') cloudcheck_provider_name = "Microsoft" delimiters = ("", "-") diff --git a/bbot/modules/bufferoverrun.py b/bbot/modules/bufferoverrun.py index 18b10b2e56..aa3b5c6c99 100644 --- a/bbot/modules/bufferoverrun.py +++ b/bbot/modules/bufferoverrun.py @@ -1,4 +1,6 @@ from bbot.modules.templates.subdomain_enum import subdomain_enum_apikey +from pydantic import Field +from bbot.core.config.models import BaseModuleConfig class BufferOverrun(subdomain_enum_apikey): @@ -11,8 +13,9 @@ class BufferOverrun(subdomain_enum_apikey): "author": "@TheTechromancer", "auth_required": True, } - options = {"api_key": "", "commercial": False} - options_desc = {"api_key": "BufferOverrun API key", "commercial": "Use commercial API"} + class Config(BaseModuleConfig): + api_key: str = Field('', description='BufferOverrun API key') + commercial: bool = Field(False, description='Use commercial API') base_url = "https://tls.bufferover.run/dns" commercial_base_url = "https://bufferover-run-tls.p.rapidapi.com/ipv4/dns" diff --git a/bbot/modules/builtwith.py b/bbot/modules/builtwith.py index 054818c942..6dad9b6e5c 100644 --- a/bbot/modules/builtwith.py +++ b/bbot/modules/builtwith.py @@ -11,6 +11,8 @@ ############################################################ from bbot.modules.templates.subdomain_enum import subdomain_enum_apikey +from pydantic import Field +from bbot.core.config.models import BaseModuleConfig class builtwith(subdomain_enum_apikey): @@ -23,8 +25,9 @@ class builtwith(subdomain_enum_apikey): "author": "@TheTechromancer", "auth_required": True, } - options = {"api_key": "", "redirects": True} - options_desc = {"api_key": "Builtwith API key", "redirects": "Also look up inbound and outbound redirects"} + class Config(BaseModuleConfig): + api_key: str = Field('', description='Builtwith API key') + redirects: bool = Field(True, description='Also look up inbound and outbound redirects') base_url = "https://api.builtwith.com" async def handle_event(self, event): diff --git a/bbot/modules/c99.py b/bbot/modules/c99.py index 84b3eafc6c..686861cb07 100644 --- a/bbot/modules/c99.py +++ b/bbot/modules/c99.py @@ -1,4 +1,6 @@ from bbot.modules.templates.subdomain_enum import subdomain_enum_apikey +from pydantic import Field +from bbot.core.config.models import BaseModuleConfig class c99(subdomain_enum_apikey): @@ -11,8 +13,8 @@ class c99(subdomain_enum_apikey): "author": "@TheTechromancer", "auth_required": True, } - options = {"api_key": ""} - options_desc = {"api_key": "c99.nl API key"} + class Config(BaseModuleConfig): + api_key: str = Field('', description='c99.nl API key') base_url = "https://api.c99.nl" ping_url = f"{base_url}/randomnumber?key={{api_key}}&between=1,100&json" diff --git a/bbot/modules/censys_dns.py b/bbot/modules/censys_dns.py index 7e4a397eb8..dc58d5bb7c 100644 --- a/bbot/modules/censys_dns.py +++ b/bbot/modules/censys_dns.py @@ -1,4 +1,6 @@ from bbot.modules.templates.censys import censys +from pydantic import Field +from bbot.core.config.models import BaseModuleConfig class censys_dns(censys): @@ -16,11 +18,9 @@ class censys_dns(censys): "author": "@TheTechromancer", "auth_required": True, } - options = {"api_key": "", "max_pages": 5} - options_desc = { - "api_key": "Censys.io API Key in the format of 'key:secret'", - "max_pages": "Maximum number of pages to fetch (100 results per page)", - } + class Config(BaseModuleConfig): + api_key: str = Field('', description="Censys.io API Key in the format of 'key:secret'") + max_pages: int = Field(5, description='Maximum number of pages to fetch (100 results per page)') async def setup(self): self.max_pages = self.config.get("max_pages", 5) diff --git a/bbot/modules/censys_ip.py b/bbot/modules/censys_ip.py index 374323b8fe..a631c82e7e 100644 --- a/bbot/modules/censys_ip.py +++ b/bbot/modules/censys_ip.py @@ -1,4 +1,6 @@ from bbot.modules.templates.censys import censys +from pydantic import Field +from bbot.core.config.models import BaseModuleConfig class censys_ip(censys): @@ -23,12 +25,10 @@ class censys_ip(censys): "author": "@TheTechromancer", "auth_required": True, } - options = {"api_key": "", "dns_names_limit": 100, "in_scope_only": True} - options_desc = { - "api_key": "Censys.io API Key in the format of 'key:secret'", - "dns_names_limit": "Maximum number of DNS names to extract from dns.names (default 100)", - "in_scope_only": "Only query in-scope IPs. If False, will query up to distance 1.", - } + class Config(BaseModuleConfig): + api_key: str = Field('', description="Censys.io API Key in the format of 'key:secret'") + dns_names_limit: int = Field(100, description='Maximum number of DNS names to extract from dns.names (default 100)') + in_scope_only: bool = Field(True, description='Only query in-scope IPs. If False, will query up to distance 1.') scope_distance_modifier = 1 async def setup(self): diff --git a/bbot/modules/chaos.py b/bbot/modules/chaos.py index 9c4255e542..ea094fd44d 100644 --- a/bbot/modules/chaos.py +++ b/bbot/modules/chaos.py @@ -1,4 +1,6 @@ from bbot.modules.templates.subdomain_enum import subdomain_enum_apikey +from pydantic import Field +from bbot.core.config.models import BaseModuleConfig class chaos(subdomain_enum_apikey): @@ -11,8 +13,8 @@ class chaos(subdomain_enum_apikey): "author": "@TheTechromancer", "auth_required": True, } - options = {"api_key": ""} - options_desc = {"api_key": "Chaos API key"} + class Config(BaseModuleConfig): + api_key: str = Field('', description='Chaos API key') base_url = "https://dns.projectdiscovery.io/dns" ping_url = f"{base_url}/example.com" diff --git a/bbot/modules/credshed.py b/bbot/modules/credshed.py index ea1bf707c1..a363a31f39 100644 --- a/bbot/modules/credshed.py +++ b/bbot/modules/credshed.py @@ -1,6 +1,8 @@ from contextlib import suppress from bbot.modules.templates.subdomain_enum import subdomain_enum +from pydantic import Field +from bbot.core.config.models import BaseModuleConfig class credshed(subdomain_enum): @@ -13,12 +15,10 @@ class credshed(subdomain_enum): "author": "@SpamFaux", "auth_required": True, } - options = {"username": "", "password": "", "credshed_url": ""} - options_desc = { - "username": "Credshed username", - "password": "Credshed password", - "credshed_url": "URL of credshed server", - } + class Config(BaseModuleConfig): + username: str = Field('', description='Credshed username') + password: str = Field('', description='Credshed password') + credshed_url: str = Field('', description='URL of credshed server') target_only = True async def setup(self): diff --git a/bbot/modules/dehashed.py b/bbot/modules/dehashed.py index 701425407f..6d0cdc907b 100644 --- a/bbot/modules/dehashed.py +++ b/bbot/modules/dehashed.py @@ -1,6 +1,8 @@ from contextlib import suppress from bbot.modules.templates.subdomain_enum import subdomain_enum +from pydantic import Field +from bbot.core.config.models import BaseModuleConfig class dehashed(subdomain_enum): @@ -13,8 +15,8 @@ class dehashed(subdomain_enum): "author": "@SpamFaux", "auth_required": True, } - options = {"api_key": ""} - options_desc = {"api_key": "DeHashed API Key"} + class Config(BaseModuleConfig): + api_key: str = Field('', description='DeHashed API Key') target_only = True base_url = "https://api.dehashed.com/v2/search" diff --git a/bbot/modules/dnsbimi.py b/bbot/modules/dnsbimi.py index d301857d30..152496c732 100644 --- a/bbot/modules/dnsbimi.py +++ b/bbot/modules/dnsbimi.py @@ -28,6 +28,8 @@ from bbot.core.helpers.dns.helpers import record_to_text, service_record import re +from pydantic import Field +from bbot.core.config.models import BaseModuleConfig # Handle "v=BIMI1; l=; a=;" == RFC conformant explicit declination to publish, e.g. useful on a sub-domain if you don't want the sub-domain to have a BIMI logo, yet your registered domain does? # Handle "v=BIMI1; l=; a=" == RFC non-conformant explicit declination to publish @@ -52,16 +54,10 @@ class dnsbimi(BaseModule): "author": "@colin-stubbs", "created_date": "2024-11-15", } - options = { - "emit_raw_dns_records": False, - "emit_urls": True, - "selectors": "default,email,mail,bimi", - } - options_desc = { - "emit_raw_dns_records": "Emit RAW_DNS_RECORD events", - "emit_urls": "Emit URL_UNVERIFIED events", - "selectors": "CSV list of BIMI selectors to check", - } + class Config(BaseModuleConfig): + emit_raw_dns_records: bool = Field(False, description='Emit RAW_DNS_RECORD events') + emit_urls: bool = Field(True, description='Emit URL_UNVERIFIED events') + selectors: str = Field('default,email,mail,bimi', description='CSV list of BIMI selectors to check') async def setup(self): self.emit_raw_dns_records = self.config.get("emit_raw_dns_records", False) diff --git a/bbot/modules/dnsbrute.py b/bbot/modules/dnsbrute.py index 77df6124ac..00b194cacb 100644 --- a/bbot/modules/dnsbrute.py +++ b/bbot/modules/dnsbrute.py @@ -1,4 +1,6 @@ from bbot.modules.templates.subdomain_enum import subdomain_enum +from pydantic import Field +from bbot.core.config.models import BaseModuleConfig class dnsbrute(subdomain_enum): @@ -10,14 +12,9 @@ class dnsbrute(subdomain_enum): "author": "@TheTechromancer", "created_date": "2024-04-24", } - options = { - "wordlist": "https://raw.githubusercontent.com/danielmiessler/SecLists/master/Discovery/DNS/subdomains-top1million-5000.txt", - "max_depth": 5, - } - options_desc = { - "wordlist": "Subdomain wordlist URL", - "max_depth": "How many subdomains deep to brute force, i.e. 5.4.3.2.1.evilcorp.com", - } + class Config(BaseModuleConfig): + wordlist: str = Field('https://raw.githubusercontent.com/danielmiessler/SecLists/master/Discovery/DNS/subdomains-top1million-5000.txt', description='Subdomain wordlist URL') + max_depth: int = Field(5, description='How many subdomains deep to brute force, i.e. 5.4.3.2.1.evilcorp.com') deps_common = ["massdns"] reject_wildcards = "strict" dedup_strategy = "lowest_parent" diff --git a/bbot/modules/dnsbrute_mutations.py b/bbot/modules/dnsbrute_mutations.py index aeb695fb6d..4ef4fd29bb 100644 --- a/bbot/modules/dnsbrute_mutations.py +++ b/bbot/modules/dnsbrute_mutations.py @@ -1,6 +1,8 @@ import time from bbot.modules.base import BaseModule +from pydantic import Field +from bbot.core.config.models import BaseModuleConfig class dnsbrute_mutations(BaseModule): @@ -12,12 +14,8 @@ class dnsbrute_mutations(BaseModule): "author": "@TheTechromancer", "created_date": "2024-04-25", } - options = { - "max_mutations": 100, - } - options_desc = { - "max_mutations": "Maximum number of target-specific mutations to try per subdomain", - } + class Config(BaseModuleConfig): + max_mutations: int = Field(100, description='Maximum number of target-specific mutations to try per subdomain') deps_common = ["massdns"] _qsize = 10000 diff --git a/bbot/modules/dnscaa.py b/bbot/modules/dnscaa.py index 0f8fa9ee96..136c09339e 100644 --- a/bbot/modules/dnscaa.py +++ b/bbot/modules/dnscaa.py @@ -22,6 +22,8 @@ from bbot.modules.base import BaseModule from bbot.core.helpers.regexes import dns_name_extraction_regex, email_regex, url_regexes +from pydantic import Field +from bbot.core.config.models import BaseModuleConfig class dnscaa(BaseModule): @@ -29,18 +31,11 @@ class dnscaa(BaseModule): produced_events = ["DNS_NAME", "EMAIL_ADDRESS", "URL_UNVERIFIED"] flags = ["safe", "subdomain-enum", "email-enum", "passive"] meta = {"description": "Check for CAA records", "author": "@colin-stubbs", "created_date": "2024-05-26"} - options = { - "in_scope_only": True, - "dns_names": True, - "emails": True, - "urls": True, - } - options_desc = { - "in_scope_only": "Only check in-scope domains", - "dns_names": "emit DNS_NAME events", - "emails": "emit EMAIL_ADDRESS events", - "urls": "emit URL_UNVERIFIED events", - } + class Config(BaseModuleConfig): + in_scope_only: bool = Field(True, description='Only check in-scope domains') + dns_names: bool = Field(True, description='emit DNS_NAME events') + emails: bool = Field(True, description='emit EMAIL_ADDRESS events') + urls: bool = Field(True, description='emit URL_UNVERIFIED events') # accept DNS_NAMEs out to 2 hops if in_scope_only is False scope_distance_modifier = 2 diff --git a/bbot/modules/dnscommonsrv.py b/bbot/modules/dnscommonsrv.py index 65bff03a2f..203f82939b 100644 --- a/bbot/modules/dnscommonsrv.py +++ b/bbot/modules/dnscommonsrv.py @@ -1,5 +1,7 @@ from bbot.core.helpers.dns.helpers import common_srvs from bbot.modules.templates.subdomain_enum import subdomain_enum +from pydantic import Field +from bbot.core.config.models import BaseModuleConfig class dnscommonsrv(subdomain_enum): @@ -10,8 +12,8 @@ class dnscommonsrv(subdomain_enum): dedup_strategy = "lowest_parent" deps_common = ["massdns"] - options = {"max_depth": 2} - options_desc = {"max_depth": "The maximum subdomain depth to brute-force SRV records"} + class Config(BaseModuleConfig): + max_depth: int = Field(2, description='The maximum subdomain depth to brute-force SRV records') async def setup(self): self.max_subdomain_depth = self.config.get("max_depth", 2) diff --git a/bbot/modules/dnstlsrpt.py b/bbot/modules/dnstlsrpt.py index 7c45759478..0dc5efce44 100644 --- a/bbot/modules/dnstlsrpt.py +++ b/bbot/modules/dnstlsrpt.py @@ -20,6 +20,8 @@ import re from bbot.core.helpers.regexes import email_regex, url_regexes +from pydantic import Field +from bbot.core.config.models import BaseModuleConfig _tlsrpt_regex = r"^v=(?PTLSRPTv[0-9]+); *(?P.*)$" tlsrpt_regex = re.compile(_tlsrpt_regex, re.I) @@ -40,16 +42,10 @@ class dnstlsrpt(BaseModule): "author": "@colin-stubbs", "created_date": "2024-07-26", } - options = { - "emit_emails": True, - "emit_raw_dns_records": False, - "emit_urls": True, - } - options_desc = { - "emit_emails": "Emit EMAIL_ADDRESS events", - "emit_raw_dns_records": "Emit RAW_DNS_RECORD events", - "emit_urls": "Emit URL_UNVERIFIED events", - } + class Config(BaseModuleConfig): + emit_emails: bool = Field(True, description='Emit EMAIL_ADDRESS events') + emit_raw_dns_records: bool = Field(False, description='Emit RAW_DNS_RECORD events') + emit_urls: bool = Field(True, description='Emit URL_UNVERIFIED events') async def setup(self): self.emit_emails = self.config.get("emit_emails", True) diff --git a/bbot/modules/docker_pull.py b/bbot/modules/docker_pull.py index ad702cc5e3..bbf193d593 100644 --- a/bbot/modules/docker_pull.py +++ b/bbot/modules/docker_pull.py @@ -3,6 +3,8 @@ import tarfile from pathlib import Path from bbot.modules.base import BaseModule +from pydantic import Field +from bbot.core.config.models import BaseModuleConfig class docker_pull(BaseModule): @@ -14,11 +16,9 @@ class docker_pull(BaseModule): "created_date": "2024-03-24", "author": "@domwhewell-sage", } - options = {"all_tags": False, "output_folder": ""} - options_desc = { - "all_tags": "Download all tags from each registry (Default False)", - "output_folder": "Folder to download docker repositories to. If not specified, downloaded docker images will be deleted when the scan completes, to minimize disk usage.", - } + class Config(BaseModuleConfig): + all_tags: bool = Field(False, description='Download all tags from each registry (Default False)') + output_folder: str = Field('', description='Folder to download docker repositories to. If not specified, downloaded docker images will be deleted when the scan completes, to minimize disk usage.') scope_distance_modifier = 2 diff --git a/bbot/modules/ffuf.py b/bbot/modules/ffuf.py index 4076b02024..97e7d46265 100644 --- a/bbot/modules/ffuf.py +++ b/bbot/modules/ffuf.py @@ -4,6 +4,8 @@ import string import json import base64 +from pydantic import Field +from bbot.core.config.models import BaseModuleConfig class ffuf(BaseModule): @@ -12,23 +14,14 @@ class ffuf(BaseModule): flags = ["loud", "active"] meta = {"description": "A fast web fuzzer written in Go", "created_date": "2022-04-10", "author": "@liquidsec"} - options = { - "wordlist": "https://raw.githubusercontent.com/danielmiessler/SecLists/master/Discovery/Web-Content/raft-small-directories.txt", - "lines": 5000, - "max_depth": 0, - "extensions": "", - "ignore_case": False, - "rate": 0, - } - - options_desc = { - "wordlist": "Specify wordlist to use when finding directories", - "lines": "take only the first N lines from the wordlist when finding directories", - "max_depth": "the maximum directory depth to attempt to solve", - "extensions": "Optionally include a list of extensions to extend the keyword with (comma separated)", - "ignore_case": "Only put lowercase words into the wordlist", - "rate": "Rate of requests per second (default: 0)", - } + class Config(BaseModuleConfig): + wordlist: str = Field('https://raw.githubusercontent.com/danielmiessler/SecLists/master/Discovery/Web-Content/raft-small-directories.txt', description='Specify wordlist to use when finding directories') + lines: int = Field(5000, description='take only the first N lines from the wordlist when finding directories') + max_depth: int = Field(0, description='the maximum directory depth to attempt to solve') + extensions: str = Field('', description='Optionally include a list of extensions to extend the keyword with (comma separated)') + ignore_case: bool = Field(False, description='Only put lowercase words into the wordlist') + rate: int = Field(0, description='Rate of requests per second (default: 0)') + deps_common = ["ffuf"] diff --git a/bbot/modules/ffuf_shortnames.py b/bbot/modules/ffuf_shortnames.py index f002b4e613..ceab878337 100644 --- a/bbot/modules/ffuf_shortnames.py +++ b/bbot/modules/ffuf_shortnames.py @@ -4,6 +4,8 @@ import string from bbot.modules.ffuf import ffuf +from pydantic import Field +from bbot.core.config.models import BaseModuleConfig class ffuf_shortnames(ffuf): @@ -16,31 +18,18 @@ class ffuf_shortnames(ffuf): "author": "@liquidsec", } - options = { - "wordlist_extensions": "", # default is defined within setup function - "max_depth": 1, - "version": "2.0.0", - "extensions": "", - "ignore_redirects": True, - "find_common_prefixes": False, - "find_delimiters": True, - "find_subwords": False, - "max_predictions": 250, - "rate": 0, - } + class Config(BaseModuleConfig): + wordlist_extensions: str = Field('', description='Specify wordlist to use when making extension lists') + max_depth: int = Field(1, description='the maximum directory depth to attempt to solve') + version: str = Field('2.0.0', description='ffuf version') + extensions: str = Field('', description='Optionally include a list of extensions to extend the keyword with (comma separated)') + ignore_redirects: bool = Field(True, description='Explicitly ignore redirects (301,302)') + find_common_prefixes: bool = Field(False, description='Attempt to automatically detect common prefixes and make additional ffuf runs against them') + find_delimiters: bool = Field(True, description='Attempt to detect common delimiters and make additional ffuf runs against them') + find_subwords: bool = Field(False, description='Attempt to detect subwords and make additional ffuf runs against them') + max_predictions: int = Field(250, description='The maximum number of predictions to generate per shortname prefix') + rate: int = Field(0, description='Rate of requests per second (default: 0)') - options_desc = { - "wordlist_extensions": "Specify wordlist to use when making extension lists", - "max_depth": "the maximum directory depth to attempt to solve", - "version": "ffuf version", - "extensions": "Optionally include a list of extensions to extend the keyword with (comma separated)", - "ignore_redirects": "Explicitly ignore redirects (301,302)", - "find_common_prefixes": "Attempt to automatically detect common prefixes and make additional ffuf runs against them", - "find_delimiters": "Attempt to detect common delimiters and make additional ffuf runs against them", - "find_subwords": "Attempt to detect subwords and make additional ffuf runs against them", - "max_predictions": "The maximum number of predictions to generate per shortname prefix", - "rate": "Rate of requests per second (default: 0)", - } deps_pip = ["numpy"] deps_common = ["ffuf"] diff --git a/bbot/modules/filedownload.py b/bbot/modules/filedownload.py index adb50f7857..ac15e0b576 100644 --- a/bbot/modules/filedownload.py +++ b/bbot/modules/filedownload.py @@ -2,6 +2,8 @@ from pathlib import Path from bbot.modules.base import BaseModule +from pydantic import Field +from bbot.core.config.models import BaseModuleConfig class filedownload(BaseModule): @@ -20,77 +22,10 @@ class filedownload(BaseModule): "created_date": "2023-10-11", "author": "@TheTechromancer", } - options = { - "extensions": [ - "bak", # Backup File - "bash", # Bash Script or Configuration - "bashrc", # Bash Script or Configuration - "cfg", # Configuration File - "conf", # Configuration File - "crt", # Certificate File - "csv", # Comma Separated Values File - "db", # SQLite Database File - "dll", # Windows Dynamic Link Library - "doc", # Microsoft Word Document (Old Format) - "docx", # Microsoft Word Document - "exe", # Windows PE executable - "ica", # Citrix Independent Computing Architecture File - "indd", # Adobe InDesign Document - "ini", # Initialization File - "jar", # Java Archive - "json", # JSON File - "key", # Private Key File - "log", # Log File - "markdown", # Markdown File - "md", # Markdown File - "msi", # Windows setup file - "odg", # OpenDocument Graphics (LibreOffice, OpenOffice) - "odp", # OpenDocument Presentation (LibreOffice, OpenOffice) - "ods", # OpenDocument Spreadsheet (LibreOffice, OpenOffice) - "odt", # OpenDocument Text (LibreOffice, OpenOffice) - "pdf", # Adobe Portable Document Format - "pem", # Privacy Enhanced Mail (SSL certificate) - "pps", # Microsoft PowerPoint Slideshow (Old Format) - "ppsx", # Microsoft PowerPoint Slideshow - "ppt", # Microsoft PowerPoint Presentation (Old Format) - "pptx", # Microsoft PowerPoint Presentation - "ps1", # PowerShell Script - "pub", # Public Key File - "raw", # Raw Image File Format - "rdp", # Remote Desktop Protocol File - "rsa", # RSA Private Key File - "sh", # Shell Script - "sql", # SQL Database Dump - "sqlite", # SQLite Database File - "swp", # Swap File (temporary file, often Vim) - "sxw", # OpenOffice.org Writer document - "tar.gz", # Gzip-Compressed Tar Archive - "tgz", # Gzip-Compressed Tar Archive - "tar", # Tar Archive - "txt", # Plain Text Document - "vbs", # Visual Basic Script - "war", # Java Web Archive - "wpd", # WordPerfect Document - "xls", # Microsoft Excel Spreadsheet (Old Format) - "xlsx", # Microsoft Excel Spreadsheet - "xml", # eXtensible Markup Language File - "yaml", # YAML Ain't Markup Language - "yml", # YAML Ain't Markup Language - "zip", # Zip Archive - "lzma", # LZMA Compressed File - "rar", # RAR Compressed File - "7z", # 7-Zip Compressed File - "xz", # XZ Compressed File - "bz2", # Bzip2 Compressed File - ], - "max_filesize": "10MB", - "output_folder": "", - } - options_desc = { - "extensions": "File extensions to download", - "max_filesize": "Cancel download if filesize is greater than this size", - "output_folder": "Folder to download files to. If not specified, downloaded files will be deleted when the scan completes, to minimize disk usage.", - } + class Config(BaseModuleConfig): + extensions: list[str] = Field(['bak', 'bash', 'bashrc', 'cfg', 'conf', 'crt', 'csv', 'db', 'dll', 'doc', 'docx', 'exe', 'ica', 'indd', 'ini', 'jar', 'json', 'key', 'log', 'markdown', 'md', 'msi', 'odg', 'odp', 'ods', 'odt', 'pdf', 'pem', 'pps', 'ppsx', 'ppt', 'pptx', 'ps1', 'pub', 'raw', 'rdp', 'rsa', 'sh', 'sql', 'sqlite', 'swp', 'sxw', 'tar.gz', 'tgz', 'tar', 'txt', 'vbs', 'war', 'wpd', 'xls', 'xlsx', 'xml', 'yaml', 'yml', 'zip', 'lzma', 'rar', '7z', 'xz', 'bz2'], description='File extensions to download') + max_filesize: str = Field('10MB', description='Cancel download if filesize is greater than this size') + output_folder: str = Field('', description='Folder to download files to. If not specified, downloaded files will be deleted when the scan completes, to minimize disk usage.') scope_distance_modifier = 3 diff --git a/bbot/modules/fingerprintx.py b/bbot/modules/fingerprintx.py index bea096dfeb..667b8cc45d 100644 --- a/bbot/modules/fingerprintx.py +++ b/bbot/modules/fingerprintx.py @@ -1,6 +1,8 @@ import json import subprocess from bbot.modules.base import BaseModule +from pydantic import Field +from bbot.core.config.models import BaseModuleConfig class fingerprintx(BaseModule): @@ -12,14 +14,13 @@ class fingerprintx(BaseModule): "created_date": "2023-01-30", "author": "@TheTechromancer", } - options = {"version": "1.1.4"} - options_desc = {"version": "fingerprintx version"} _batch_size = 10 _module_threads = 2 _priority = 2 - options = {"skip_common_web": True} - options_desc = {"skip_common_web": "Skip common web ports such as 80, 443, 8080, 8443, etc."} + class Config(BaseModuleConfig): + version: str = Field("1.1.4", description="fingerprintx version") + skip_common_web: bool = Field(True, description="Skip common web ports such as 80, 443, 8080, 8443, etc.") deps_ansible = [ { diff --git a/bbot/modules/fullhunt.py b/bbot/modules/fullhunt.py index 584dafb143..90839b195c 100644 --- a/bbot/modules/fullhunt.py +++ b/bbot/modules/fullhunt.py @@ -1,4 +1,6 @@ from bbot.modules.templates.subdomain_enum import subdomain_enum_apikey +from pydantic import Field +from bbot.core.config.models import BaseModuleConfig class fullhunt(subdomain_enum_apikey): @@ -11,8 +13,8 @@ class fullhunt(subdomain_enum_apikey): "author": "@TheTechromancer", "auth_required": True, } - options = {"api_key": ""} - options_desc = {"api_key": "FullHunt API Key"} + class Config(BaseModuleConfig): + api_key: str = Field('', description='FullHunt API Key') base_url = "https://fullhunt.io/api/v1" diff --git a/bbot/modules/git_clone.py b/bbot/modules/git_clone.py index ad4bbe40a5..17a52c05ef 100644 --- a/bbot/modules/git_clone.py +++ b/bbot/modules/git_clone.py @@ -1,6 +1,8 @@ from pathlib import Path from subprocess import CalledProcessError from bbot.modules.templates.github import github +from pydantic import Field +from bbot.core.config.models import BaseModuleConfig class git_clone(github): @@ -12,11 +14,9 @@ class git_clone(github): "created_date": "2024-03-08", "author": "@domwhewell-sage", } - options = {"api_key": "", "output_folder": ""} - options_desc = { - "api_key": "Github token", - "output_folder": "Folder to clone repositories to. If not specified, cloned repositories will be deleted when the scan completes, to minimize disk usage.", - } + class Config(BaseModuleConfig): + api_key: str = Field('', description='Github token') + output_folder: str = Field('', description='Folder to clone repositories to. If not specified, cloned repositories will be deleted when the scan completes, to minimize disk usage.') deps_apt = ["git"] diff --git a/bbot/modules/gitdumper.py b/bbot/modules/gitdumper.py index 71ce02a338..ab837cc174 100644 --- a/bbot/modules/gitdumper.py +++ b/bbot/modules/gitdumper.py @@ -2,6 +2,8 @@ from pathlib import Path from subprocess import CalledProcessError from bbot.modules.base import BaseModule +from pydantic import Field +from bbot.core.config.models import BaseModuleConfig class gitdumper(BaseModule): @@ -13,16 +15,10 @@ class gitdumper(BaseModule): "created_date": "2025-02-11", "author": "@domwhewell-sage", } - options = { - "output_folder": "", - "fuzz_tags": False, - "max_semanic_version": 10, - } - options_desc = { - "output_folder": "Folder to download repositories to. If not specified, downloaded repositories will be deleted when the scan completes, to minimize disk usage.", - "fuzz_tags": "Fuzz for common git tag names (v0.0.1, 0.0.2, etc.) up to the max_semanic_version", - "max_semanic_version": "Maximum version number to fuzz for (default < v10.10.10)", - } + class Config(BaseModuleConfig): + output_folder: str = Field('', description='Folder to download repositories to. If not specified, downloaded repositories will be deleted when the scan completes, to minimize disk usage.') + fuzz_tags: bool = Field(False, description='Fuzz for common git tag names (v0.0.1, 0.0.2, etc.) up to the max_semanic_version') + max_semanic_version: int = Field(10, description='Maximum version number to fuzz for (default < v10.10.10)') scope_distance_modifier = 2 diff --git a/bbot/modules/github_codesearch.py b/bbot/modules/github_codesearch.py index 6901221a30..3fb15c7678 100644 --- a/bbot/modules/github_codesearch.py +++ b/bbot/modules/github_codesearch.py @@ -1,5 +1,7 @@ from bbot.modules.templates.github import github from bbot.modules.templates.subdomain_enum import subdomain_enum +from pydantic import Field +from bbot.core.config.models import BaseModuleConfig class github_codesearch(github, subdomain_enum): @@ -12,8 +14,9 @@ class github_codesearch(github, subdomain_enum): "author": "@domwhewell-sage", "auth_required": True, } - options = {"api_key": "", "limit": 100} - options_desc = {"api_key": "Github token", "limit": "Limit code search to this many results"} + class Config(BaseModuleConfig): + api_key: str = Field('', description='Github token') + limit: int = Field(100, description='Limit code search to this many results') github_raw_url = "https://raw.githubusercontent.com/" diff --git a/bbot/modules/github_org.py b/bbot/modules/github_org.py index 6225c21710..2e96fdad1e 100644 --- a/bbot/modules/github_org.py +++ b/bbot/modules/github_org.py @@ -1,4 +1,6 @@ from bbot.modules.templates.github import github +from pydantic import Field +from bbot.core.config.models import BaseModuleConfig class github_org(github): @@ -10,12 +12,10 @@ class github_org(github): "created_date": "2023-12-14", "author": "@domwhewell-sage", } - options = {"api_key": "", "include_members": True, "include_member_repos": False} - options_desc = { - "api_key": "Github token", - "include_members": "Enumerate organization members", - "include_member_repos": "Also enumerate organization members' repositories", - } + class Config(BaseModuleConfig): + api_key: str = Field('', description='Github token') + include_members: bool = Field(True, description='Enumerate organization members') + include_member_repos: bool = Field(False, description="Also enumerate organization members' repositories") scope_distance_modifier = 2 diff --git a/bbot/modules/github_usersearch.py b/bbot/modules/github_usersearch.py index 5e330c3e07..47693c2764 100644 --- a/bbot/modules/github_usersearch.py +++ b/bbot/modules/github_usersearch.py @@ -1,5 +1,7 @@ from bbot.modules.templates.github import github from bbot.modules.templates.subdomain_enum import subdomain_enum +from pydantic import Field +from bbot.core.config.models import BaseModuleConfig class github_usersearch(github, subdomain_enum): @@ -12,8 +14,8 @@ class github_usersearch(github, subdomain_enum): "author": "@domwhewell-sage", "auth_required": True, } - options = {"api_key": ""} - options_desc = {"api_key": "Github token"} + class Config(BaseModuleConfig): + api_key: str = Field('', description='Github token') async def handle_event(self, event): self.verbose("Searching for users with emails matching in scope domains") diff --git a/bbot/modules/github_workflows.py b/bbot/modules/github_workflows.py index e64a303022..12931ffdd4 100644 --- a/bbot/modules/github_workflows.py +++ b/bbot/modules/github_workflows.py @@ -3,6 +3,8 @@ from pathlib import Path from bbot.modules.templates.github import github +from pydantic import Field +from bbot.core.config.models import BaseModuleConfig class github_workflows(github): @@ -15,12 +17,10 @@ class github_workflows(github): "author": "@domwhewell-sage", "auth_required": True, } - options = {"api_key": "", "num_logs": 1, "output_folder": ""} - options_desc = { - "api_key": "Github token", - "num_logs": "For each workflow fetch the last N successful runs logs (max 100)", - "output_folder": "Folder to download workflow logs and artifacts to", - } + class Config(BaseModuleConfig): + api_key: str = Field('', description='Github token') + num_logs: int = Field(1, description='For each workflow fetch the last N successful runs logs (max 100)') + output_folder: str = Field('', description='Folder to download workflow logs and artifacts to') scope_distance_modifier = 2 diff --git a/bbot/modules/gitlab_com.py b/bbot/modules/gitlab_com.py index c72f94cb57..a08f645cc7 100644 --- a/bbot/modules/gitlab_com.py +++ b/bbot/modules/gitlab_com.py @@ -1,4 +1,6 @@ from bbot.modules.templates.gitlab import GitLabBaseModule +from pydantic import Field +from bbot.core.config.models import BaseModuleConfig class gitlab_com(GitLabBaseModule): @@ -13,8 +15,8 @@ class gitlab_com(GitLabBaseModule): "author": "@TheTechromancer", } - options = {"api_key": ""} - options_desc = {"api_key": "GitLab access token (for gitlab.com/org only)"} + class Config(BaseModuleConfig): + api_key: str = Field('', description='GitLab access token (for gitlab.com/org only)') # This is needed because we are consuming SOCIAL events, which aren't in scope scope_distance_modifier = 2 diff --git a/bbot/modules/gitlab_onprem.py b/bbot/modules/gitlab_onprem.py index 26fbf0a917..c76a0c91a2 100644 --- a/bbot/modules/gitlab_onprem.py +++ b/bbot/modules/gitlab_onprem.py @@ -1,4 +1,6 @@ from bbot.modules.templates.gitlab import GitLabBaseModule +from pydantic import Field +from bbot.core.config.models import BaseModuleConfig class gitlab_onprem(GitLabBaseModule): @@ -18,8 +20,8 @@ class gitlab_onprem(GitLabBaseModule): # Optional GitLab access token (only required for gitlab.com, but still # supported for on-prem installations that expose private projects). - options = {"api_key": ""} - options_desc = {"api_key": "GitLab access token (for self-hosted instances only)"} + class Config(BaseModuleConfig): + api_key: str = Field('', description='GitLab access token (for self-hosted instances only)') # Allow accepting events slightly beyond configured max distance so we can # discover repos on neighbouring infrastructure. diff --git a/bbot/modules/gowitness.py b/bbot/modules/gowitness.py index b26018d58c..46ef779b4e 100644 --- a/bbot/modules/gowitness.py +++ b/bbot/modules/gowitness.py @@ -9,6 +9,8 @@ from shutil import copyfile, copymode from bbot.modules.base import BaseModule +from pydantic import Field +from bbot.core.config.models import BaseModuleConfig class gowitness(BaseModule): @@ -16,28 +18,16 @@ class gowitness(BaseModule): produced_events = ["WEBSCREENSHOT", "URL", "URL_UNVERIFIED", "TECHNOLOGY"] flags = ["safe", "active", "web-screenshots"] meta = {"description": "Take screenshots of webpages", "created_date": "2022-07-08", "author": "@TheTechromancer"} - options = { - "version": "3.1.1", - "threads": 0, - "timeout": 10, - "resolution_x": 1440, - "resolution_y": 900, - "output_path": "", - "social": False, - "idle_timeout": 1800, - "chrome_path": "", - } - options_desc = { - "version": "Gowitness version", - "threads": "How many gowitness threads to spawn (default is number of CPUs x 2)", - "timeout": "Preflight check timeout", - "resolution_x": "Screenshot resolution x", - "resolution_y": "Screenshot resolution y", - "output_path": "Where to save screenshots", - "social": "Whether to screenshot social media webpages", - "idle_timeout": "Skip the current gowitness batch if it stalls for longer than this many seconds", - "chrome_path": "Path to chrome executable", - } + class Config(BaseModuleConfig): + version: str = Field('3.1.1', description='Gowitness version') + threads: int = Field(0, description='How many gowitness threads to spawn (default is number of CPUs x 2)') + timeout: int = Field(10, description='Preflight check timeout') + resolution_x: int = Field(1440, description='Screenshot resolution x') + resolution_y: int = Field(900, description='Screenshot resolution y') + output_path: str = Field('', description='Where to save screenshots') + social: bool = Field(False, description='Whether to screenshot social media webpages') + idle_timeout: int = Field(1800, description='Skip the current gowitness batch if it stalls for longer than this many seconds') + chrome_path: str = Field('', description='Path to chrome executable') deps_common = ["chromium"] deps_pip = ["aiosqlite"] deps_ansible = [ diff --git a/bbot/modules/graphql_introspection.py b/bbot/modules/graphql_introspection.py index 5cfcb7c16c..246e103abc 100644 --- a/bbot/modules/graphql_introspection.py +++ b/bbot/modules/graphql_introspection.py @@ -1,6 +1,8 @@ import json from pathlib import Path from bbot.modules.base import BaseModule +from pydantic import Field +from bbot.core.config.models import BaseModuleConfig class graphql_introspection(BaseModule): @@ -12,14 +14,9 @@ class graphql_introspection(BaseModule): "created_date": "2025-07-01", "author": "@mukesh-dream11", } - options = { - "graphql_endpoint_urls": ["/", "/graphql", "/v1/graphql"], - "output_folder": "", - } - options_desc = { - "graphql_endpoint_urls": "List of GraphQL endpoint to suffix to the target URL", - "output_folder": "Folder to save the GraphQL schemas to", - } + class Config(BaseModuleConfig): + graphql_endpoint_urls: list[str] = Field(['/', '/graphql', '/v1/graphql'], description='List of GraphQL endpoint to suffix to the target URL') + output_folder: str = Field('', description='Folder to save the GraphQL schemas to') async def setup(self): output_folder = self.config.get("output_folder", "") diff --git a/bbot/modules/httpx.py b/bbot/modules/httpx.py index 860894015b..a810d5edab 100644 --- a/bbot/modules/httpx.py +++ b/bbot/modules/httpx.py @@ -7,6 +7,8 @@ from http.cookies import SimpleCookie from bbot.modules.base import BaseModule +from pydantic import Field +from bbot.core.config.models import BaseModuleConfig class httpx(BaseModule): @@ -20,22 +22,13 @@ class httpx(BaseModule): "author": "@TheTechromancer", } - options = { - "threads": 50, - "in_scope_only": True, - "version": "1.2.5", - "max_response_size": 5242880, - "store_responses": False, - "probe_all_ips": False, - } - options_desc = { - "threads": "Number of httpx threads to use", - "in_scope_only": "Only visit web reparents that are in scope.", - "version": "httpx version", - "max_response_size": "Max response size in bytes", - "store_responses": "Save raw HTTP responses to scan folder", - "probe_all_ips": "Probe all the ips associated with same host", - } + class Config(BaseModuleConfig): + threads: int = Field(50, description='Number of httpx threads to use') + in_scope_only: bool = Field(True, description='Only visit web reparents that are in scope.') + version: str = Field('1.2.5', description='httpx version') + max_response_size: int = Field(5242880, description='Max response size in bytes') + store_responses: bool = Field(False, description='Save raw HTTP responses to scan folder') + probe_all_ips: bool = Field(False, description='Probe all the ips associated with same host') deps_ansible = [ { "name": "Download httpx", diff --git a/bbot/modules/hunterio.py b/bbot/modules/hunterio.py index f732addf3e..832c3aa792 100644 --- a/bbot/modules/hunterio.py +++ b/bbot/modules/hunterio.py @@ -1,4 +1,6 @@ from bbot.modules.templates.subdomain_enum import subdomain_enum_apikey +from pydantic import Field +from bbot.core.config.models import BaseModuleConfig class hunterio(subdomain_enum_apikey): @@ -11,8 +13,8 @@ class hunterio(subdomain_enum_apikey): "author": "@TheTechromancer", "auth_required": True, } - options = {"api_key": ""} - options_desc = {"api_key": "Hunter.IO API key"} + class Config(BaseModuleConfig): + api_key: str = Field('', description='Hunter.IO API key') base_url = "https://api.hunter.io/v2" ping_url = f"{base_url}/account?api_key={{api_key}}" diff --git a/bbot/modules/iis_shortnames.py b/bbot/modules/iis_shortnames.py index 42c5cb8693..5be0e95793 100644 --- a/bbot/modules/iis_shortnames.py +++ b/bbot/modules/iis_shortnames.py @@ -1,6 +1,8 @@ import re from bbot.modules.base import BaseModule +from pydantic import Field +from bbot.core.config.models import BaseModuleConfig valid_chars = "ETAONRISHDLFCMUGYPWBVKJXQZ0123456789_-$~()&!#%'@^`{}]]" @@ -22,12 +24,10 @@ class iis_shortnames(BaseModule): "created_date": "2022-04-15", "author": "@liquidsec", } - options = {"detect_only": True, "max_node_count": 50, "speculate_magic_urls": True} - options_desc = { - "detect_only": "Only detect the vulnerability and do not run the shortname scanner", - "max_node_count": "Limit how many nodes to attempt to resolve on any given recursion branch", - "speculate_magic_urls": "Attempt to discover iis 'magic' special folders", - } + class Config(BaseModuleConfig): + detect_only: bool = Field(True, description='Only detect the vulnerability and do not run the shortname scanner') + max_node_count: int = Field(50, description='Limit how many nodes to attempt to resolve on any given recursion branch') + speculate_magic_urls: bool = Field(True, description="Attempt to discover iis 'magic' special folders") in_scope_only = True _module_threads = 8 diff --git a/bbot/modules/internal/excavate.py b/bbot/modules/internal/excavate.py index 5a0fc04d43..4c84b1b870 100644 --- a/bbot/modules/internal/excavate.py +++ b/bbot/modules/internal/excavate.py @@ -10,6 +10,8 @@ from bbot.modules.base import BaseInterceptModule from bbot.modules.internal.base import BaseInternalModule from urllib.parse import urlparse, urljoin, parse_qs, urlunparse, urldefrag +from pydantic import Field +from bbot.core.config.models import BaseModuleConfig def find_subclasses(obj, base_class): @@ -335,16 +337,10 @@ class excavateTestRule(ExcavateRule): "author": "@liquidsec", } - options = { - "yara_max_match_data": 2000, - "custom_yara_rules": "", - "speculate_params": False, - } - options_desc = { - "yara_max_match_data": "Sets the maximum amount of text that can extracted from a YARA regex", - "custom_yara_rules": "Include custom Yara rules", - "speculate_params": "Enable speculative parameter extraction from JSON and XML content", - } + class Config(BaseModuleConfig): + yara_max_match_data: int = Field(2000, description='Sets the maximum amount of text that can extracted from a YARA regex') + custom_yara_rules: str = Field('', description='Include custom Yara rules') + speculate_params: bool = Field(False, description='Enable speculative parameter extraction from JSON and XML content') scope_distance_modifier = None accept_dupes = False diff --git a/bbot/modules/internal/speculate.py b/bbot/modules/internal/speculate.py index 0e31e9158b..2eae9124e5 100644 --- a/bbot/modules/internal/speculate.py +++ b/bbot/modules/internal/speculate.py @@ -3,6 +3,8 @@ from bbot.core.helpers import validators from bbot.modules.internal.base import BaseInternalModule +from pydantic import Field +from bbot.core.config.models import BaseModuleConfig class speculate(BaseInternalModule): @@ -32,12 +34,10 @@ class speculate(BaseInternalModule): "author": "@liquidsec", } - options = {"ip_range_max_hosts": 65536, "ports": "80,443", "essential_only": False} - options_desc = { - "ip_range_max_hosts": "Max number of hosts an IP_RANGE can contain to allow conversion into IP_ADDRESS events", - "ports": "The set of ports to speculate on", - "essential_only": "Only enable essential speculate features (no extra discovery)", - } + class Config(BaseModuleConfig): + ip_range_max_hosts: int = Field(65536, description='Max number of hosts an IP_RANGE can contain to allow conversion into IP_ADDRESS events') + ports: str = Field('80,443', description='The set of ports to speculate on') + essential_only: bool = Field(False, description='Only enable essential speculate features (no extra discovery)') scope_distance_modifier = 1 _priority = 4 diff --git a/bbot/modules/ip2location.py b/bbot/modules/ip2location.py index 80625f2852..8abf7fcc69 100644 --- a/bbot/modules/ip2location.py +++ b/bbot/modules/ip2location.py @@ -1,4 +1,6 @@ from bbot.modules.base import BaseModule +from pydantic import Field +from bbot.core.config.models import BaseModuleConfig class IP2Location(BaseModule): @@ -15,11 +17,9 @@ class IP2Location(BaseModule): "author": "@TheTechromancer", "auth_required": True, } - options = {"api_key": "", "lang": ""} - options_desc = { - "api_key": "IP2location.io API Key", - "lang": "Translation information(ISO639-1). The translation is only applicable for continent, country, region and city name.", - } + class Config(BaseModuleConfig): + api_key: str = Field('', description='IP2location.io API Key') + lang: str = Field('', description='Translation information(ISO639-1). The translation is only applicable for continent, country, region and city name.') scope_distance_modifier = 1 _priority = 2 suppress_dupes = False diff --git a/bbot/modules/ipneighbor.py b/bbot/modules/ipneighbor.py index f5f1007b00..badb48cd48 100644 --- a/bbot/modules/ipneighbor.py +++ b/bbot/modules/ipneighbor.py @@ -1,6 +1,8 @@ import ipaddress from bbot.modules.base import BaseModule +from pydantic import Field +from bbot.core.config.models import BaseModuleConfig class ipneighbor(BaseModule): @@ -12,8 +14,8 @@ class ipneighbor(BaseModule): "created_date": "2022-06-08", "author": "@TheTechromancer", } - options = {"num_bits": 4} - options_desc = {"num_bits": "Netmask size (in CIDR notation) to check. Default is 4 bits (16 hosts)"} + class Config(BaseModuleConfig): + num_bits: int = Field(4, description='Netmask size (in CIDR notation) to check. Default is 4 bits (16 hosts)') scope_distance_modifier = 1 async def setup(self): diff --git a/bbot/modules/ipstack.py b/bbot/modules/ipstack.py index 6a00935f96..f298c12b55 100644 --- a/bbot/modules/ipstack.py +++ b/bbot/modules/ipstack.py @@ -1,4 +1,6 @@ from bbot.modules.base import BaseModule +from pydantic import Field +from bbot.core.config.models import BaseModuleConfig class Ipstack(BaseModule): @@ -16,8 +18,8 @@ class Ipstack(BaseModule): "author": "@tycoonslive", "auth_required": True, } - options = {"api_key": ""} - options_desc = {"api_key": "IPStack GeoIP API Key"} + class Config(BaseModuleConfig): + api_key: str = Field('', description='IPStack GeoIP API Key') scope_distance_modifier = 1 _priority = 2 suppress_dupes = False diff --git a/bbot/modules/jadx.py b/bbot/modules/jadx.py index d2f8437af6..44ae831e49 100644 --- a/bbot/modules/jadx.py +++ b/bbot/modules/jadx.py @@ -1,6 +1,8 @@ from pathlib import Path from subprocess import CalledProcessError from bbot.modules.internal.base import BaseModule +from pydantic import Field +from bbot.core.config.models import BaseModuleConfig class jadx(BaseModule): @@ -12,12 +14,8 @@ class jadx(BaseModule): "created_date": "2024-11-04", "author": "@domwhewell-sage", } - options = { - "threads": 4, - } - options_desc = { - "threads": "Maximum jadx threads for extracting apk's, default: 4", - } + class Config(BaseModuleConfig): + threads: int = Field(4, description="Maximum jadx threads for extracting apk's, default: 4") deps_common = ["java"] deps_ansible = [ { diff --git a/bbot/modules/kreuzberg.py b/bbot/modules/kreuzberg.py index 373168180c..a71ce524b5 100644 --- a/bbot/modules/kreuzberg.py +++ b/bbot/modules/kreuzberg.py @@ -2,6 +2,8 @@ from kreuzberg import extract_file from bbot.modules.base import BaseModule +from pydantic import Field +from bbot.core.config.models import BaseModuleConfig class kreuzberg(BaseModule): @@ -13,58 +15,8 @@ class kreuzberg(BaseModule): "created_date": "2024-06-03", "author": "@domwhewell-sage", } - options = { - "extensions": [ - "bak", # Backup File - "bash", # Bash Script or Configuration - "bashrc", # Bash Script or Configuration - "conf", # Configuration File - "cfg", # Configuration File - "crt", # Certificate File - "csv", # Comma Separated Values File - "db", # SQLite Database File - "sqlite", # SQLite Database File - "doc", # Microsoft Word Document (Old Format) - "docx", # Microsoft Word Document - "ica", # Citrix Independent Computing Architecture File - "indd", # Adobe InDesign Document - "ini", # Initialization File - "json", # JSON File - "key", # Private Key File - "pub", # Public Key File - "log", # Log File - "markdown", # Markdown File - "md", # Markdown File - "odg", # OpenDocument Graphics (LibreOffice, OpenOffice) - "odp", # OpenDocument Presentation (LibreOffice, OpenOffice) - "ods", # OpenDocument Spreadsheet (LibreOffice, OpenOffice) - "odt", # OpenDocument Text (LibreOffice, OpenOffice) - "pdf", # Adobe Portable Document Format - "pem", # Privacy Enhanced Mail (SSL certificate) - "pps", # Microsoft PowerPoint Slideshow (Old Format) - "ppsx", # Microsoft PowerPoint Slideshow - "ppt", # Microsoft PowerPoint Presentation (Old Format) - "pptx", # Microsoft PowerPoint Presentation - "ps1", # PowerShell Script - "rdp", # Remote Desktop Protocol File - "rsa", # RSA Private Key File - "sh", # Shell Script - "sql", # SQL Database Dump - "swp", # Swap File (temporary file, often Vim) - "sxw", # OpenOffice.org Writer document - "txt", # Plain Text Document - "vbs", # Visual Basic Script - "wpd", # WordPerfect Document - "xls", # Microsoft Excel Spreadsheet (Old Format) - "xlsx", # Microsoft Excel Spreadsheet - "xml", # eXtensible Markup Language File - "yml", # YAML Ain't Markup Language - "yaml", # YAML Ain't Markup Language - ], - } - options_desc = { - "extensions": "File extensions to parse", - } + class Config(BaseModuleConfig): + extensions: list[str] = Field(['bak', 'bash', 'bashrc', 'conf', 'cfg', 'crt', 'csv', 'db', 'sqlite', 'doc', 'docx', 'ica', 'indd', 'ini', 'json', 'key', 'pub', 'log', 'markdown', 'md', 'odg', 'odp', 'ods', 'odt', 'pdf', 'pem', 'pps', 'ppsx', 'ppt', 'pptx', 'ps1', 'rdp', 'rsa', 'sh', 'sql', 'swp', 'sxw', 'txt', 'vbs', 'wpd', 'xls', 'xlsx', 'xml', 'yml', 'yaml'], description='File extensions to parse') deps_pip = ["kreuzberg>=4.3,<4.5", "pypdfium2~=5.0"] scope_distance_modifier = 1 diff --git a/bbot/modules/leakix.py b/bbot/modules/leakix.py index 1f79dc8b12..f4f44bbd09 100644 --- a/bbot/modules/leakix.py +++ b/bbot/modules/leakix.py @@ -1,13 +1,15 @@ from bbot.modules.templates.subdomain_enum import subdomain_enum_apikey +from pydantic import Field +from bbot.core.config.models import BaseModuleConfig class leakix(subdomain_enum_apikey): watched_events = ["DNS_NAME"] produced_events = ["DNS_NAME"] flags = ["safe", "subdomain-enum", "passive"] - options = {"api_key": ""} + class Config(BaseModuleConfig): + api_key: str = Field('', description='LeakIX API Key') # NOTE: API key is not required (but having one will get you more results) - options_desc = {"api_key": "LeakIX API Key"} meta = { "description": "Query leakix.net for subdomains", "created_date": "2022-07-11", diff --git a/bbot/modules/legba.py b/bbot/modules/legba.py index 66b73cde42..f76a73b038 100644 --- a/bbot/modules/legba.py +++ b/bbot/modules/legba.py @@ -2,6 +2,8 @@ from pathlib import Path from bbot.errors import WordlistError from bbot.modules.base import BaseModule +from pydantic import Field +from bbot.core.config.models import BaseModuleConfig # key: value: # List with `legba -L` @@ -28,31 +30,18 @@ class legba(BaseModule): _module_threads = 25 scope_distance_modifier = None - options = { - "ssh_wordlist": "https://raw.githubusercontent.com/danielmiessler/SecLists/refs/heads/master/Passwords/Default-Credentials/ssh-betterdefaultpasslist.txt", - "ftp_wordlist": "https://raw.githubusercontent.com/danielmiessler/SecLists/refs/heads/master/Passwords/Default-Credentials/ftp-betterdefaultpasslist.txt", - "telnet_wordlist": "https://raw.githubusercontent.com/danielmiessler/SecLists/refs/heads/master/Passwords/Default-Credentials/telnet-betterdefaultpasslist.txt", - "vnc_wordlist": "https://raw.githubusercontent.com/danielmiessler/SecLists/refs/heads/master/Passwords/Default-Credentials/vnc-betterdefaultpasslist.txt", - "mssql_wordlist": "https://raw.githubusercontent.com/danielmiessler/SecLists/refs/heads/master/Passwords/Default-Credentials/mssql-betterdefaultpasslist.txt", - "mysql_wordlist": "https://raw.githubusercontent.com/danielmiessler/SecLists/refs/heads/master/Passwords/Default-Credentials/mysql-betterdefaultpasslist.txt", - "postgresql_wordlist": "https://raw.githubusercontent.com/danielmiessler/SecLists/refs/heads/master/Passwords/Default-Credentials/postgres-betterdefaultpasslist.txt", - "concurrency": 3, - "rate_limit": 3, - "version": "1.1.1", - } + class Config(BaseModuleConfig): + ssh_wordlist: str = Field('https://raw.githubusercontent.com/danielmiessler/SecLists/refs/heads/master/Passwords/Default-Credentials/ssh-betterdefaultpasslist.txt', description='Wordlist URL for SSH combined username:password wordlist, newline separated') + ftp_wordlist: str = Field('https://raw.githubusercontent.com/danielmiessler/SecLists/refs/heads/master/Passwords/Default-Credentials/ftp-betterdefaultpasslist.txt', description='Wordlist URL for FTP combined username:password wordlist, newline separated') + telnet_wordlist: str = Field('https://raw.githubusercontent.com/danielmiessler/SecLists/refs/heads/master/Passwords/Default-Credentials/telnet-betterdefaultpasslist.txt', description='Wordlist URL for TELNET combined username:password wordlist, newline separated') + vnc_wordlist: str = Field('https://raw.githubusercontent.com/danielmiessler/SecLists/refs/heads/master/Passwords/Default-Credentials/vnc-betterdefaultpasslist.txt', description='Wordlist URL for VNC password wordlist, newline separated') + mssql_wordlist: str = Field('https://raw.githubusercontent.com/danielmiessler/SecLists/refs/heads/master/Passwords/Default-Credentials/mssql-betterdefaultpasslist.txt', description='Wordlist URL for MSSQL combined username:password wordlist, newline separated') + mysql_wordlist: str = Field('https://raw.githubusercontent.com/danielmiessler/SecLists/refs/heads/master/Passwords/Default-Credentials/mysql-betterdefaultpasslist.txt', description='Wordlist URL for MySQL combined username:password wordlist, newline separated') + postgresql_wordlist: str = Field('https://raw.githubusercontent.com/danielmiessler/SecLists/refs/heads/master/Passwords/Default-Credentials/postgres-betterdefaultpasslist.txt', description='Wordlist URL for PostgreSQL combined username:password wordlist, newline separated') + concurrency: int = Field(3, description='Number of concurrent workers, gets overridden for SSH') + rate_limit: int = Field(3, description='Limit the number of requests per second, gets overridden for SSH') + version: str = Field('1.1.1', description='legba version') - options_desc = { - "ssh_wordlist": "Wordlist URL for SSH combined username:password wordlist, newline separated", - "ftp_wordlist": "Wordlist URL for FTP combined username:password wordlist, newline separated", - "telnet_wordlist": "Wordlist URL for TELNET combined username:password wordlist, newline separated", - "vnc_wordlist": "Wordlist URL for VNC password wordlist, newline separated", - "mssql_wordlist": "Wordlist URL for MSSQL combined username:password wordlist, newline separated", - "mysql_wordlist": "Wordlist URL for MySQL combined username:password wordlist, newline separated", - "postgresql_wordlist": "Wordlist URL for PostgreSQL combined username:password wordlist, newline separated", - "concurrency": "Number of concurrent workers, gets overridden for SSH", - "rate_limit": "Limit the number of requests per second, gets overridden for SSH", - "version": "legba version", - } deps_ansible = [ { diff --git a/bbot/modules/lightfuzz/lightfuzz.py b/bbot/modules/lightfuzz/lightfuzz.py index 231081006f..ba9989dd24 100644 --- a/bbot/modules/lightfuzz/lightfuzz.py +++ b/bbot/modules/lightfuzz/lightfuzz.py @@ -2,6 +2,8 @@ from bbot.modules.base import BaseModule from bbot.errors import InteractshError +from pydantic import Field +from bbot.core.config.models import BaseModuleConfig class lightfuzz(BaseModule): @@ -9,22 +11,13 @@ class lightfuzz(BaseModule): produced_events = ["FINDING"] flags = ["active", "loud", "web-heavy", "invasive"] - options = { - "force_common_headers": False, - "enabled_submodules": ["sqli", "cmdi", "xss", "path", "ssti", "crypto", "serial", "esi", "ssrf"], - "disable_post": False, - "try_post_as_get": False, - "try_get_as_post": False, - "avoid_wafs": True, - } - options_desc = { - "force_common_headers": "Force emit commonly exploitable parameters that may be difficult to detect", - "enabled_submodules": "A list of submodules to enable. Empty list enabled all modules.", - "disable_post": "Disable processing of POST parameters, avoiding form submissions.", - "try_post_as_get": "For each POSTPARAM, also fuzz it as a GETPARAM (in addition to normal POST fuzzing).", - "try_get_as_post": "For each GETPARAM, also fuzz it as a POSTPARAM (in addition to normal GET fuzzing).", - "avoid_wafs": "Avoid running against confirmed WAFs, which are likely to block lightfuzz requests", - } + class Config(BaseModuleConfig): + force_common_headers: bool = Field(False, description='Force emit commonly exploitable parameters that may be difficult to detect') + enabled_submodules: list[str] = Field(['sqli', 'cmdi', 'xss', 'path', 'ssti', 'crypto', 'serial', 'esi', 'ssrf'], description='A list of submodules to enable. Empty list enabled all modules.') + disable_post: bool = Field(False, description='Disable processing of POST parameters, avoiding form submissions.') + try_post_as_get: bool = Field(False, description='For each POSTPARAM, also fuzz it as a GETPARAM (in addition to normal POST fuzzing).') + try_get_as_post: bool = Field(False, description='For each GETPARAM, also fuzz it as a POSTPARAM (in addition to normal GET fuzzing).') + avoid_wafs: bool = Field(True, description='Avoid running against confirmed WAFs, which are likely to block lightfuzz requests') meta = { "description": "Find Web Parameters and Lightly Fuzz them using a heuristic based scanner", diff --git a/bbot/modules/medusa.py b/bbot/modules/medusa.py index a3f8cebd85..5184eef871 100644 --- a/bbot/modules/medusa.py +++ b/bbot/modules/medusa.py @@ -1,5 +1,7 @@ import re from bbot.modules.base import BaseModule +from pydantic import Field +from bbot.core.config.models import BaseModuleConfig class medusa(BaseModule): @@ -14,21 +16,13 @@ class medusa(BaseModule): } scope_distance_modifier = None - options = { - "snmp_wordlist": "https://raw.githubusercontent.com/danielmiessler/SecLists/refs/heads/master/Discovery/SNMP/common-snmp-community-strings.txt", - "snmp_versions": ["1", "2C"], # Only 1 and 2C are available with medusa 2.3. - "wait_microseconds": 200, - "timeout_s": 5, - "threads": 5, - } + class Config(BaseModuleConfig): + snmp_wordlist: str = Field('https://raw.githubusercontent.com/danielmiessler/SecLists/refs/heads/master/Discovery/SNMP/common-snmp-community-strings.txt', description='Wordlist url for SNMP community strings, newline separated (default https://raw.githubusercontent.com/danielmiessler/SecLists/refs/heads/master/Discovery/SNMP/snmp.txt)') + snmp_versions: list[str] = Field(['1', '2C'], description="List of SNMP versions to attempt against the SNMP server (default ['1', '2C'])") + wait_microseconds: int = Field(200, description='Wait time after every SNMP request in microseconds (default 200)') + timeout_s: int = Field(5, description='Wait time for the SNMP response(s) once at the end of all attempts (default 5)') + threads: int = Field(5, description='Number of communities to be tested concurrently (default 5)') - options_desc = { - "snmp_wordlist": "Wordlist url for SNMP community strings, newline separated (default https://raw.githubusercontent.com/danielmiessler/SecLists/refs/heads/master/Discovery/SNMP/snmp.txt)", - "snmp_versions": "List of SNMP versions to attempt against the SNMP server (default ['1', '2C'])", - "wait_microseconds": "Wait time after every SNMP request in microseconds (default 200)", - "timeout_s": "Wait time for the SNMP response(s) once at the end of all attempts (default 5)", - "threads": "Number of communities to be tested concurrently (default 5)", - } deps_ansible = [ { diff --git a/bbot/modules/ntlm.py b/bbot/modules/ntlm.py index 164f26efd5..19d29af238 100644 --- a/bbot/modules/ntlm.py +++ b/bbot/modules/ntlm.py @@ -1,5 +1,7 @@ from bbot.errors import NTLMError from bbot.modules.base import BaseModule +from pydantic import Field +from bbot.core.config.models import BaseModuleConfig ntlm_discovery_endpoints = [ "", @@ -74,8 +76,8 @@ class ntlm(BaseModule): "created_date": "2022-07-25", "author": "@liquidsec", } - options = {"try_all": False} - options_desc = {"try_all": "Try every NTLM endpoint"} + class Config(BaseModuleConfig): + try_all: bool = Field(False, description='Try every NTLM endpoint') in_scope_only = True diff --git a/bbot/modules/nuclei.py b/bbot/modules/nuclei.py index 1965a1c669..579f2dbf84 100644 --- a/bbot/modules/nuclei.py +++ b/bbot/modules/nuclei.py @@ -1,7 +1,10 @@ import json import yaml +from typing import Literal from itertools import islice from bbot.modules.base import BaseModule +from pydantic import Field +from bbot.core.config.models import BaseModuleConfig class nuclei(BaseModule): @@ -14,38 +17,30 @@ class nuclei(BaseModule): "author": "@TheTechromancer", } - options = { - "version": "3.8.0", - "tags": "", - "templates": "", - "severity": "", - "ratelimit": 150, - "concurrency": 25, - "mode": "manual", - "etags": "", - "budget": 1, - "silent": False, - "directory_only": True, - "retries": 0, - "batch_size": 200, - "module_timeout": 21600, # 6 hours - } - options_desc = { - "version": "nuclei version", - "tags": "execute a subset of templates that contain the provided tags", - "templates": "template or template directory paths to include in the scan", - "severity": "Filter based on severity field available in the template.", - "ratelimit": "maximum number of requests to send per second (default 150)", - "concurrency": "maximum number of templates to be executed in parallel (default 25)", - "mode": "manual | technology | severe | budget. Technology: Only activate based on technology events that match nuclei tags (nuclei -as mode). Manual (DEFAULT): Fully manual settings. Severe: Only critical and high severity templates without intrusive. Budget: Limit Nuclei to a specified number of HTTP requests", - "etags": "tags to exclude from the scan", - "budget": "Used in budget mode to set the number of allowed requests per host", - "silent": "Don't display nuclei's banner or status messages", - "directory_only": "Filter out 'file' URL event (default True)", - "retries": "number of times to retry a failed request (default 0)", - "batch_size": "Number of targets to send to Nuclei per batch (default 200)", - "module_timeout": "Max time in seconds to spend handling each batch of events", - } + class Config(BaseModuleConfig): + version: str = Field('3.8.0', description='nuclei version') + tags: str = Field('', description='execute a subset of templates that contain the provided tags') + templates: str = Field('', description='template or template directory paths to include in the scan') + severity: str = Field('', description='Filter based on severity field available in the template.') + ratelimit: int = Field(150, description='maximum number of requests to send per second (default 150)') + concurrency: int = Field(25, description='maximum number of templates to be executed in parallel (default 25)') + mode: Literal["manual", "technology", "severe", "budget"] = Field( + "manual", + description=( + "manual | technology | severe | budget. " + "Technology: Only activate based on technology events that match nuclei tags (nuclei -as mode). " + "Manual (DEFAULT): Fully manual settings. " + "Severe: Only critical and high severity templates without intrusive. " + "Budget: Limit Nuclei to a specified number of HTTP requests" + ), + ) + etags: str = Field('', description='tags to exclude from the scan') + budget: int = Field(1, description='Used in budget mode to set the number of allowed requests per host') + silent: bool = Field(False, description="Don't display nuclei's banner or status messages") + directory_only: bool = Field(True, description="Filter out 'file' URL event (default True)") + retries: int = Field(0, description='number of times to retry a failed request (default 0)') + batch_size: int = Field(200, description='Number of targets to send to Nuclei per batch (default 200)') + module_timeout: int = Field(21600, description='Max time in seconds to spend handling each batch of events') deps_ansible = [ { "name": "Download nuclei", diff --git a/bbot/modules/oauth.py b/bbot/modules/oauth.py index 7bcbc972fa..0843f25353 100644 --- a/bbot/modules/oauth.py +++ b/bbot/modules/oauth.py @@ -1,6 +1,8 @@ from bbot.core.helpers.regexes import url_regexes from .base import BaseModule +from pydantic import Field +from bbot.core.config.models import BaseModuleConfig class OAUTH(BaseModule): @@ -12,8 +14,8 @@ class OAUTH(BaseModule): "created_date": "2023-07-12", "author": "@TheTechromancer", } - options = {"try_all": False} - options_desc = {"try_all": "Check for OAUTH/IODC on every subdomain and URL."} + class Config(BaseModuleConfig): + try_all: bool = Field(False, description='Check for OAUTH/IODC on every subdomain and URL.') in_scope_only = False scope_distance_modifier = 1 diff --git a/bbot/modules/otx.py b/bbot/modules/otx.py index a22a5636b5..40100a81d3 100644 --- a/bbot/modules/otx.py +++ b/bbot/modules/otx.py @@ -1,4 +1,6 @@ from bbot.modules.templates.subdomain_enum import subdomain_enum_apikey +from pydantic import Field +from bbot.core.config.models import BaseModuleConfig class otx(subdomain_enum_apikey): @@ -11,8 +13,8 @@ class otx(subdomain_enum_apikey): "author": "@TheTechromancer", "auth_required": True, } - options = {"api_key": ""} - options_desc = {"api_key": "OTX API key"} + class Config(BaseModuleConfig): + api_key: str = Field('', description='OTX API key') base_url = "https://otx.alienvault.com" diff --git a/bbot/modules/output/asset_inventory.py b/bbot/modules/output/asset_inventory.py index 7901d6ac8b..22d91e3542 100644 --- a/bbot/modules/output/asset_inventory.py +++ b/bbot/modules/output/asset_inventory.py @@ -4,6 +4,8 @@ from .csv import CSV from bbot.core.helpers.misc import make_ip_type, is_ip, is_port, best_http_status +from pydantic import Field +from bbot.core.config.models import BaseModuleConfig severity_map = { "INFO": 0, @@ -37,13 +39,11 @@ class asset_inventory(CSV): "created_date": "2022-09-30", "author": "@liquidsec", } - options = {"output_file": "", "use_previous": False, "recheck": False, "summary_netmask": 16} - options_desc = { - "output_file": "Set a custom output file", - "use_previous": "Emit previous asset inventory as new events (use in conjunction with -n )", - "recheck": "When use_previous=True, don't retain past details like open ports or findings. Instead, allow them to be rediscovered by the new scan", - "summary_netmask": "Subnet mask to use when summarizing IP addresses at end of scan", - } + class Config(BaseModuleConfig): + output_file: str = Field('', description='Set a custom output file') + use_previous: bool = Field(False, description='Emit previous asset inventory as new events (use in conjunction with -n )') + recheck: bool = Field(False, description="When use_previous=True, don't retain past details like open ports or findings. Instead, allow them to be rediscovered by the new scan") + summary_netmask: int = Field(16, description='Subnet mask to use when summarizing IP addresses at end of scan') header_row = [ "Host", diff --git a/bbot/modules/output/csv.py b/bbot/modules/output/csv.py index 9b7d4b4bd9..677e04ebd9 100644 --- a/bbot/modules/output/csv.py +++ b/bbot/modules/output/csv.py @@ -2,13 +2,15 @@ from contextlib import suppress from bbot.modules.output.base import BaseOutputModule +from pydantic import Field +from bbot.core.config.models import BaseModuleConfig class CSV(BaseOutputModule): watched_events = ["*"] meta = {"description": "Output to CSV", "created_date": "2022-04-07", "author": "@TheTechromancer"} - options = {"output_file": ""} - options_desc = {"output_file": "Output to CSV file"} + class Config(BaseModuleConfig): + output_file: str = Field('', description='Output to CSV file') header_row = [ "Event type", diff --git a/bbot/modules/output/discord.py b/bbot/modules/output/discord.py index 934d89e670..f77ada86d8 100644 --- a/bbot/modules/output/discord.py +++ b/bbot/modules/output/discord.py @@ -1,4 +1,6 @@ from bbot.modules.templates.webhook import WebhookOutputModule +from pydantic import Field +from bbot.core.config.models import BaseModuleConfig class Discord(WebhookOutputModule): @@ -8,10 +10,8 @@ class Discord(WebhookOutputModule): "created_date": "2023-08-14", "author": "@TheTechromancer", } - options = {"webhook_url": "", "event_types": ["FINDING"], "min_severity": "LOW", "retries": 10} - options_desc = { - "webhook_url": "Discord webhook URL", - "event_types": "Types of events to send", - "min_severity": "Only allow FINDING events of this severity or higher", - "retries": "Number of times to retry sending the message before skipping the event", - } + class Config(BaseModuleConfig): + webhook_url: str = Field('', description='Discord webhook URL') + event_types: list[str] = Field(['FINDING'], description='Types of events to send') + min_severity: str = Field('LOW', description='Only allow FINDING events of this severity or higher') + retries: int = Field(10, description='Number of times to retry sending the message before skipping the event') diff --git a/bbot/modules/output/elastic.py b/bbot/modules/output/elastic.py index 064c00af7c..b35e6ab19c 100644 --- a/bbot/modules/output/elastic.py +++ b/bbot/modules/output/elastic.py @@ -1,4 +1,6 @@ from .http import HTTP +from pydantic import Field +from bbot.core.config.models import BaseModuleConfig class Elastic(HTTP): @@ -12,18 +14,11 @@ class Elastic(HTTP): "created_date": "2022-11-21", "author": "@TheTechromancer", } - options = { - "url": "https://localhost:9200/bbot_events/_doc", - "username": "elastic", - "password": "bbotislife", - "timeout": 10, - } - options_desc = { - "url": "Elastic URL (e.g. https://localhost:9200//_doc)", - "username": "Elastic username", - "password": "Elastic password", - "timeout": "HTTP timeout", - } + class Config(BaseModuleConfig): + url: str = Field('https://localhost:9200/bbot_events/_doc', description='Elastic URL (e.g. https://localhost:9200//_doc)') + username: str = Field('elastic', description='Elastic username') + password: str = Field('bbotislife', description='Elastic password') + timeout: int = Field(10, description='HTTP timeout') async def cleanup(self): # refresh the index diff --git a/bbot/modules/output/emails.py b/bbot/modules/output/emails.py index 6bfe940689..3fa9c1a839 100644 --- a/bbot/modules/output/emails.py +++ b/bbot/modules/output/emails.py @@ -1,5 +1,7 @@ from bbot.modules.output.txt import TXT from bbot.modules.base import BaseModule +from pydantic import Field +from bbot.core.config.models import BaseModuleConfig class Emails(TXT): @@ -10,8 +12,8 @@ class Emails(TXT): "created_date": "2023-12-23", "author": "@domwhewell-sage", } - options = {"output_file": ""} - options_desc = {"output_file": "Output to file"} + class Config(BaseModuleConfig): + output_file: str = Field('', description='Output to file') in_scope_only = True accept_dupes = False diff --git a/bbot/modules/output/http.py b/bbot/modules/output/http.py index e2ad87d898..c79f4eb0dd 100644 --- a/bbot/modules/output/http.py +++ b/bbot/modules/output/http.py @@ -1,5 +1,7 @@ from bbot.models.pydantic import Event from bbot.modules.output.base import BaseOutputModule +from pydantic import Field +from bbot.core.config.models import BaseModuleConfig class HTTP(BaseOutputModule): @@ -9,24 +11,14 @@ class HTTP(BaseOutputModule): "created_date": "2022-04-13", "author": "@TheTechromancer", } - options = { - "url": "", - "method": "POST", - "bearer": "", - "username": "", - "password": "", - "headers": {}, - "timeout": 10, - } - options_desc = { - "url": "Web URL", - "method": "HTTP method", - "bearer": "Authorization Bearer token", - "username": "Username (basic auth)", - "password": "Password (basic auth)", - "headers": "Additional headers to send with the request", - "timeout": "HTTP timeout", - } + class Config(BaseModuleConfig): + url: str = Field('', description='Web URL') + method: str = Field('POST', description='HTTP method') + bearer: str = Field('', description='Authorization Bearer token') + username: str = Field('', description='Username (basic auth)') + password: str = Field('', description='Password (basic auth)') + headers: dict = Field({}, description='Additional headers to send with the request') + timeout: int = Field(10, description='HTTP timeout') async def setup(self): self.url = self.config.get("url", "") diff --git a/bbot/modules/output/json.py b/bbot/modules/output/json.py index b93d1e4e3f..c0e69f36bb 100644 --- a/bbot/modules/output/json.py +++ b/bbot/modules/output/json.py @@ -2,6 +2,8 @@ from contextlib import suppress from bbot.modules.output.base import BaseOutputModule +from pydantic import Field +from bbot.core.config.models import BaseModuleConfig class JSON(BaseOutputModule): @@ -11,10 +13,8 @@ class JSON(BaseOutputModule): "created_date": "2022-04-07", "author": "@TheTechromancer", } - options = {"output_file": ""} - options_desc = { - "output_file": "Output to file", - } + class Config(BaseModuleConfig): + output_file: str = Field('', description='Output to file') _preserve_graph = True async def setup(self): diff --git a/bbot/modules/output/kafka.py b/bbot/modules/output/kafka.py index 01eeeb2fd6..52dd95b11c 100644 --- a/bbot/modules/output/kafka.py +++ b/bbot/modules/output/kafka.py @@ -2,6 +2,8 @@ from aiokafka import AIOKafkaProducer from bbot.modules.output.base import BaseOutputModule +from pydantic import Field +from bbot.core.config.models import BaseModuleConfig class Kafka(BaseOutputModule): @@ -11,14 +13,9 @@ class Kafka(BaseOutputModule): "created_date": "2024-11-22", "author": "@TheTechromancer", } - options = { - "bootstrap_servers": "localhost:9092", - "topic": "bbot_events", - } - options_desc = { - "bootstrap_servers": "A comma-separated list of Kafka server addresses", - "topic": "The Kafka topic to publish events to", - } + class Config(BaseModuleConfig): + bootstrap_servers: str = Field('localhost:9092', description='A comma-separated list of Kafka server addresses') + topic: str = Field('bbot_events', description='The Kafka topic to publish events to') deps_pip = ["aiokafka~=0.12.0"] async def setup(self): diff --git a/bbot/modules/output/mongo.py b/bbot/modules/output/mongo.py index 833eab2304..f9c2ad185a 100644 --- a/bbot/modules/output/mongo.py +++ b/bbot/modules/output/mongo.py @@ -4,6 +4,8 @@ from bbot.models.pydantic import Event, Scan, Target from bbot.modules.output.base import BaseOutputModule +from pydantic import Field +from bbot.core.config.models import BaseModuleConfig class Mongo(BaseOutputModule): @@ -17,20 +19,12 @@ class Mongo(BaseOutputModule): "created_date": "2024-11-17", "author": "@TheTechromancer", } - options = { - "uri": "mongodb://localhost:27017", - "database": "bbot", - "username": "", - "password": "", - "collection_prefix": "", - } - options_desc = { - "uri": "The URI of the MongoDB server", - "database": "The name of the database to use", - "username": "The username to use to connect to the database", - "password": "The password to use to connect to the database", - "collection_prefix": "Prefix the name of each collection with this string", - } + class Config(BaseModuleConfig): + uri: str = Field('mongodb://localhost:27017', description='The URI of the MongoDB server') + database: str = Field('bbot', description='The name of the database to use') + username: str = Field('', description='The username to use to connect to the database') + password: str = Field('', description='The password to use to connect to the database') + collection_prefix: str = Field('', description='Prefix the name of each collection with this string') deps_pip = ["pymongo~=4.15"] async def setup(self): diff --git a/bbot/modules/output/mysql.py b/bbot/modules/output/mysql.py index 8d9a1f7f4c..f66fc68558 100644 --- a/bbot/modules/output/mysql.py +++ b/bbot/modules/output/mysql.py @@ -1,4 +1,6 @@ from bbot.modules.templates.sql import SQLTemplate +from pydantic import Field +from bbot.core.config.models import BaseModuleConfig class MySQL(SQLTemplate): @@ -8,20 +10,12 @@ class MySQL(SQLTemplate): "created_date": "2024-11-13", "author": "@TheTechromancer", } - options = { - "username": "root", - "password": "bbotislife", - "host": "localhost", - "port": 3306, - "database": "bbot", - } - options_desc = { - "username": "The username to connect to MySQL", - "password": "The password to connect to MySQL", - "host": "The server running MySQL", - "port": "The port to connect to MySQL", - "database": "The database name to connect to", - } + class Config(BaseModuleConfig): + username: str = Field('root', description='The username to connect to MySQL') + password: str = Field('bbotislife', description='The password to connect to MySQL') + host: str = Field('localhost', description='The server running MySQL') + port: int = Field(3306, description='The port to connect to MySQL') + database: str = Field('bbot', description='The database name to connect to') deps_pip = ["sqlmodel", "aiomysql"] protocol = "mysql+aiomysql" diff --git a/bbot/modules/output/nats.py b/bbot/modules/output/nats.py index 569645cc3b..75c3fdd741 100644 --- a/bbot/modules/output/nats.py +++ b/bbot/modules/output/nats.py @@ -1,6 +1,8 @@ import json import nats from bbot.modules.output.base import BaseOutputModule +from pydantic import Field +from bbot.core.config.models import BaseModuleConfig class NATS(BaseOutputModule): @@ -10,14 +12,9 @@ class NATS(BaseOutputModule): "created_date": "2024-11-22", "author": "@TheTechromancer", } - options = { - "servers": [], - "subject": "bbot_events", - } - options_desc = { - "servers": "A list of NATS server addresses", - "subject": "The NATS subject to publish events to", - } + class Config(BaseModuleConfig): + servers: list = Field([], description='A list of NATS server addresses') + subject: str = Field('bbot_events', description='The NATS subject to publish events to') deps_pip = ["nats-py"] async def setup(self): diff --git a/bbot/modules/output/neo4j.py b/bbot/modules/output/neo4j.py index 8d88572575..0641269799 100644 --- a/bbot/modules/output/neo4j.py +++ b/bbot/modules/output/neo4j.py @@ -4,6 +4,8 @@ from neo4j import AsyncGraphDatabase from bbot.modules.output.base import BaseOutputModule +from pydantic import Field +from bbot.core.config.models import BaseModuleConfig # silence annoying neo4j logger @@ -33,12 +35,10 @@ class neo4j(BaseOutputModule): watched_events = ["*"] meta = {"description": "Output to Neo4j", "created_date": "2022-04-07", "author": "@TheTechromancer"} - options = {"uri": "bolt://localhost:7687", "username": "neo4j", "password": "bbotislife"} - options_desc = { - "uri": "Neo4j server + port", - "username": "Neo4j username", - "password": "Neo4j password", - } + class Config(BaseModuleConfig): + uri: str = Field('bolt://localhost:7687', description='Neo4j server + port') + username: str = Field('neo4j', description='Neo4j username') + password: str = Field('bbotislife', description='Neo4j password') deps_pip = ["neo4j"] _batch_size = 500 _preserve_graph = True diff --git a/bbot/modules/output/postgres.py b/bbot/modules/output/postgres.py index 45beb7c7bc..61694b2f4b 100644 --- a/bbot/modules/output/postgres.py +++ b/bbot/modules/output/postgres.py @@ -1,4 +1,6 @@ from bbot.modules.templates.sql import SQLTemplate +from pydantic import Field +from bbot.core.config.models import BaseModuleConfig class Postgres(SQLTemplate): @@ -8,20 +10,12 @@ class Postgres(SQLTemplate): "created_date": "2024-11-08", "author": "@TheTechromancer", } - options = { - "username": "postgres", - "password": "bbotislife", - "host": "localhost", - "port": 5432, - "database": "bbot", - } - options_desc = { - "username": "The username to connect to Postgres", - "password": "The password to connect to Postgres", - "host": "The server running Postgres", - "port": "The port to connect to Postgres", - "database": "The database name to connect to", - } + class Config(BaseModuleConfig): + username: str = Field('postgres', description='The username to connect to Postgres') + password: str = Field('bbotislife', description='The password to connect to Postgres') + host: str = Field('localhost', description='The server running Postgres') + port: int = Field(5432, description='The port to connect to Postgres') + database: str = Field('bbot', description='The database name to connect to') deps_pip = ["sqlmodel", "asyncpg"] protocol = "postgresql+asyncpg" diff --git a/bbot/modules/output/rabbitmq.py b/bbot/modules/output/rabbitmq.py index ba4205940d..8e28e4c649 100644 --- a/bbot/modules/output/rabbitmq.py +++ b/bbot/modules/output/rabbitmq.py @@ -2,6 +2,8 @@ import aio_pika from bbot.modules.output.base import BaseOutputModule +from pydantic import Field +from bbot.core.config.models import BaseModuleConfig class RabbitMQ(BaseOutputModule): @@ -11,14 +13,9 @@ class RabbitMQ(BaseOutputModule): "created_date": "2024-11-22", "author": "@TheTechromancer", } - options = { - "url": "amqp://guest:guest@localhost/", - "queue": "bbot_events", - } - options_desc = { - "url": "The RabbitMQ connection URL", - "queue": "The RabbitMQ queue to publish events to", - } + class Config(BaseModuleConfig): + url: str = Field('amqp://guest:guest@localhost/', description='The RabbitMQ connection URL') + queue: str = Field('bbot_events', description='The RabbitMQ queue to publish events to') deps_pip = ["aio_pika~=9.5.0"] async def setup(self): diff --git a/bbot/modules/output/slack.py b/bbot/modules/output/slack.py index 299d7334cf..056bed4389 100644 --- a/bbot/modules/output/slack.py +++ b/bbot/modules/output/slack.py @@ -1,6 +1,8 @@ import yaml from bbot.modules.templates.webhook import WebhookOutputModule +from pydantic import Field +from bbot.core.config.models import BaseModuleConfig class Slack(WebhookOutputModule): @@ -10,13 +12,11 @@ class Slack(WebhookOutputModule): "created_date": "2023-08-14", "author": "@TheTechromancer", } - options = {"webhook_url": "", "event_types": ["FINDING"], "min_severity": "LOW", "retries": 10} - options_desc = { - "webhook_url": "Discord webhook URL", - "event_types": "Types of events to send", - "min_severity": "Only allow FINDING events of this severity or higher", - "retries": "Number of times to retry sending the message before skipping the event", - } + class Config(BaseModuleConfig): + webhook_url: str = Field('', description='Discord webhook URL') + event_types: list[str] = Field(['FINDING'], description='Types of events to send') + min_severity: str = Field('LOW', description='Only allow FINDING events of this severity or higher') + retries: int = Field(10, description='Number of times to retry sending the message before skipping the event') content_key = "text" def format_message_str(self, event): diff --git a/bbot/modules/output/splunk.py b/bbot/modules/output/splunk.py index 0c0a0dd804..97d33a6bc1 100644 --- a/bbot/modules/output/splunk.py +++ b/bbot/modules/output/splunk.py @@ -1,5 +1,7 @@ from bbot.errors import WebError from bbot.modules.output.base import BaseOutputModule +from pydantic import Field +from bbot.core.config.models import BaseModuleConfig class Splunk(BaseOutputModule): @@ -9,20 +11,12 @@ class Splunk(BaseOutputModule): "created_date": "2024-02-17", "author": "@w0Tx", } - options = { - "url": "", - "hectoken": "", - "index": "", - "source": "", - "timeout": 10, - } - options_desc = { - "url": "Web URL", - "hectoken": "HEC Token", - "index": "Index to send data to", - "source": "Source path to be added to the metadata", - "timeout": "HTTP timeout", - } + class Config(BaseModuleConfig): + url: str = Field('', description='Web URL') + hectoken: str = Field('', description='HEC Token') + index: str = Field('', description='Index to send data to') + source: str = Field('', description='Source path to be added to the metadata') + timeout: int = Field(10, description='HTTP timeout') async def setup(self): self.url = self.config.get("url", "") diff --git a/bbot/modules/output/sqlite.py b/bbot/modules/output/sqlite.py index 261b13b6e2..b6a0e11d37 100644 --- a/bbot/modules/output/sqlite.py +++ b/bbot/modules/output/sqlite.py @@ -1,6 +1,8 @@ from pathlib import Path from bbot.modules.templates.sql import SQLTemplate +from pydantic import Field +from bbot.core.config.models import BaseModuleConfig class SQLite(SQLTemplate): @@ -10,12 +12,8 @@ class SQLite(SQLTemplate): "created_date": "2024-11-07", "author": "@TheTechromancer", } - options = { - "database": "", - } - options_desc = { - "database": "The path to the sqlite database file", - } + class Config(BaseModuleConfig): + database: str = Field('', description='The path to the sqlite database file') deps_pip = ["sqlmodel", "aiosqlite"] async def setup(self): diff --git a/bbot/modules/output/stdout.py b/bbot/modules/output/stdout.py index 974b7ede16..73d59370dc 100644 --- a/bbot/modules/output/stdout.py +++ b/bbot/modules/output/stdout.py @@ -2,19 +2,19 @@ from bbot.logger import log_to_stderr from bbot.modules.output.base import BaseOutputModule +from pydantic import Field +from bbot.core.config.models import BaseModuleConfig class Stdout(BaseOutputModule): watched_events = ["*"] meta = {"description": "Output to text", "created_date": "2024-04-03", "author": "@TheTechromancer"} - options = {"format": "text", "event_types": [], "event_fields": [], "in_scope_only": False, "accept_dupes": True} - options_desc = { - "format": "Which text format to display, choices: text,json", - "event_types": "Which events to display, default all event types", - "event_fields": "Which event fields to display", - "in_scope_only": "Whether to only show in-scope events", - "accept_dupes": "Whether to show duplicate events, default True", - } + class Config(BaseModuleConfig): + format: str = Field('text', description='Which text format to display, choices: text,json') + event_types: list = Field([], description='Which events to display, default all event types') + event_fields: list = Field([], description='Which event fields to display') + in_scope_only: bool = Field(False, description='Whether to only show in-scope events') + accept_dupes: bool = Field(True, description='Whether to show duplicate events, default True') vuln_severity_map = { "INFO": "HUGEINFO", "LOW": "HUGEWARNING", diff --git a/bbot/modules/output/subdomains.py b/bbot/modules/output/subdomains.py index 65d082355e..0e58d3e842 100644 --- a/bbot/modules/output/subdomains.py +++ b/bbot/modules/output/subdomains.py @@ -1,5 +1,7 @@ from bbot.modules.output.txt import TXT from bbot.modules.base import BaseModule +from pydantic import Field +from bbot.core.config.models import BaseModuleConfig class Subdomains(TXT): @@ -10,8 +12,9 @@ class Subdomains(TXT): "created_date": "2023-07-31", "author": "@TheTechromancer", } - options = {"output_file": "", "include_unresolved": False} - options_desc = {"output_file": "Output to file", "include_unresolved": "Include unresolved subdomains in output"} + class Config(BaseModuleConfig): + output_file: str = Field('', description='Output to file') + include_unresolved: bool = Field(False, description='Include unresolved subdomains in output') accept_dupes = False in_scope_only = True diff --git a/bbot/modules/output/teams.py b/bbot/modules/output/teams.py index 2ab461d5a6..c4fd7a3bac 100644 --- a/bbot/modules/output/teams.py +++ b/bbot/modules/output/teams.py @@ -1,4 +1,6 @@ from bbot.modules.templates.webhook import WebhookOutputModule +from pydantic import Field +from bbot.core.config.models import BaseModuleConfig class Teams(WebhookOutputModule): @@ -8,13 +10,11 @@ class Teams(WebhookOutputModule): "created_date": "2023-08-14", "author": "@TheTechromancer", } - options = {"webhook_url": "", "event_types": ["FINDING"], "min_severity": "LOW", "retries": 10} - options_desc = { - "webhook_url": "Teams webhook URL", - "event_types": "Types of events to send", - "min_severity": "Only allow FINDING events of this severity or higher", - "retries": "Number of times to retry sending the message before skipping the event", - } + class Config(BaseModuleConfig): + webhook_url: str = Field('', description='Teams webhook URL') + event_types: list[str] = Field(['FINDING'], description='Types of events to send') + min_severity: str = Field('LOW', description='Only allow FINDING events of this severity or higher') + retries: int = Field(10, description='Number of times to retry sending the message before skipping the event') async def handle_event(self, event): data = self.format_message(event) diff --git a/bbot/modules/output/txt.py b/bbot/modules/output/txt.py index 2dfb14c106..42fd12d1fe 100644 --- a/bbot/modules/output/txt.py +++ b/bbot/modules/output/txt.py @@ -1,13 +1,15 @@ from contextlib import suppress from bbot.modules.output.base import BaseOutputModule +from pydantic import Field +from bbot.core.config.models import BaseModuleConfig class TXT(BaseOutputModule): watched_events = ["*"] meta = {"description": "Output to text", "created_date": "2024-04-03", "author": "@TheTechromancer"} - options = {"output_file": ""} - options_desc = {"output_file": "Output to file"} + class Config(BaseModuleConfig): + output_file: str = Field('', description='Output to file') output_filename = "output.txt" diff --git a/bbot/modules/output/web_parameters.py b/bbot/modules/output/web_parameters.py index 634a623720..1edf5a1b7a 100644 --- a/bbot/modules/output/web_parameters.py +++ b/bbot/modules/output/web_parameters.py @@ -2,6 +2,8 @@ from collections import defaultdict from bbot.modules.output.base import BaseOutputModule +from pydantic import Field +from bbot.core.config.models import BaseModuleConfig class Web_parameters(BaseOutputModule): @@ -11,11 +13,9 @@ class Web_parameters(BaseOutputModule): "created_date": "2025-01-25", "author": "@liquidsec", } - options = {"output_file": "", "include_count": False} - options_desc = { - "output_file": "Output to file", - "include_count": "Include the count of each parameter in the output", - } + class Config(BaseModuleConfig): + output_file: str = Field('', description='Output to file') + include_count: bool = Field(False, description='Include the count of each parameter in the output') output_filename = "web_parameters.txt" diff --git a/bbot/modules/output/web_report.py b/bbot/modules/output/web_report.py index 4cd2412046..81f984540e 100644 --- a/bbot/modules/output/web_report.py +++ b/bbot/modules/output/web_report.py @@ -1,6 +1,8 @@ from bbot.modules.output.base import BaseOutputModule import markdown import html +from pydantic import Field +from bbot.core.config.models import BaseModuleConfig class web_report(BaseOutputModule): @@ -10,11 +12,9 @@ class web_report(BaseOutputModule): "created_date": "2023-02-08", "author": "@liquidsec", } - options = { - "output_file": "", - "css_theme_file": "https://cdnjs.cloudflare.com/ajax/libs/github-markdown-css/5.1.0/github-markdown.min.css", - } - options_desc = {"output_file": "Output to file", "css_theme_file": "CSS theme URL for HTML output"} + class Config(BaseModuleConfig): + output_file: str = Field('', description='Output to file') + css_theme_file: str = Field('https://cdnjs.cloudflare.com/ajax/libs/github-markdown-css/5.1.0/github-markdown.min.css', description='CSS theme URL for HTML output') deps_pip = ["markdown~=3.4.3"] async def setup(self): diff --git a/bbot/modules/output/websocket.py b/bbot/modules/output/websocket.py index 13ff355014..d5f3d92c8f 100644 --- a/bbot/modules/output/websocket.py +++ b/bbot/modules/output/websocket.py @@ -4,18 +4,18 @@ import websockets from bbot.modules.output.base import BaseOutputModule +from pydantic import Field +from bbot.core.config.models import BaseModuleConfig class Websocket(BaseOutputModule): watched_events = ["*"] meta = {"description": "Output to websockets", "created_date": "2022-04-15", "author": "@TheTechromancer"} - options = {"url": "", "token": "", "preserve_graph": True, "ignore_ssl": False} - options_desc = { - "url": "Web URL", - "token": "Authorization Bearer token", - "preserve_graph": "Preserve full chains of events in the graph (prevents orphans)", - "ignore_ssl": "Ignores all Websocket SSL related errors (like Self-Signed Certificates, etc.)", - } + class Config(BaseModuleConfig): + url: str = Field('', description='Web URL') + token: str = Field('', description='Authorization Bearer token') + preserve_graph: bool = Field(True, description='Preserve full chains of events in the graph (prevents orphans)') + ignore_ssl: bool = Field(False, description='Ignores all Websocket SSL related errors (like Self-Signed Certificates, etc.)') async def setup(self): self.url = self.config.get("url", "") diff --git a/bbot/modules/output/zeromq.py b/bbot/modules/output/zeromq.py index 938f234545..c9a7e4059f 100644 --- a/bbot/modules/output/zeromq.py +++ b/bbot/modules/output/zeromq.py @@ -2,6 +2,8 @@ import json from bbot.modules.output.base import BaseOutputModule +from pydantic import Field +from bbot.core.config.models import BaseModuleConfig class ZeroMQ(BaseOutputModule): @@ -11,12 +13,8 @@ class ZeroMQ(BaseOutputModule): "created_date": "2024-11-22", "author": "@TheTechromancer", } - options = { - "zmq_address": "", - } - options_desc = { - "zmq_address": "The ZeroMQ socket address to publish events to (e.g. tcp://localhost:5555)", - } + class Config(BaseModuleConfig): + zmq_address: str = Field('', description='The ZeroMQ socket address to publish events to (e.g. tcp://localhost:5555)') async def setup(self): self.zmq_address = self.config.get("zmq_address", "") diff --git a/bbot/modules/paramminer_cookies.py b/bbot/modules/paramminer_cookies.py index 871238d803..c85b8e92f5 100644 --- a/bbot/modules/paramminer_cookies.py +++ b/bbot/modules/paramminer_cookies.py @@ -1,4 +1,6 @@ from .paramminer_headers import paramminer_headers +from pydantic import Field +from bbot.core.config.models import BaseModuleConfig class paramminer_cookies(paramminer_headers): @@ -14,17 +16,14 @@ class paramminer_cookies(paramminer_headers): "created_date": "2022-06-27", "author": "@liquidsec", } - options = { - "wordlist": "", # default is defined within setup function - "recycle_words": False, - "skip_boring_words": True, - } - options_desc = { - "wordlist": "Define the wordlist to be used to derive headers", - "recycle_words": "Attempt to use words found during the scan on all other endpoints", - "skip_boring_words": "Remove commonly uninteresting words from the wordlist", - } - options_desc = {"wordlist": "Define the wordlist to be used to derive cookies"} + class Config(BaseModuleConfig): + wordlist: str = Field("", description="Define the wordlist to be used to derive cookies") + recycle_words: bool = Field( + False, description="Attempt to use words found during the scan on all other endpoints" + ) + skip_boring_words: bool = Field( + True, description="Remove commonly uninteresting words from the wordlist" + ) scanned_hosts = [] boring_words = set() _module_threads = 12 diff --git a/bbot/modules/paramminer_getparams.py b/bbot/modules/paramminer_getparams.py index 27a99f8ab4..98e9497f69 100644 --- a/bbot/modules/paramminer_getparams.py +++ b/bbot/modules/paramminer_getparams.py @@ -1,4 +1,6 @@ from .paramminer_headers import paramminer_headers +from pydantic import Field +from bbot.core.config.models import BaseModuleConfig class paramminer_getparams(paramminer_headers): @@ -15,16 +17,10 @@ class paramminer_getparams(paramminer_headers): "author": "@liquidsec", } scanned_hosts = [] - options = { - "wordlist": "", # default is defined within setup function - "recycle_words": False, - "skip_boring_words": True, - } - options_desc = { - "wordlist": "Define the wordlist to be used to derive headers", - "recycle_words": "Attempt to use words found during the scan on all other endpoints", - "skip_boring_words": "Remove commonly uninteresting words from the wordlist", - } + class Config(BaseModuleConfig): + wordlist: str = Field('', description='Define the wordlist to be used to derive headers') + recycle_words: bool = Field(False, description='Attempt to use words found during the scan on all other endpoints') + skip_boring_words: bool = Field(True, description='Remove commonly uninteresting words from the wordlist') boring_words = {"utm_source", "utm_campaign", "utm_medium", "utm_term", "utm_content"} in_scope_only = True compare_mode = "getparam" diff --git a/bbot/modules/paramminer_headers.py b/bbot/modules/paramminer_headers.py index ae573abadf..c99388ffae 100644 --- a/bbot/modules/paramminer_headers.py +++ b/bbot/modules/paramminer_headers.py @@ -2,6 +2,8 @@ from bbot.errors import HttpCompareError from bbot.modules.base import BaseModule +from pydantic import Field +from bbot.core.config.models import BaseModuleConfig class paramminer_headers(BaseModule): @@ -17,16 +19,10 @@ class paramminer_headers(BaseModule): "created_date": "2022-04-15", "author": "@liquidsec", } - options = { - "wordlist": "", # default is defined within setup function - "recycle_words": False, - "skip_boring_words": True, - } - options_desc = { - "wordlist": "Define the wordlist to be used to derive headers", - "recycle_words": "Attempt to use words found during the scan on all other endpoints", - "skip_boring_words": "Remove commonly uninteresting words from the wordlist", - } + class Config(BaseModuleConfig): + wordlist: str = Field('', description='Define the wordlist to be used to derive headers') + recycle_words: bool = Field(False, description='Attempt to use words found during the scan on all other endpoints') + skip_boring_words: bool = Field(True, description='Remove commonly uninteresting words from the wordlist') scanned_hosts = [] boring_words = { "accept", diff --git a/bbot/modules/passivetotal.py b/bbot/modules/passivetotal.py index 1010980945..d5d5f26611 100644 --- a/bbot/modules/passivetotal.py +++ b/bbot/modules/passivetotal.py @@ -1,4 +1,6 @@ from bbot.modules.templates.subdomain_enum import subdomain_enum_apikey +from pydantic import Field +from bbot.core.config.models import BaseModuleConfig class passivetotal(subdomain_enum_apikey): @@ -11,8 +13,8 @@ class passivetotal(subdomain_enum_apikey): "author": "@TheTechromancer", "auth_required": True, } - options = {"api_key": ""} - options_desc = {"api_key": "PassiveTotal API Key in the format of 'username:api_key'"} + class Config(BaseModuleConfig): + api_key: str = Field('', description="PassiveTotal API Key in the format of 'username:api_key'") base_url = "https://api.passivetotal.org/v2" diff --git a/bbot/modules/pgp.py b/bbot/modules/pgp.py index b12372dd1e..8e53c4e48c 100644 --- a/bbot/modules/pgp.py +++ b/bbot/modules/pgp.py @@ -1,4 +1,6 @@ from bbot.modules.templates.subdomain_enum import subdomain_enum +from pydantic import Field +from bbot.core.config.models import BaseModuleConfig class pgp(subdomain_enum): @@ -10,15 +12,8 @@ class pgp(subdomain_enum): "created_date": "2022-08-10", "author": "@TheTechromancer", } - options = { - "search_urls": [ - "https://keyserver.ubuntu.com/pks/lookup?fingerprint=on&op=vindex&search=", - "http://the.earth.li:11371/pks/lookup?fingerprint=on&op=vindex&search=", - "https://pgpkeys.eu/pks/lookup?search=&op=index", - "https://pgp.mit.edu/pks/lookup?search=&op=index", - ] - } - options_desc = {"search_urls": "PGP key servers to search"} + class Config(BaseModuleConfig): + search_urls: list[str] = Field(['https://keyserver.ubuntu.com/pks/lookup?fingerprint=on&op=vindex&search=', 'http://the.earth.li:11371/pks/lookup?fingerprint=on&op=vindex&search=', 'https://pgpkeys.eu/pks/lookup?search=&op=index', 'https://pgp.mit.edu/pks/lookup?search=&op=index'], description='PGP key servers to search') async def handle_event(self, event): query = self.make_query(event) diff --git a/bbot/modules/portfilter.py b/bbot/modules/portfilter.py index 21b0313194..9d006b2af5 100644 --- a/bbot/modules/portfilter.py +++ b/bbot/modules/portfilter.py @@ -1,4 +1,6 @@ from bbot.modules.base import BaseInterceptModule +from pydantic import Field +from bbot.core.config.models import BaseModuleConfig class portfilter(BaseInterceptModule): @@ -9,14 +11,9 @@ class portfilter(BaseInterceptModule): "created_date": "2025-01-06", "author": "@TheTechromancer", } - options = { - "cdn_tags": "cdn,waf", - "allowed_cdn_ports": "80,443", - } - options_desc = { - "cdn_tags": "Comma-separated list of tags to skip, e.g. 'cdn,waf'", - "allowed_cdn_ports": "Comma-separated list of ports that are allowed to be scanned for CDNs", - } + class Config(BaseModuleConfig): + cdn_tags: str = Field('cdn,waf', description="Comma-separated list of tags to skip, e.g. 'cdn,waf'") + allowed_cdn_ports: str = Field('80,443', description='Comma-separated list of ports that are allowed to be scanned for CDNs') _priority = 4 # we consume URLs but we don't want to automatically enable httpx diff --git a/bbot/modules/portscan.py b/bbot/modules/portscan.py index c4897d507b..1d991e403f 100644 --- a/bbot/modules/portscan.py +++ b/bbot/modules/portscan.py @@ -4,6 +4,8 @@ from radixtarget import RadixTarget, host_size_key from bbot.modules.base import BaseModule +from pydantic import Field +from bbot.core.config.models import BaseModuleConfig # TODO: this module is getting big. It should probably be two modules: one for ping and one for SYN. @@ -18,33 +20,18 @@ class portscan(BaseModule): "created_date": "2024-05-15", "author": "@TheTechromancer", } - options = { - "top_ports": 100, - "ports": "", - # ping scan at 600 packets/s ~= private IP space in 8 hours - "rate": 300, - "wait": 5, - "ping_first": False, - "ping_only": False, - "adapter": "", - "adapter_ip": "", - "adapter_mac": "", - "router_mac": "", - "module_timeout": 259200, # 3 days - } - options_desc = { - "top_ports": "Top ports to scan (default 100) (to override, specify 'ports')", - "ports": "Ports to scan", - "rate": "Rate in packets per second", - "wait": "Seconds to wait for replies after scan is complete", - "ping_first": "Only portscan hosts that reply to pings", - "ping_only": "Ping sweep only, no portscan", - "adapter": 'Manually specify a network interface, such as "eth0" or "tun0". If not specified, the first network interface found with a default gateway will be used.', - "adapter_ip": "Send packets using this IP address. Not needed unless masscan's autodetection fails", - "adapter_mac": "Send packets using this as the source MAC address. Not needed unless masscan's autodetection fails", - "router_mac": "Send packets to this MAC address as the destination. Not needed unless masscan's autodetection fails", - "module_timeout": "Max time in seconds to spend handling each batch of events", - } + class Config(BaseModuleConfig): + top_ports: int = Field(100, description="Top ports to scan (default 100) (to override, specify 'ports')") + ports: str = Field('', description='Ports to scan') + rate: int = Field(300, description='Rate in packets per second') + wait: int = Field(5, description='Seconds to wait for replies after scan is complete') + ping_first: bool = Field(False, description='Only portscan hosts that reply to pings') + ping_only: bool = Field(False, description='Ping sweep only, no portscan') + adapter: str = Field('', description='Manually specify a network interface, such as "eth0" or "tun0". If not specified, the first network interface found with a default gateway will be used.') + adapter_ip: str = Field('', description="Send packets using this IP address. Not needed unless masscan's autodetection fails") + adapter_mac: str = Field('', description="Send packets using this as the source MAC address. Not needed unless masscan's autodetection fails") + router_mac: str = Field('', description="Send packets to this MAC address as the destination. Not needed unless masscan's autodetection fails") + module_timeout: int = Field(259200, description='Max time in seconds to spend handling each batch of events') deps_common = ["masscan"] batch_size = 1000000 _shuffle_incoming_queue = False diff --git a/bbot/modules/postman.py b/bbot/modules/postman.py index ee0bf25cd2..076e763c1f 100644 --- a/bbot/modules/postman.py +++ b/bbot/modules/postman.py @@ -1,4 +1,6 @@ from bbot.modules.templates.postman import postman +from pydantic import Field +from bbot.core.config.models import BaseModuleConfig class postman(postman): @@ -10,8 +12,8 @@ class postman(postman): "created_date": "2024-09-07", "author": "@domwhewell-sage", } - options = {"api_key": ""} - options_desc = {"api_key": "Postman API Key"} + class Config(BaseModuleConfig): + api_key: str = Field('', description='Postman API Key') reject_wildcards = False async def handle_event(self, event): diff --git a/bbot/modules/postman_download.py b/bbot/modules/postman_download.py index 77644435ad..34fdf8c6bd 100644 --- a/bbot/modules/postman_download.py +++ b/bbot/modules/postman_download.py @@ -2,6 +2,8 @@ import json from pathlib import Path from bbot.modules.templates.postman import postman +from pydantic import Field +from bbot.core.config.models import BaseModuleConfig class postman_download(postman): @@ -13,11 +15,9 @@ class postman_download(postman): "created_date": "2024-09-07", "author": "@domwhewell-sage", } - options = {"output_folder": "", "api_key": ""} - options_desc = { - "output_folder": "Folder to download postman workspaces to. If not specified, downloaded workspaces will be deleted when the scan completes, to minimize disk usage.", - "api_key": "Postman API Key", - } + class Config(BaseModuleConfig): + output_folder: str = Field('', description='Folder to download postman workspaces to. If not specified, downloaded workspaces will be deleted when the scan completes, to minimize disk usage.') + api_key: str = Field('', description='Postman API Key') scope_distance_modifier = 2 async def setup(self): diff --git a/bbot/modules/retirejs.py b/bbot/modules/retirejs.py index 78fc72dfed..d07dbeb7b0 100644 --- a/bbot/modules/retirejs.py +++ b/bbot/modules/retirejs.py @@ -1,6 +1,8 @@ import json from enum import IntEnum from bbot.modules.base import BaseModule +from pydantic import Field +from bbot.core.config.models import BaseModuleConfig class RetireJSSeverity(IntEnum): @@ -27,16 +29,10 @@ class retirejs(BaseModule): "created_date": "2025-08-19", "author": "@liquidsec", } - options = { - "version": "5.3.0", - "node_version": "18.19.1", - "severity": "medium", - } - options_desc = { - "version": "retire.js version", - "node_version": "Node.js version to install locally", - "severity": "Minimum severity level to report (none, low, medium, high, critical)", - } + class Config(BaseModuleConfig): + version: str = Field('5.3.0', description='retire.js version') + node_version: str = Field('18.19.1', description='Node.js version to install locally') + severity: str = Field('medium', description='Minimum severity level to report (none, low, medium, high, critical)') deps_ansible = [ # Download Node.js binary (Linux x64) diff --git a/bbot/modules/robots.py b/bbot/modules/robots.py index 1240f11959..df83d6cbb8 100644 --- a/bbot/modules/robots.py +++ b/bbot/modules/robots.py @@ -1,4 +1,6 @@ from bbot.modules.base import BaseModule +from pydantic import Field +from bbot.core.config.models import BaseModuleConfig class robots(BaseModule): @@ -7,12 +9,10 @@ class robots(BaseModule): flags = ["safe", "active", "web"] meta = {"description": "Look for and parse robots.txt", "created_date": "2023-02-01", "author": "@liquidsec"} - options = {"include_sitemap": False, "include_allow": True, "include_disallow": True} - options_desc = { - "include_sitemap": "Include 'sitemap' entries", - "include_allow": "Include 'Allow' Entries", - "include_disallow": "Include 'Disallow' Entries", - } + class Config(BaseModuleConfig): + include_sitemap: bool = Field(False, description="Include 'sitemap' entries") + include_allow: bool = Field(True, description="Include 'Allow' Entries") + include_disallow: bool = Field(True, description="Include 'Disallow' Entries") in_scope_only = True per_hostport_only = True diff --git a/bbot/modules/securitytrails.py b/bbot/modules/securitytrails.py index 7ab333ee76..c686f4c120 100644 --- a/bbot/modules/securitytrails.py +++ b/bbot/modules/securitytrails.py @@ -1,4 +1,6 @@ from bbot.modules.templates.subdomain_enum import subdomain_enum_apikey +from pydantic import Field +from bbot.core.config.models import BaseModuleConfig class securitytrails(subdomain_enum_apikey): @@ -11,8 +13,8 @@ class securitytrails(subdomain_enum_apikey): "author": "@TheTechromancer", "auth_required": True, } - options = {"api_key": ""} - options_desc = {"api_key": "SecurityTrails API key"} + class Config(BaseModuleConfig): + api_key: str = Field('', description='SecurityTrails API key') base_url = "https://api.securitytrails.com/v1" ping_url = f"{base_url}/ping?apikey={{api_key}}" diff --git a/bbot/modules/securitytxt.py b/bbot/modules/securitytxt.py index d8d3d4aea2..fb84e5d1b2 100644 --- a/bbot/modules/securitytxt.py +++ b/bbot/modules/securitytxt.py @@ -50,6 +50,8 @@ import re from bbot.core.helpers.regexes import email_regex, url_regexes +from pydantic import Field +from bbot.core.config.models import BaseModuleConfig _securitytxt_regex = r"^(?P\w+): *(?P.*)$" securitytxt_regex = re.compile(_securitytxt_regex, re.I | re.M) @@ -64,14 +66,9 @@ class securitytxt(BaseModule): "author": "@colin-stubbs", "created_date": "2024-05-26", } - options = { - "emails": True, - "urls": True, - } - options_desc = { - "emails": "emit EMAIL_ADDRESS events", - "urls": "emit URL_UNVERIFIED events", - } + class Config(BaseModuleConfig): + emails: bool = Field(True, description='emit EMAIL_ADDRESS events') + urls: bool = Field(True, description='emit URL_UNVERIFIED events') async def setup(self): self._emails = self.config.get("emails", True) diff --git a/bbot/modules/shodan_dns.py b/bbot/modules/shodan_dns.py index 4e68ed11d4..f3654fabf1 100644 --- a/bbot/modules/shodan_dns.py +++ b/bbot/modules/shodan_dns.py @@ -1,4 +1,6 @@ from bbot.modules.templates.shodan import shodan +from pydantic import Field +from bbot.core.config.models import BaseModuleConfig class shodan_dns(shodan): @@ -11,8 +13,8 @@ class shodan_dns(shodan): "author": "@TheTechromancer", "auth_required": True, } - options = {"api_key": ""} - options_desc = {"api_key": "Shodan API key"} + class Config(BaseModuleConfig): + api_key: str = Field('', description='Shodan API key') base_url = "https://api.shodan.io" diff --git a/bbot/modules/shodan_enterprise.py b/bbot/modules/shodan_enterprise.py index 273fe837da..3cd732e0a4 100644 --- a/bbot/modules/shodan_enterprise.py +++ b/bbot/modules/shodan_enterprise.py @@ -1,4 +1,6 @@ from bbot.modules.base import BaseModule +from pydantic import Field +from bbot.core.config.models import BaseModuleConfig class shodan_enterprise(BaseModule): @@ -11,11 +13,9 @@ class shodan_enterprise(BaseModule): "description": "Shodan Enterprise API integration module.", "auth_required": True, } - options = {"api_key": "", "in_scope_only": True} - options_desc = { - "api_key": "Shodan API Key", - "in_scope_only": "Only query in-scope IPs. If False, will query up to distance 1.", - } + class Config(BaseModuleConfig): + api_key: str = Field('', description='Shodan API Key') + in_scope_only: bool = Field(True, description='Only query in-scope IPs. If False, will query up to distance 1.') in_scope_only = True base_url = "https://api.shodan.io" diff --git a/bbot/modules/shodan_idb.py b/bbot/modules/shodan_idb.py index 59ad329d35..31d8fbda36 100644 --- a/bbot/modules/shodan_idb.py +++ b/bbot/modules/shodan_idb.py @@ -1,5 +1,8 @@ from bbot.modules.base import BaseModule import time +from pydantic import Field +from bbot.core.config.models import BaseModuleConfig +from typing import Optional class shodan_idb(BaseModule): @@ -47,10 +50,8 @@ class shodan_idb(BaseModule): "created_date": "2023-12-22", "author": "@TheTechromancer", } - options = {"retries": None} - options_desc = { - "retries": "How many times to retry API requests (e.g. after a 429 error). Overrides the global web.api_retries setting." - } + class Config(BaseModuleConfig): + retries: Optional[str] = Field(None, description='How many times to retry API requests (e.g. after a 429 error). Overrides the global web.api_retries setting.') # we typically don't want to abort this module _api_failure_abort_threshold = 9999999999 diff --git a/bbot/modules/sslcert.py b/bbot/modules/sslcert.py index ff6f6cf402..ffc0890f2e 100644 --- a/bbot/modules/sslcert.py +++ b/bbot/modules/sslcert.py @@ -6,6 +6,8 @@ from bbot.modules.base import BaseModule from bbot.core.helpers.async_helpers import NamedLock from bbot.core.helpers.web.ssl_context import ssl_context_noverify +from pydantic import Field +from bbot.core.config.models import BaseModuleConfig class sslcert(BaseModule): @@ -17,8 +19,9 @@ class sslcert(BaseModule): "created_date": "2022-03-30", "author": "@TheTechromancer", } - options = {"timeout": 5.0, "skip_non_ssl": True} - options_desc = {"timeout": "Socket connect timeout in seconds", "skip_non_ssl": "Don't try common non-SSL ports"} + class Config(BaseModuleConfig): + timeout: float = Field(5.0, description='Socket connect timeout in seconds') + skip_non_ssl: bool = Field(True, description="Don't try common non-SSL ports") deps_apt = ["openssl"] deps_pip = ["pyOpenSSL~=25.3.0"] _module_threads = 25 diff --git a/bbot/modules/subdomainradar.py b/bbot/modules/subdomainradar.py index eaca2374c5..5d98a7a7fd 100644 --- a/bbot/modules/subdomainradar.py +++ b/bbot/modules/subdomainradar.py @@ -2,6 +2,8 @@ import asyncio from bbot.modules.templates.subdomain_enum import subdomain_enum_apikey +from pydantic import Field +from bbot.core.config.models import BaseModuleConfig class SubdomainRadar(subdomain_enum_apikey): @@ -14,12 +16,10 @@ class SubdomainRadar(subdomain_enum_apikey): "author": "@TheTechromancer", "auth_required": True, } - options = {"api_key": "", "group": "fast", "timeout": 120} - options_desc = { - "api_key": "SubDomainRadar.io API key", - "group": "The enumeration group to use. Choose from fast, medium, deep", - "timeout": "Timeout in seconds", - } + class Config(BaseModuleConfig): + api_key: str = Field('', description='SubDomainRadar.io API key') + group: str = Field('fast', description='The enumeration group to use. Choose from fast, medium, deep') + timeout: int = Field(120, description='Timeout in seconds') base_url = "https://api.subdomainradar.io" ping_url = f"{base_url}/profile" diff --git a/bbot/modules/telerik.py b/bbot/modules/telerik.py index 493117fc7d..70c168bea2 100644 --- a/bbot/modules/telerik.py +++ b/bbot/modules/telerik.py @@ -1,6 +1,8 @@ from sys import executable from bbot.modules.base import BaseModule +from pydantic import Field +from bbot.core.config.models import BaseModuleConfig class telerik(BaseModule): @@ -153,11 +155,9 @@ class telerik(BaseModule): RAUConfirmed = [] - options = {"exploit_RAU_crypto": False, "include_subdirs": False} - options_desc = { - "exploit_RAU_crypto": "Attempt to confirm any RAU AXD detections are vulnerable", - "include_subdirs": "Include subdirectories in the scan (off by default)", # will create many finding events if used in conjunction with web spider or ffuf - } + class Config(BaseModuleConfig): + exploit_RAU_crypto: bool = Field(False, description='Attempt to confirm any RAU AXD detections are vulnerable') + include_subdirs: bool = Field(False, description='Include subdirectories in the scan (off by default)') in_scope_only = True diff --git a/bbot/modules/trajan.py b/bbot/modules/trajan.py index dec96d94fa..a8400b48d7 100644 --- a/bbot/modules/trajan.py +++ b/bbot/modules/trajan.py @@ -2,6 +2,8 @@ from urllib.parse import urlparse from bbot.modules.base import BaseModule +from pydantic import Field +from bbot.core.config.models import BaseModuleConfig class trajan(BaseModule): @@ -15,26 +17,15 @@ class trajan(BaseModule): } # Configuration options - options = { - "version": "1.0.0", - "github_token": "", - "gitlab_token": "", - "ado_token": "", - "jfrog_token": "", - "jenkins_username": "", - "jenkins_password": "", - "jenkins_token": "", - } - options_desc = { - "version": "Trajan version to download and use", - "github_token": "GitHub API token for rate-limiting and private repo access", - "gitlab_token": "GitLab API token for private repo access", - "ado_token": "Azure DevOps Personal Access Token (PAT)", - "jfrog_token": "JFrog API token", - "jenkins_username": "Jenkins username for basic auth", - "jenkins_password": "Jenkins password for basic auth", - "jenkins_token": "Jenkins API token", - } + class Config(BaseModuleConfig): + version: str = Field('1.0.0', description='Trajan version to download and use') + github_token: str = Field('', description='GitHub API token for rate-limiting and private repo access') + gitlab_token: str = Field('', description='GitLab API token for private repo access') + ado_token: str = Field('', description='Azure DevOps Personal Access Token (PAT)') + jfrog_token: str = Field('', description='JFrog API token') + jenkins_username: str = Field('', description='Jenkins username for basic auth') + jenkins_password: str = Field('', description='Jenkins password for basic auth') + jenkins_token: str = Field('', description='Jenkins API token') deps_ansible = [ { diff --git a/bbot/modules/trickest.py b/bbot/modules/trickest.py index 7cbbfda3c4..c008a89bd5 100644 --- a/bbot/modules/trickest.py +++ b/bbot/modules/trickest.py @@ -1,4 +1,6 @@ from bbot.modules.templates.subdomain_enum import subdomain_enum_apikey +from pydantic import Field +from bbot.core.config.models import BaseModuleConfig class Trickest(subdomain_enum_apikey): @@ -11,12 +13,8 @@ class Trickest(subdomain_enum_apikey): "created_date": "2024-07-27", "auth_required": True, } - options = { - "api_key": "", - } - options_desc = { - "api_key": "Trickest API key", - } + class Config(BaseModuleConfig): + api_key: str = Field('', description='Trickest API key') base_url = "https://api.trickest.io/solutions/v1/public/solution/a7cba1f1-df07-4a5c-876a-953f178996be" ping_url = f"{base_url}/dataset" diff --git a/bbot/modules/trufflehog.py b/bbot/modules/trufflehog.py index 21a233262a..95cfd6d016 100644 --- a/bbot/modules/trufflehog.py +++ b/bbot/modules/trufflehog.py @@ -1,6 +1,8 @@ import json from functools import partial from bbot.modules.base import BaseModule +from pydantic import Field +from bbot.core.config.models import BaseModuleConfig class trufflehog(BaseModule): @@ -13,20 +15,12 @@ class trufflehog(BaseModule): "author": "@domwhewell-sage", } - options = { - "version": "3.90.8", - "config": "", - "only_verified": True, - "concurrency": 8, - "deleted_forks": False, - } - options_desc = { - "version": "trufflehog version", - "config": "File path or URL to YAML trufflehog config", - "only_verified": "Only report credentials that have been verified", - "concurrency": "Number of concurrent workers", - "deleted_forks": "Scan for deleted github forks. WARNING: This is SLOW. For a smaller repository, this process can take 20 minutes. For a larger repository, it could take hours.", - } + class Config(BaseModuleConfig): + version: str = Field('3.90.8', description='trufflehog version') + config: str = Field('', description='File path or URL to YAML trufflehog config') + only_verified: bool = Field(True, description='Only report credentials that have been verified') + concurrency: int = Field(8, description='Number of concurrent workers') + deleted_forks: bool = Field(False, description='Scan for deleted github forks. WARNING: This is SLOW. For a smaller repository, this process can take 20 minutes. For a larger repository, it could take hours.') deps_ansible = [ { "name": "Download trufflehog", diff --git a/bbot/modules/url_manipulation.py b/bbot/modules/url_manipulation.py index d9b7db466c..cca6effc88 100644 --- a/bbot/modules/url_manipulation.py +++ b/bbot/modules/url_manipulation.py @@ -1,5 +1,7 @@ from bbot.errors import HttpCompareError from bbot.modules.base import BaseModule +from pydantic import Field +from bbot.core.config.models import BaseModuleConfig class url_manipulation(BaseModule): @@ -13,10 +15,8 @@ class url_manipulation(BaseModule): } in_scope_only = True - options = {"allow_redirects": True} - options_desc = { - "allow_redirects": "Allowing redirects will sometimes create false positives. Disallowing will sometimes create false negatives. Allowed by default." - } + class Config(BaseModuleConfig): + allow_redirects: bool = Field(True, description='Allowing redirects will sometimes create false positives. Disallowing will sometimes create false negatives. Allowed by default.') async def setup(self): # ([string]method,[string]path,[bool]strip trailing slash) diff --git a/bbot/modules/urlscan.py b/bbot/modules/urlscan.py index 9a09222fe2..c433e9a0d1 100644 --- a/bbot/modules/urlscan.py +++ b/bbot/modules/urlscan.py @@ -1,4 +1,6 @@ from bbot.modules.templates.subdomain_enum import subdomain_enum +from pydantic import Field +from bbot.core.config.models import BaseModuleConfig class urlscan(subdomain_enum): @@ -10,8 +12,8 @@ class urlscan(subdomain_enum): "created_date": "2022-06-09", "author": "@TheTechromancer", } - options = {"urls": False} - options_desc = {"urls": "Emit URLs in addition to DNS_NAMEs"} + class Config(BaseModuleConfig): + urls: bool = Field(False, description='Emit URLs in addition to DNS_NAMEs') base_url = "https://urlscan.io/api/v1" diff --git a/bbot/modules/virustotal.py b/bbot/modules/virustotal.py index 75766b38d8..ec7e1e3ebd 100644 --- a/bbot/modules/virustotal.py +++ b/bbot/modules/virustotal.py @@ -1,4 +1,6 @@ from bbot.modules.templates.subdomain_enum import subdomain_enum_apikey +from pydantic import Field +from bbot.core.config.models import BaseModuleConfig class virustotal(subdomain_enum_apikey): @@ -11,8 +13,8 @@ class virustotal(subdomain_enum_apikey): "author": "@TheTechromancer", "auth_required": True, } - options = {"api_key": ""} - options_desc = {"api_key": "VirusTotal API Key"} + class Config(BaseModuleConfig): + api_key: str = Field('', description='VirusTotal API Key') base_url = "https://www.virustotal.com/api/v3" api_page_iter_kwargs = {"json": False, "next_key": lambda r: r.json().get("links", {}).get("next", "")} diff --git a/bbot/modules/wafw00f.py b/bbot/modules/wafw00f.py index 6063a3be33..58d2085c62 100644 --- a/bbot/modules/wafw00f.py +++ b/bbot/modules/wafw00f.py @@ -3,6 +3,8 @@ # disable wafw00f logging import logging +from pydantic import Field +from bbot.core.config.models import BaseModuleConfig wafw00f_logger = logging.getLogger("wafw00f") wafw00f_logger.setLevel(logging.CRITICAL + 100) @@ -24,8 +26,8 @@ class wafw00f(BaseModule): deps_pip = ["wafw00f~=2.3.1"] - options = {"generic_detect": True} - options_desc = {"generic_detect": "When no specific WAF detections are made, try to perform a generic detect"} + class Config(BaseModuleConfig): + generic_detect: bool = Field(True, description='When no specific WAF detections are made, try to perform a generic detect') in_scope_only = True per_hostport_only = True diff --git a/bbot/modules/wayback.py b/bbot/modules/wayback.py index 49010f451a..c07b3ff827 100644 --- a/bbot/modules/wayback.py +++ b/bbot/modules/wayback.py @@ -1,6 +1,8 @@ from datetime import datetime from bbot.modules.templates.subdomain_enum import subdomain_enum +from pydantic import Field +from bbot.core.config.models import BaseModuleConfig class wayback(subdomain_enum): @@ -12,11 +14,9 @@ class wayback(subdomain_enum): "created_date": "2022-04-01", "author": "@liquidsec", } - options = {"urls": False, "garbage_threshold": 10} - options_desc = { - "urls": "emit URLs in addition to DNS_NAMEs", - "garbage_threshold": "Dedupe similar urls if they are in a group of this size or higher (lower values == less garbage data)", - } + class Config(BaseModuleConfig): + urls: bool = Field(False, description='emit URLs in addition to DNS_NAMEs') + garbage_threshold: int = Field(10, description='Dedupe similar urls if they are in a group of this size or higher (lower values == less garbage data)') in_scope_only = True base_url = "http://web.archive.org" diff --git a/bbot/modules/wpscan.py b/bbot/modules/wpscan.py index b7a703b90d..4fc1c0ddc1 100644 --- a/bbot/modules/wpscan.py +++ b/bbot/modules/wpscan.py @@ -1,5 +1,7 @@ import json from bbot.modules.base import BaseModule +from pydantic import Field +from bbot.core.config.models import BaseModuleConfig class wpscan(BaseModule): @@ -12,24 +14,14 @@ class wpscan(BaseModule): "author": "@domwhewell-sage", } - options = { - "api_key": "", - "enumerate": "vp,vt,cb,dbe", - "threads": 5, - "request_timeout": 5, - "connection_timeout": 2, - "disable_tls_checks": True, - "force": False, - } - options_desc = { - "api_key": "WPScan API Key", - "enumerate": "Enumeration Process see wpscan help documentation (default: vp,vt,cb,dbe)", - "threads": "How many wpscan threads to spawn (default is 5)", - "request_timeout": "The request timeout in seconds (default 5)", - "connection_timeout": "The connection timeout in seconds (default 2)", - "disable_tls_checks": "Disables the SSL/TLS certificate verification (Default True)", - "force": "Do not check if the target is running WordPress or returns a 403", - } + class Config(BaseModuleConfig): + api_key: str = Field('', description='WPScan API Key') + enumerate: str = Field('vp,vt,cb,dbe', description='Enumeration Process see wpscan help documentation (default: vp,vt,cb,dbe)') + threads: int = Field(5, description='How many wpscan threads to spawn (default is 5)') + request_timeout: int = Field(5, description='The request timeout in seconds (default 5)') + connection_timeout: int = Field(2, description='The connection timeout in seconds (default 2)') + disable_tls_checks: bool = Field(True, description='Disables the SSL/TLS certificate verification (Default True)') + force: bool = Field(False, description='Do not check if the target is running WordPress or returns a 403') deps_apt = ["curl", "make", "gcc"] deps_ansible = [ { diff --git a/bbot/scanner/preset/validate.py b/bbot/scanner/preset/validate.py index f1162ae795..a50e3a0523 100644 --- a/bbot/scanner/preset/validate.py +++ b/bbot/scanner/preset/validate.py @@ -2,29 +2,30 @@ Public `validate_preset()` API for BBOT presets. Given a preset-as-dict (i.e. what comes out of a YAML file before it's -instantiated), return a list of human-readable errors in three layers: +instantiated), return a list of human-readable errors. -1. Top-level preset keys (e.g. `modlues:` typo) -2. Global config (e.g. `scope.strct`, `web.sslverify`) -3. Per-module config (e.g. `modules.nuclei.tgas`) +The validator does a single pass against a composite pydantic schema +(built once per module-loader state by `ModuleLoader.validation_schema`) +that covers every layer in one tree: -Errors are aggregated — a user with multiple typos gets all the errors at once -rather than having to fix one at a time. The caller decides whether to raise -or just print them. +- top-level preset keys (e.g. `modlues:` typo) +- global config (e.g. `scope.strct`, `web.http_timoeut`) +- unknown module names (e.g. `modules.nucleii`) +- per-module config (e.g. `modules.nuclei.tgas`) + +Errors are aggregated so a user with multiple typos sees them all. The +caller decides whether to raise or just print them. """ from __future__ import annotations -import importlib import logging from dataclasses import dataclass from pathlib import Path -from typing import Iterable, Optional +from typing import Any from pydantic import ValidationError -from bbot.core.config.models import BBOTConfig, BaseModuleConfig, PresetSchema - log = logging.getLogger("bbot.presets.validate") @@ -35,189 +36,138 @@ class PresetValidationError: where: str # "preset", "config", "module:" path: str # dotted location, e.g. "scope.strict" or "" - message: str # pydantic-derived message, reformatted + message: str def __str__(self) -> str: loc = f"{self.where}:{self.path}" if self.path else self.where return f"[{loc}] {self.message}" -def _format_pydantic_errors(exc: ValidationError, where: str) -> list[PresetValidationError]: +def _classify_loc(loc: tuple) -> tuple[str, str]: + """ + Map a pydantic error path to a (where, path) pair. + + Examples: + ('modlues',) -> ('preset', 'modlues') + ('config', 'scope', 'strct') -> ('config', 'scope.strct') + ('config', 'modules', 'nucleii') -> ('preset', 'config.modules.nucleii') + ('config', 'modules', 'nuclei', 'tgas') -> ('module:nuclei', 'tgas') + """ + parts = [str(p) for p in loc] + + if len(parts) >= 2 and parts[0] == "config" and parts[1] == "modules": + # Error is somewhere under config.modules.* + if len(parts) == 3: + # The module name itself is unknown (extra_forbidden on ModulesSchema) + return ("preset", ".".join(parts)) + # Error is within a known module's config + module_name = parts[2] + return (f"module:{module_name}", ".".join(parts[3:])) + + if parts and parts[0] == "config": + return ("config", ".".join(parts[1:])) + + return ("preset", ".".join(parts)) + + +def _format_msg(err: dict) -> str: + kind = err["type"] + input_value = err.get("input") + loc = err["loc"] + field = str(loc[-1]) if loc else "" + path = ".".join(str(p) for p in loc) + + if kind == "extra_forbidden": + # Special-case the unknown-module-name error so users get + # "Unknown module: 'nucleii'" instead of "Unknown option: 'nucleii'". + if len(loc) == 3 and loc[0] == "config" and loc[1] == "modules": + return f'Unknown module: "{field}"' + msg = f"Unknown option: {field!r}" + if isinstance(input_value, (str, int, bool, float)): + msg += f" (value: {input_value!r})" + elif isinstance(input_value, list) and len(input_value) <= 5: + msg += f" (value: {input_value!r})" + return msg + + if kind in {"int_parsing", "int_type"}: + return f"Expected an integer, got {type(input_value).__name__}: {input_value!r}" + if kind in {"bool_parsing", "bool_type"}: + return f"Expected a boolean, got {type(input_value).__name__}: {input_value!r}" + if kind == "string_type": + return f"Expected a string, got {type(input_value).__name__}: {input_value!r}" + if kind == "list_type": + return f"Expected a list, got {type(input_value).__name__}" + if kind == "dict_type": + return f"Expected a mapping, got {type(input_value).__name__}" + if kind == "literal_error": + # pydantic stashes the allowed values in ctx["expected"] formatted like: + # "'manual', 'technology', 'severe' or 'budget'" + ctx = err.get("ctx") or {} + expected = ctx.get("expected", "") + return f"Expected one of {expected}, got {input_value!r}" if expected else err.get("msg", "") + if kind == "missing": + return f"Required option {field!r} is missing" + + # Fallback to pydantic's own message + return err["msg"] if err.get("msg") else f"validation error at {path}" + + +def _format_errors(exc: ValidationError) -> list[PresetValidationError]: out: list[PresetValidationError] = [] for err in exc.errors(): - path = ".".join(str(p) for p in err["loc"]) - kind = err["type"] - input_value = err.get("input") - if kind == "extra_forbidden": - msg = f"Unknown option: {path!r}" - if input_value is not None and isinstance(input_value, (str, int, bool, float)): - msg += f" (value: {input_value!r})" - elif kind in {"int_parsing", "int_type"}: - msg = f"Expected an integer, got {type(input_value).__name__}: {input_value!r}" - elif kind in {"bool_parsing", "bool_type"}: - msg = f"Expected a boolean, got {type(input_value).__name__}: {input_value!r}" - elif kind in {"string_type"}: - msg = f"Expected a string, got {type(input_value).__name__}: {input_value!r}" - elif kind == "list_type": - msg = f"Expected a list, got {type(input_value).__name__}" - elif kind == "missing": - msg = "Required option is missing" - else: - msg = err["msg"] - out.append(PresetValidationError(where=where, path=path, message=msg)) + where, path = _classify_loc(err["loc"]) + out.append(PresetValidationError(where=where, path=path, message=_format_msg(err))) return out -def _get_module_config_class(module_name: str, module_loader) -> Optional[type[BaseModuleConfig]]: +def validate_preset(preset_dict: Any, module_loader=None) -> list[PresetValidationError]: """ - Return the module's `Config` class, or None if the module doesn't declare one - (legacy `options`/`options_desc` dict modules). - """ - preloaded = module_loader.preloaded().get(module_name) - if not preloaded: - return None - module_path = Path(preloaded["path"]) - namespace = preloaded["namespace"] - full_namespace = f"{namespace}.{module_name}" - - # Re-import by path. Uses the existing importlib pattern from ModuleLoader.load_module. - spec = importlib.util.spec_from_file_location(full_namespace, str(module_path)) - if spec is None or spec.loader is None: - return None - mod = importlib.util.module_from_spec(spec) - try: - spec.loader.exec_module(mod) - except Exception as e: - log.debug(f"Could not import {module_name} for validation: {e}") - return None - - # Find the module class (same heuristic as ModuleLoader.load_module) - for attr_name in vars(mod): - value = getattr(mod, attr_name) - if not hasattr(value, "watched_events") or not hasattr(value, "produced_events"): - continue - if not isinstance(getattr(value, "watched_events"), list): - continue - if getattr(value, "__name__", "").lower() != module_name.lower(): - continue - cfg = getattr(value, "Config", None) - if cfg is not None and isinstance(cfg, type) and issubclass(cfg, BaseModuleConfig): - return cfg - return None - return None - - -def _modules_referenced(preset_dict: dict, config_dict: dict) -> set[str]: - """Every module mentioned anywhere in the preset — enabled, configured, or both.""" - names: set[str] = set() - for key in ("modules", "output_modules", "exclude_modules"): - names.update(preset_dict.get(key) or []) - modules_config = config_dict.get("modules") or {} - if isinstance(modules_config, dict): - names.update(modules_config.keys()) - return names - - -def validate_preset( - preset_dict: dict, - module_loader=None, - *, - validate_modules: bool = True, - known_modules: Optional[Iterable[str]] = None, -) -> list[PresetValidationError]: - """ - Validate a preset dict against BBOT's schemas. + Validate a preset dict against BBOT's composite schema. - Returns a list of `PresetValidationError` objects. An empty list means the - preset is valid. The function aggregates errors across all three layers - (preset, config, per-module) so a user with multiple typos sees them all. + Returns a list of `PresetValidationError` objects. An empty list means + the preset is valid. Errors from all layers are aggregated in a single + pass, so a user with multiple typos sees them all at once. Args: preset_dict: Preset as a plain dict (e.g. from `yaml.safe_load`). - module_loader: Optional module loader for per-module validation. Falls - back to the global MODULE_LOADER if not provided. If neither is - available, per-module validation is skipped. - validate_modules: If False, only validate preset top-level keys and the - global config tree (skip per-module Config validation). - known_modules: Optional set of known module names. If provided, module - names not in this set are reported as errors. Defaults to the - module loader's known modules. + module_loader: Optional module loader. Falls back to the global + `MODULE_LOADER` if not provided. Examples: >>> errors = validate_preset({"modlues": ["nuclei"]}) >>> print(errors[0]) [preset:modlues] Unknown option: 'modlues' (value: ['nuclei']) """ - errors: list[PresetValidationError] = [] - if not isinstance(preset_dict, dict): return [PresetValidationError("preset", "", f"Expected a dict, got {type(preset_dict).__name__}")] - # 1. Top-level preset keys + if module_loader is None: + from bbot.core.modules import MODULE_LOADER + + module_loader = MODULE_LOADER + + errors: list[PresetValidationError] = [] + + # Single-pass validation against the composite schema try: - PresetSchema.model_validate(preset_dict) + module_loader.validation_schema.model_validate(preset_dict) except ValidationError as e: - errors.extend(_format_pydantic_errors(e, where="preset")) - - # 2. Global config tree - config_dict = preset_dict.get("config") or {} - if not isinstance(config_dict, dict): - errors.append( - PresetValidationError("config", "", f"`config` must be a dict, got {type(config_dict).__name__}") - ) - config_dict = {} - else: - # Exclude the `modules` section from root-level BBOTConfig validation; - # per-module schemas are validated separately below. - config_for_root = {k: v for k, v in config_dict.items() if k != "modules"} - try: - BBOTConfig.model_validate(config_for_root) - except ValidationError as e: - errors.extend(_format_pydantic_errors(e, where="config")) - - # 3. Per-module config - if validate_modules: - if module_loader is None: - try: - from bbot.core.modules import MODULE_LOADER - - module_loader = MODULE_LOADER - except Exception: - module_loader = None - - referenced = _modules_referenced(preset_dict, config_dict) - - if module_loader is not None: - if known_modules is None: - known_modules = set(module_loader.all_module_choices) - else: - known_modules = set(known_modules) - - modules_config = config_dict.get("modules") or {} - for name in sorted(referenced): - if name not in known_modules: - errors.append(PresetValidationError("preset", "modules", f'Unknown module: "{name}"')) - continue - - raw = modules_config.get(name) or {} - if not isinstance(raw, dict): - errors.append( - PresetValidationError( - f"module:{name}", - "", - f"module config must be a dict, got {type(raw).__name__}", - ) - ) - continue + errors.extend(_format_errors(e)) - cfg_cls = _get_module_config_class(name, module_loader) - if cfg_cls is None: - # legacy module — no Config class yet. Skip strict validation. - continue - try: - cfg_cls.model_validate(raw) - except ValidationError as e: - errors.extend(_format_pydantic_errors(e, where=f"module:{name}")) + # Module names listed in top-level `modules`/`output_modules`/`exclude_modules` + # aren't covered by the composite schema (they're a list of strings, not a + # nested mapping). Check them here. + known_modules = set(module_loader.all_module_choices) + for key in ("modules", "output_modules", "exclude_modules"): + for name in preset_dict.get(key) or []: + if name not in known_modules: + errors.append( + PresetValidationError( + where="preset", + path=key, + message=f'Unknown module: "{name}"', + ) + ) return errors @@ -234,12 +184,3 @@ def validate_preset_file(path: str | Path, **kwargs) -> list[PresetValidationErr __all__ = ["PresetValidationError", "validate_preset", "validate_preset_file"] - - -def _assert_preset_valid(preset_dict: dict, **kwargs) -> None: - """Raise if a preset dict has any validation errors. Tests use this.""" - from bbot.errors import ValidationError as BBOTValidationError - - errs = validate_preset(preset_dict, **kwargs) - if errs: - raise BBOTValidationError("\n".join(str(e) for e in errs)) diff --git a/bbot/test/test_step_1/test_validate_preset.py b/bbot/test/test_step_1/test_validate_preset.py index 4c697893fa..128a5c1979 100644 --- a/bbot/test/test_step_1/test_validate_preset.py +++ b/bbot/test/test_step_1/test_validate_preset.py @@ -9,14 +9,14 @@ def test_validate_preset_valid(): def test_validate_preset_typo_top_level(): - errs = validate_preset({"modlues": ["nuclei"]}, validate_modules=False) + errs = validate_preset({"modlues": ["nuclei"]}) assert len(errs) == 1 assert errs[0].where == "preset" assert "modlues" in errs[0].message def test_validate_preset_typo_in_config(): - errs = validate_preset({"config": {"scope": {"strct": True}}}, validate_modules=False) + errs = validate_preset({"config": {"scope": {"strct": True}}}) assert len(errs) == 1 assert errs[0].where == "config" assert "strct" in errs[0].message @@ -24,7 +24,7 @@ def test_validate_preset_typo_in_config(): def test_validate_preset_wrong_type(): - errs = validate_preset({"config": {"web": {"http_timeout": "not-a-number"}}}, validate_modules=False) + errs = validate_preset({"config": {"web": {"http_timeout": "not-a-number"}}}) assert len(errs) == 1 assert errs[0].where == "config" assert errs[0].path == "web.http_timeout" @@ -36,6 +36,30 @@ def test_validate_preset_unknown_module(): assert any('Unknown module: "nucleii"' in str(e) for e in errs) +def test_validate_preset_unknown_module_option(): + """Typo in a known module's option key gets tagged `module:`.""" + errs = validate_preset({"config": {"modules": {"nuclei": {"tgas": "apache"}}}}) + assert len(errs) == 1 + assert errs[0].where == "module:nuclei" + assert errs[0].path == "tgas" + assert "tgas" in errs[0].message + + +def test_validate_preset_wrong_type_on_module_option(): + """Known module option with wrong type (nuclei.ratelimit is int).""" + errs = validate_preset({"config": {"modules": {"nuclei": {"ratelimit": "not-a-number"}}}}) + assert len(errs) == 1 + assert errs[0].where == "module:nuclei" + assert errs[0].path == "ratelimit" + assert "integer" in errs[0].message + + +def test_validate_preset_unknown_module_in_config(): + """Unknown module name nested under config.modules gets a clean error.""" + errs = validate_preset({"config": {"modules": {"nucleii": {"tgas": "x"}}}}) + assert any('Unknown module: "nucleii"' in str(e) for e in errs) + + def test_validate_preset_multiple_errors(): """A preset with several typos should produce errors for all of them, not just the first.""" errs = validate_preset( @@ -46,7 +70,6 @@ def test_validate_preset_multiple_errors(): "web": {"http_timeout": "bad"}, # wrong type }, }, - validate_modules=False, ) assert len(errs) >= 3 messages = " ".join(str(e) for e in errs) From 393ae72b17303892002fb9f3df1dba4894f321b4 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Mon, 27 Apr 2026 13:57:54 -0400 Subject: [PATCH 04/15] ruffed --- bbot/modules/anubisdb.py | 6 ++- bbot/modules/apkpure.py | 6 ++- bbot/modules/baddns.py | 2 + bbot/modules/baddns_direct.py | 10 ++-- bbot/modules/baddns_zone.py | 10 ++-- bbot/modules/badsecrets.py | 4 +- bbot/modules/bevigil.py | 5 +- bbot/modules/bucket_amazon.py | 4 +- bbot/modules/bucket_digitalocean.py | 3 +- bbot/modules/bucket_file_enum.py | 4 +- bbot/modules/bucket_firebase.py | 3 +- bbot/modules/bucket_google.py | 3 +- bbot/modules/bucket_microsoft.py | 3 +- bbot/modules/bufferoverrun.py | 5 +- bbot/modules/builtwith.py | 6 ++- bbot/modules/c99.py | 3 +- bbot/modules/censys_dns.py | 5 +- bbot/modules/censys_ip.py | 12 +++-- bbot/modules/chaos.py | 3 +- bbot/modules/credshed.py | 8 +-- bbot/modules/dehashed.py | 4 +- bbot/modules/dnsbimi.py | 7 +-- bbot/modules/dnsbrute.py | 9 +++- bbot/modules/dnsbrute_mutations.py | 4 +- bbot/modules/dnscaa.py | 10 ++-- bbot/modules/dnscommonsrv.py | 2 +- bbot/modules/dnstlsrpt.py | 7 +-- bbot/modules/docker_pull.py | 8 ++- bbot/modules/ffuf.py | 18 ++++--- bbot/modules/ffuf_shortnames.py | 32 +++++++---- bbot/modules/filedownload.py | 74 ++++++++++++++++++++++++-- bbot/modules/fullhunt.py | 3 +- bbot/modules/git_clone.py | 8 ++- bbot/modules/gitdumper.py | 12 +++-- bbot/modules/github_codesearch.py | 5 +- bbot/modules/github_org.py | 5 +- bbot/modules/github_usersearch.py | 3 +- bbot/modules/github_workflows.py | 7 +-- bbot/modules/gitlab_com.py | 2 +- bbot/modules/gitlab_onprem.py | 2 +- bbot/modules/gowitness.py | 22 ++++---- bbot/modules/graphql_introspection.py | 7 ++- bbot/modules/httpx.py | 13 ++--- bbot/modules/hunterio.py | 3 +- bbot/modules/iis_shortnames.py | 10 +++- bbot/modules/internal/excavate.py | 11 ++-- bbot/modules/internal/speculate.py | 11 ++-- bbot/modules/ip2location.py | 9 +++- bbot/modules/ipneighbor.py | 4 +- bbot/modules/ipstack.py | 4 +- bbot/modules/jadx.py | 2 + bbot/modules/kreuzberg.py | 52 +++++++++++++++++- bbot/modules/leakix.py | 4 +- bbot/modules/legba.py | 42 +++++++++++---- bbot/modules/lightfuzz/lightfuzz.py | 25 ++++++--- bbot/modules/medusa.py | 20 ++++--- bbot/modules/ntlm.py | 3 +- bbot/modules/nuclei.py | 23 ++++---- bbot/modules/oauth.py | 3 +- bbot/modules/otx.py | 3 +- bbot/modules/output/asset_inventory.py | 15 ++++-- bbot/modules/output/csv.py | 3 +- bbot/modules/output/discord.py | 9 ++-- bbot/modules/output/elastic.py | 12 +++-- bbot/modules/output/emails.py | 4 +- bbot/modules/output/http.py | 15 +++--- bbot/modules/output/json.py | 4 +- bbot/modules/output/kafka.py | 8 ++- bbot/modules/output/mongo.py | 12 +++-- bbot/modules/output/mysql.py | 12 +++-- bbot/modules/output/nats.py | 6 ++- bbot/modules/output/neo4j.py | 8 +-- bbot/modules/output/postgres.py | 12 +++-- bbot/modules/output/rabbitmq.py | 6 ++- bbot/modules/output/slack.py | 10 ++-- bbot/modules/output/splunk.py | 11 ++-- bbot/modules/output/sqlite.py | 4 +- bbot/modules/output/stdout.py | 12 +++-- bbot/modules/output/subdomains.py | 6 ++- bbot/modules/output/teams.py | 9 ++-- bbot/modules/output/txt.py | 3 +- bbot/modules/output/web_parameters.py | 5 +- bbot/modules/output/web_report.py | 9 +++- bbot/modules/output/websocket.py | 13 +++-- bbot/modules/output/zeromq.py | 5 +- bbot/modules/paramminer_cookies.py | 6 +-- bbot/modules/paramminer_getparams.py | 10 ++-- bbot/modules/paramminer_headers.py | 10 ++-- bbot/modules/passivetotal.py | 3 +- bbot/modules/pgp.py | 11 +++- bbot/modules/portfilter.py | 7 ++- bbot/modules/portscan.py | 33 ++++++++---- bbot/modules/postman.py | 4 +- bbot/modules/postman_download.py | 9 +++- bbot/modules/retirejs.py | 9 ++-- bbot/modules/securitytrails.py | 3 +- bbot/modules/securitytxt.py | 5 +- bbot/modules/shodan_dns.py | 3 +- bbot/modules/shodan_enterprise.py | 8 ++- bbot/modules/shodan_idb.py | 6 ++- bbot/modules/sslcert.py | 4 +- bbot/modules/subdomainradar.py | 7 +-- bbot/modules/telerik.py | 4 +- bbot/modules/trajan.py | 16 +++--- bbot/modules/trickest.py | 3 +- bbot/modules/trufflehog.py | 14 +++-- bbot/modules/url_manipulation.py | 5 +- bbot/modules/urlscan.py | 3 +- bbot/modules/virustotal.py | 3 +- bbot/modules/wafw00f.py | 4 +- bbot/modules/wayback.py | 9 +++- bbot/modules/wpscan.py | 19 ++++--- bbot/scanner/preset/validate.py | 55 ++++++++++++------- 113 files changed, 765 insertions(+), 319 deletions(-) diff --git a/bbot/modules/anubisdb.py b/bbot/modules/anubisdb.py index 0d667e15a0..2e1688a489 100644 --- a/bbot/modules/anubisdb.py +++ b/bbot/modules/anubisdb.py @@ -12,8 +12,12 @@ class anubisdb(subdomain_enum): "created_date": "2022-10-04", "author": "@TheTechromancer", } + class Config(BaseModuleConfig): - limit: int = Field(1000, description='Limit the number of subdomains returned per query (increasing this may slow the scan due to garbage results from this API)') + limit: int = Field( + 1000, + description="Limit the number of subdomains returned per query (increasing this may slow the scan due to garbage results from this API)", + ) base_url = "https://jldc.me/anubis/subdomains" dns_abort_depth = 5 diff --git a/bbot/modules/apkpure.py b/bbot/modules/apkpure.py index 1e38d3ea5d..602c048725 100644 --- a/bbot/modules/apkpure.py +++ b/bbot/modules/apkpure.py @@ -14,8 +14,12 @@ class apkpure(BaseModule): "created_date": "2024-10-11", "author": "@domwhewell-sage", } + class Config(BaseModuleConfig): - output_folder: str = Field('', description='Folder to download APKs to. If not specified, downloaded APKs will be deleted when the scan completes, to minimize disk usage.') + output_folder: str = Field( + "", + description="Folder to download APKs to. If not specified, downloaded APKs will be deleted when the scan completes, to minimize disk usage.", + ) async def setup(self): output_folder = self.config.get("output_folder", "") diff --git a/bbot/modules/baddns.py b/bbot/modules/baddns.py index 5cbddf37ed..45d2f28a93 100644 --- a/bbot/modules/baddns.py +++ b/bbot/modules/baddns.py @@ -48,6 +48,7 @@ class baddns(BaseModule): "created_date": "2024-01-18", "author": "@liquidsec", } + class Config(BaseModuleConfig): custom_nameservers: list[str] = Field( default_factory=list, description="Force BadDNS to use a list of custom nameservers" @@ -62,6 +63,7 @@ class Config(BaseModuleConfig): default_factory=list, description="A list of submodules to enable. Empty list (default) enables CNAME, TXT and MX Only", ) + module_threads = 8 deps_pip = ["baddns~=2.1.0"] diff --git a/bbot/modules/baddns_direct.py b/bbot/modules/baddns_direct.py index 2bde5917b0..6f9a7ede3a 100644 --- a/bbot/modules/baddns_direct.py +++ b/bbot/modules/baddns_direct.py @@ -12,10 +12,14 @@ class baddns_direct(baddns_module): "created_date": "2024-01-29", "author": "@liquidsec", } + class Config(BaseModuleConfig): - custom_nameservers: list = Field([], description='Force BadDNS to use a list of custom nameservers') - min_severity: str = Field('LOW', description='Minimum severity to emit (INFO, LOW, MEDIUM, HIGH, CRITICAL)') - min_confidence: str = Field('MEDIUM', description='Minimum confidence to emit (UNKNOWN, LOW, MEDIUM, HIGH, CONFIRMED)') + custom_nameservers: list = Field([], description="Force BadDNS to use a list of custom nameservers") + min_severity: str = Field("LOW", description="Minimum severity to emit (INFO, LOW, MEDIUM, HIGH, CRITICAL)") + min_confidence: str = Field( + "MEDIUM", description="Minimum confidence to emit (UNKNOWN, LOW, MEDIUM, HIGH, CONFIRMED)" + ) + module_threads = 8 deps_pip = ["baddns~=2.1.0"] diff --git a/bbot/modules/baddns_zone.py b/bbot/modules/baddns_zone.py index 5193b9ef81..25eda1695d 100644 --- a/bbot/modules/baddns_zone.py +++ b/bbot/modules/baddns_zone.py @@ -12,10 +12,14 @@ class baddns_zone(baddns_module): "created_date": "2024-01-29", "author": "@liquidsec", } + class Config(BaseModuleConfig): - custom_nameservers: list = Field([], description='Force BadDNS to use a list of custom nameservers') - min_severity: str = Field('INFO', description='Minimum severity to emit (INFO, LOW, MEDIUM, HIGH, CRITICAL)') - min_confidence: str = Field('MEDIUM', description='Minimum confidence to emit (UNKNOWN, LOW, MEDIUM, HIGH, CONFIRMED)') + custom_nameservers: list = Field([], description="Force BadDNS to use a list of custom nameservers") + min_severity: str = Field("INFO", description="Minimum severity to emit (INFO, LOW, MEDIUM, HIGH, CRITICAL)") + min_confidence: str = Field( + "MEDIUM", description="Minimum confidence to emit (UNKNOWN, LOW, MEDIUM, HIGH, CONFIRMED)" + ) + module_threads = 8 deps_pip = ["baddns~=2.1.0"] diff --git a/bbot/modules/badsecrets.py b/bbot/modules/badsecrets.py index b398661604..f9847e8fb1 100644 --- a/bbot/modules/badsecrets.py +++ b/bbot/modules/badsecrets.py @@ -16,8 +16,10 @@ class badsecrets(BaseModule): "created_date": "2022-11-19", "author": "@liquidsec", } + class Config(BaseModuleConfig): - custom_secrets: Optional[str] = Field(None, description='Include custom secrets loaded from a local file') + custom_secrets: Optional[str] = Field(None, description="Include custom secrets loaded from a local file") + deps_pip = ["badsecrets~=1.0.0"] async def setup(self): diff --git a/bbot/modules/bevigil.py b/bbot/modules/bevigil.py index 43de0fe4f9..9103fe872b 100644 --- a/bbot/modules/bevigil.py +++ b/bbot/modules/bevigil.py @@ -17,9 +17,10 @@ class bevigil(subdomain_enum_apikey): "author": "@alt-glitch", "auth_required": True, } + class Config(BaseModuleConfig): - api_key: str = Field('', description='BeVigil OSINT API Key') - urls: bool = Field(False, description='Emit URLs in addition to DNS_NAMEs') + api_key: str = Field("", description="BeVigil OSINT API Key") + urls: bool = Field(False, description="Emit URLs in addition to DNS_NAMEs") base_url = "https://osint.bevigil.com/api" diff --git a/bbot/modules/bucket_amazon.py b/bbot/modules/bucket_amazon.py index 10cd2a3708..98c0f74846 100644 --- a/bbot/modules/bucket_amazon.py +++ b/bbot/modules/bucket_amazon.py @@ -12,8 +12,10 @@ class bucket_amazon(bucket_template): "created_date": "2022-11-04", "author": "@TheTechromancer", } + class Config(BaseModuleConfig): - permutations: bool = Field(False, description='Whether to try permutations') + permutations: bool = Field(False, description="Whether to try permutations") + scope_distance_modifier = 3 cloudcheck_provider_name = "Amazon" diff --git a/bbot/modules/bucket_digitalocean.py b/bbot/modules/bucket_digitalocean.py index a4ae6e7128..187e71f2ba 100644 --- a/bbot/modules/bucket_digitalocean.py +++ b/bbot/modules/bucket_digitalocean.py @@ -12,8 +12,9 @@ class bucket_digitalocean(bucket_template): "created_date": "2022-11-08", "author": "@TheTechromancer", } + class Config(BaseModuleConfig): - permutations: bool = Field(False, description='Whether to try permutations') + permutations: bool = Field(False, description="Whether to try permutations") cloudcheck_provider_name = "DigitalOcean" delimiters = ("", "-") diff --git a/bbot/modules/bucket_file_enum.py b/bbot/modules/bucket_file_enum.py index ac96a94219..b3d3d854b3 100644 --- a/bbot/modules/bucket_file_enum.py +++ b/bbot/modules/bucket_file_enum.py @@ -19,8 +19,10 @@ class bucket_file_enum(BaseModule): "author": "@TheTechromancer", } flags = ["safe", "passive", "cloud-enum"] + class Config(BaseModuleConfig): - file_limit: int = Field(50, description='Limit the number of files downloaded per bucket') + file_limit: int = Field(50, description="Limit the number of files downloaded per bucket") + scope_distance_modifier = 2 async def setup(self): diff --git a/bbot/modules/bucket_firebase.py b/bbot/modules/bucket_firebase.py index 81fab8bfde..ce93f713ef 100644 --- a/bbot/modules/bucket_firebase.py +++ b/bbot/modules/bucket_firebase.py @@ -12,8 +12,9 @@ class bucket_firebase(bucket_template): "created_date": "2023-03-20", "author": "@TheTechromancer", } + class Config(BaseModuleConfig): - permutations: bool = Field(False, description='Whether to try permutations') + permutations: bool = Field(False, description="Whether to try permutations") cloudcheck_provider_name = "Google" delimiters = ("", "-") diff --git a/bbot/modules/bucket_google.py b/bbot/modules/bucket_google.py index 5610c87773..9185f2ce6b 100644 --- a/bbot/modules/bucket_google.py +++ b/bbot/modules/bucket_google.py @@ -16,8 +16,9 @@ class bucket_google(bucket_template): "created_date": "2022-11-04", "author": "@TheTechromancer", } + class Config(BaseModuleConfig): - permutations: bool = Field(False, description='Whether to try permutations') + permutations: bool = Field(False, description="Whether to try permutations") cloudcheck_provider_name = "Google" delimiters = ("", "-", ".", "_") diff --git a/bbot/modules/bucket_microsoft.py b/bbot/modules/bucket_microsoft.py index b392f0307a..51e4fdb1c2 100644 --- a/bbot/modules/bucket_microsoft.py +++ b/bbot/modules/bucket_microsoft.py @@ -12,8 +12,9 @@ class bucket_microsoft(bucket_template): "created_date": "2022-11-04", "author": "@TheTechromancer", } + class Config(BaseModuleConfig): - permutations: bool = Field(False, description='Whether to try permutations') + permutations: bool = Field(False, description="Whether to try permutations") cloudcheck_provider_name = "Microsoft" delimiters = ("", "-") diff --git a/bbot/modules/bufferoverrun.py b/bbot/modules/bufferoverrun.py index aa3b5c6c99..bbcd1bc0a5 100644 --- a/bbot/modules/bufferoverrun.py +++ b/bbot/modules/bufferoverrun.py @@ -13,9 +13,10 @@ class BufferOverrun(subdomain_enum_apikey): "author": "@TheTechromancer", "auth_required": True, } + class Config(BaseModuleConfig): - api_key: str = Field('', description='BufferOverrun API key') - commercial: bool = Field(False, description='Use commercial API') + api_key: str = Field("", description="BufferOverrun API key") + commercial: bool = Field(False, description="Use commercial API") base_url = "https://tls.bufferover.run/dns" commercial_base_url = "https://bufferover-run-tls.p.rapidapi.com/ipv4/dns" diff --git a/bbot/modules/builtwith.py b/bbot/modules/builtwith.py index 6dad9b6e5c..38102ed57d 100644 --- a/bbot/modules/builtwith.py +++ b/bbot/modules/builtwith.py @@ -25,9 +25,11 @@ class builtwith(subdomain_enum_apikey): "author": "@TheTechromancer", "auth_required": True, } + class Config(BaseModuleConfig): - api_key: str = Field('', description='Builtwith API key') - redirects: bool = Field(True, description='Also look up inbound and outbound redirects') + api_key: str = Field("", description="Builtwith API key") + redirects: bool = Field(True, description="Also look up inbound and outbound redirects") + base_url = "https://api.builtwith.com" async def handle_event(self, event): diff --git a/bbot/modules/c99.py b/bbot/modules/c99.py index 686861cb07..1676117779 100644 --- a/bbot/modules/c99.py +++ b/bbot/modules/c99.py @@ -13,8 +13,9 @@ class c99(subdomain_enum_apikey): "author": "@TheTechromancer", "auth_required": True, } + class Config(BaseModuleConfig): - api_key: str = Field('', description='c99.nl API key') + api_key: str = Field("", description="c99.nl API key") base_url = "https://api.c99.nl" ping_url = f"{base_url}/randomnumber?key={{api_key}}&between=1,100&json" diff --git a/bbot/modules/censys_dns.py b/bbot/modules/censys_dns.py index dc58d5bb7c..f3a4645a52 100644 --- a/bbot/modules/censys_dns.py +++ b/bbot/modules/censys_dns.py @@ -18,9 +18,10 @@ class censys_dns(censys): "author": "@TheTechromancer", "auth_required": True, } + class Config(BaseModuleConfig): - api_key: str = Field('', description="Censys.io API Key in the format of 'key:secret'") - max_pages: int = Field(5, description='Maximum number of pages to fetch (100 results per page)') + api_key: str = Field("", description="Censys.io API Key in the format of 'key:secret'") + max_pages: int = Field(5, description="Maximum number of pages to fetch (100 results per page)") async def setup(self): self.max_pages = self.config.get("max_pages", 5) diff --git a/bbot/modules/censys_ip.py b/bbot/modules/censys_ip.py index a631c82e7e..9a798dbf7c 100644 --- a/bbot/modules/censys_ip.py +++ b/bbot/modules/censys_ip.py @@ -25,10 +25,16 @@ class censys_ip(censys): "author": "@TheTechromancer", "auth_required": True, } + class Config(BaseModuleConfig): - api_key: str = Field('', description="Censys.io API Key in the format of 'key:secret'") - dns_names_limit: int = Field(100, description='Maximum number of DNS names to extract from dns.names (default 100)') - in_scope_only: bool = Field(True, description='Only query in-scope IPs. If False, will query up to distance 1.') + api_key: str = Field("", description="Censys.io API Key in the format of 'key:secret'") + dns_names_limit: int = Field( + 100, description="Maximum number of DNS names to extract from dns.names (default 100)" + ) + in_scope_only: bool = Field( + True, description="Only query in-scope IPs. If False, will query up to distance 1." + ) + scope_distance_modifier = 1 async def setup(self): diff --git a/bbot/modules/chaos.py b/bbot/modules/chaos.py index ea094fd44d..b9e803f8e9 100644 --- a/bbot/modules/chaos.py +++ b/bbot/modules/chaos.py @@ -13,8 +13,9 @@ class chaos(subdomain_enum_apikey): "author": "@TheTechromancer", "auth_required": True, } + class Config(BaseModuleConfig): - api_key: str = Field('', description='Chaos API key') + api_key: str = Field("", description="Chaos API key") base_url = "https://dns.projectdiscovery.io/dns" ping_url = f"{base_url}/example.com" diff --git a/bbot/modules/credshed.py b/bbot/modules/credshed.py index a363a31f39..1f0f13f74f 100644 --- a/bbot/modules/credshed.py +++ b/bbot/modules/credshed.py @@ -15,10 +15,12 @@ class credshed(subdomain_enum): "author": "@SpamFaux", "auth_required": True, } + class Config(BaseModuleConfig): - username: str = Field('', description='Credshed username') - password: str = Field('', description='Credshed password') - credshed_url: str = Field('', description='URL of credshed server') + username: str = Field("", description="Credshed username") + password: str = Field("", description="Credshed password") + credshed_url: str = Field("", description="URL of credshed server") + target_only = True async def setup(self): diff --git a/bbot/modules/dehashed.py b/bbot/modules/dehashed.py index 6d0cdc907b..27903b86b3 100644 --- a/bbot/modules/dehashed.py +++ b/bbot/modules/dehashed.py @@ -15,8 +15,10 @@ class dehashed(subdomain_enum): "author": "@SpamFaux", "auth_required": True, } + class Config(BaseModuleConfig): - api_key: str = Field('', description='DeHashed API Key') + api_key: str = Field("", description="DeHashed API Key") + target_only = True base_url = "https://api.dehashed.com/v2/search" diff --git a/bbot/modules/dnsbimi.py b/bbot/modules/dnsbimi.py index 152496c732..9463966a81 100644 --- a/bbot/modules/dnsbimi.py +++ b/bbot/modules/dnsbimi.py @@ -54,10 +54,11 @@ class dnsbimi(BaseModule): "author": "@colin-stubbs", "created_date": "2024-11-15", } + class Config(BaseModuleConfig): - emit_raw_dns_records: bool = Field(False, description='Emit RAW_DNS_RECORD events') - emit_urls: bool = Field(True, description='Emit URL_UNVERIFIED events') - selectors: str = Field('default,email,mail,bimi', description='CSV list of BIMI selectors to check') + emit_raw_dns_records: bool = Field(False, description="Emit RAW_DNS_RECORD events") + emit_urls: bool = Field(True, description="Emit URL_UNVERIFIED events") + selectors: str = Field("default,email,mail,bimi", description="CSV list of BIMI selectors to check") async def setup(self): self.emit_raw_dns_records = self.config.get("emit_raw_dns_records", False) diff --git a/bbot/modules/dnsbrute.py b/bbot/modules/dnsbrute.py index 00b194cacb..5bd0e55516 100644 --- a/bbot/modules/dnsbrute.py +++ b/bbot/modules/dnsbrute.py @@ -12,9 +12,14 @@ class dnsbrute(subdomain_enum): "author": "@TheTechromancer", "created_date": "2024-04-24", } + class Config(BaseModuleConfig): - wordlist: str = Field('https://raw.githubusercontent.com/danielmiessler/SecLists/master/Discovery/DNS/subdomains-top1million-5000.txt', description='Subdomain wordlist URL') - max_depth: int = Field(5, description='How many subdomains deep to brute force, i.e. 5.4.3.2.1.evilcorp.com') + wordlist: str = Field( + "https://raw.githubusercontent.com/danielmiessler/SecLists/master/Discovery/DNS/subdomains-top1million-5000.txt", + description="Subdomain wordlist URL", + ) + max_depth: int = Field(5, description="How many subdomains deep to brute force, i.e. 5.4.3.2.1.evilcorp.com") + deps_common = ["massdns"] reject_wildcards = "strict" dedup_strategy = "lowest_parent" diff --git a/bbot/modules/dnsbrute_mutations.py b/bbot/modules/dnsbrute_mutations.py index 4ef4fd29bb..7927af3c75 100644 --- a/bbot/modules/dnsbrute_mutations.py +++ b/bbot/modules/dnsbrute_mutations.py @@ -14,8 +14,10 @@ class dnsbrute_mutations(BaseModule): "author": "@TheTechromancer", "created_date": "2024-04-25", } + class Config(BaseModuleConfig): - max_mutations: int = Field(100, description='Maximum number of target-specific mutations to try per subdomain') + max_mutations: int = Field(100, description="Maximum number of target-specific mutations to try per subdomain") + deps_common = ["massdns"] _qsize = 10000 diff --git a/bbot/modules/dnscaa.py b/bbot/modules/dnscaa.py index 136c09339e..412ed9d6ff 100644 --- a/bbot/modules/dnscaa.py +++ b/bbot/modules/dnscaa.py @@ -31,11 +31,13 @@ class dnscaa(BaseModule): produced_events = ["DNS_NAME", "EMAIL_ADDRESS", "URL_UNVERIFIED"] flags = ["safe", "subdomain-enum", "email-enum", "passive"] meta = {"description": "Check for CAA records", "author": "@colin-stubbs", "created_date": "2024-05-26"} + class Config(BaseModuleConfig): - in_scope_only: bool = Field(True, description='Only check in-scope domains') - dns_names: bool = Field(True, description='emit DNS_NAME events') - emails: bool = Field(True, description='emit EMAIL_ADDRESS events') - urls: bool = Field(True, description='emit URL_UNVERIFIED events') + in_scope_only: bool = Field(True, description="Only check in-scope domains") + dns_names: bool = Field(True, description="emit DNS_NAME events") + emails: bool = Field(True, description="emit EMAIL_ADDRESS events") + urls: bool = Field(True, description="emit URL_UNVERIFIED events") + # accept DNS_NAMEs out to 2 hops if in_scope_only is False scope_distance_modifier = 2 diff --git a/bbot/modules/dnscommonsrv.py b/bbot/modules/dnscommonsrv.py index 203f82939b..855663a28a 100644 --- a/bbot/modules/dnscommonsrv.py +++ b/bbot/modules/dnscommonsrv.py @@ -13,7 +13,7 @@ class dnscommonsrv(subdomain_enum): deps_common = ["massdns"] class Config(BaseModuleConfig): - max_depth: int = Field(2, description='The maximum subdomain depth to brute-force SRV records') + max_depth: int = Field(2, description="The maximum subdomain depth to brute-force SRV records") async def setup(self): self.max_subdomain_depth = self.config.get("max_depth", 2) diff --git a/bbot/modules/dnstlsrpt.py b/bbot/modules/dnstlsrpt.py index 0dc5efce44..2513ebe71e 100644 --- a/bbot/modules/dnstlsrpt.py +++ b/bbot/modules/dnstlsrpt.py @@ -42,10 +42,11 @@ class dnstlsrpt(BaseModule): "author": "@colin-stubbs", "created_date": "2024-07-26", } + class Config(BaseModuleConfig): - emit_emails: bool = Field(True, description='Emit EMAIL_ADDRESS events') - emit_raw_dns_records: bool = Field(False, description='Emit RAW_DNS_RECORD events') - emit_urls: bool = Field(True, description='Emit URL_UNVERIFIED events') + emit_emails: bool = Field(True, description="Emit EMAIL_ADDRESS events") + emit_raw_dns_records: bool = Field(False, description="Emit RAW_DNS_RECORD events") + emit_urls: bool = Field(True, description="Emit URL_UNVERIFIED events") async def setup(self): self.emit_emails = self.config.get("emit_emails", True) diff --git a/bbot/modules/docker_pull.py b/bbot/modules/docker_pull.py index bbf193d593..839455f9f8 100644 --- a/bbot/modules/docker_pull.py +++ b/bbot/modules/docker_pull.py @@ -16,9 +16,13 @@ class docker_pull(BaseModule): "created_date": "2024-03-24", "author": "@domwhewell-sage", } + class Config(BaseModuleConfig): - all_tags: bool = Field(False, description='Download all tags from each registry (Default False)') - output_folder: str = Field('', description='Folder to download docker repositories to. If not specified, downloaded docker images will be deleted when the scan completes, to minimize disk usage.') + all_tags: bool = Field(False, description="Download all tags from each registry (Default False)") + output_folder: str = Field( + "", + description="Folder to download docker repositories to. If not specified, downloaded docker images will be deleted when the scan completes, to minimize disk usage.", + ) scope_distance_modifier = 2 diff --git a/bbot/modules/ffuf.py b/bbot/modules/ffuf.py index 97e7d46265..89078d7d85 100644 --- a/bbot/modules/ffuf.py +++ b/bbot/modules/ffuf.py @@ -15,13 +15,17 @@ class ffuf(BaseModule): meta = {"description": "A fast web fuzzer written in Go", "created_date": "2022-04-10", "author": "@liquidsec"} class Config(BaseModuleConfig): - wordlist: str = Field('https://raw.githubusercontent.com/danielmiessler/SecLists/master/Discovery/Web-Content/raft-small-directories.txt', description='Specify wordlist to use when finding directories') - lines: int = Field(5000, description='take only the first N lines from the wordlist when finding directories') - max_depth: int = Field(0, description='the maximum directory depth to attempt to solve') - extensions: str = Field('', description='Optionally include a list of extensions to extend the keyword with (comma separated)') - ignore_case: bool = Field(False, description='Only put lowercase words into the wordlist') - rate: int = Field(0, description='Rate of requests per second (default: 0)') - + wordlist: str = Field( + "https://raw.githubusercontent.com/danielmiessler/SecLists/master/Discovery/Web-Content/raft-small-directories.txt", + description="Specify wordlist to use when finding directories", + ) + lines: int = Field(5000, description="take only the first N lines from the wordlist when finding directories") + max_depth: int = Field(0, description="the maximum directory depth to attempt to solve") + extensions: str = Field( + "", description="Optionally include a list of extensions to extend the keyword with (comma separated)" + ) + ignore_case: bool = Field(False, description="Only put lowercase words into the wordlist") + rate: int = Field(0, description="Rate of requests per second (default: 0)") deps_common = ["ffuf"] diff --git a/bbot/modules/ffuf_shortnames.py b/bbot/modules/ffuf_shortnames.py index ceab878337..293fc1402d 100644 --- a/bbot/modules/ffuf_shortnames.py +++ b/bbot/modules/ffuf_shortnames.py @@ -19,17 +19,27 @@ class ffuf_shortnames(ffuf): } class Config(BaseModuleConfig): - wordlist_extensions: str = Field('', description='Specify wordlist to use when making extension lists') - max_depth: int = Field(1, description='the maximum directory depth to attempt to solve') - version: str = Field('2.0.0', description='ffuf version') - extensions: str = Field('', description='Optionally include a list of extensions to extend the keyword with (comma separated)') - ignore_redirects: bool = Field(True, description='Explicitly ignore redirects (301,302)') - find_common_prefixes: bool = Field(False, description='Attempt to automatically detect common prefixes and make additional ffuf runs against them') - find_delimiters: bool = Field(True, description='Attempt to detect common delimiters and make additional ffuf runs against them') - find_subwords: bool = Field(False, description='Attempt to detect subwords and make additional ffuf runs against them') - max_predictions: int = Field(250, description='The maximum number of predictions to generate per shortname prefix') - rate: int = Field(0, description='Rate of requests per second (default: 0)') - + wordlist_extensions: str = Field("", description="Specify wordlist to use when making extension lists") + max_depth: int = Field(1, description="the maximum directory depth to attempt to solve") + version: str = Field("2.0.0", description="ffuf version") + extensions: str = Field( + "", description="Optionally include a list of extensions to extend the keyword with (comma separated)" + ) + ignore_redirects: bool = Field(True, description="Explicitly ignore redirects (301,302)") + find_common_prefixes: bool = Field( + False, + description="Attempt to automatically detect common prefixes and make additional ffuf runs against them", + ) + find_delimiters: bool = Field( + True, description="Attempt to detect common delimiters and make additional ffuf runs against them" + ) + find_subwords: bool = Field( + False, description="Attempt to detect subwords and make additional ffuf runs against them" + ) + max_predictions: int = Field( + 250, description="The maximum number of predictions to generate per shortname prefix" + ) + rate: int = Field(0, description="Rate of requests per second (default: 0)") deps_pip = ["numpy"] deps_common = ["ffuf"] diff --git a/bbot/modules/filedownload.py b/bbot/modules/filedownload.py index ac15e0b576..761193f14d 100644 --- a/bbot/modules/filedownload.py +++ b/bbot/modules/filedownload.py @@ -22,10 +22,78 @@ class filedownload(BaseModule): "created_date": "2023-10-11", "author": "@TheTechromancer", } + class Config(BaseModuleConfig): - extensions: list[str] = Field(['bak', 'bash', 'bashrc', 'cfg', 'conf', 'crt', 'csv', 'db', 'dll', 'doc', 'docx', 'exe', 'ica', 'indd', 'ini', 'jar', 'json', 'key', 'log', 'markdown', 'md', 'msi', 'odg', 'odp', 'ods', 'odt', 'pdf', 'pem', 'pps', 'ppsx', 'ppt', 'pptx', 'ps1', 'pub', 'raw', 'rdp', 'rsa', 'sh', 'sql', 'sqlite', 'swp', 'sxw', 'tar.gz', 'tgz', 'tar', 'txt', 'vbs', 'war', 'wpd', 'xls', 'xlsx', 'xml', 'yaml', 'yml', 'zip', 'lzma', 'rar', '7z', 'xz', 'bz2'], description='File extensions to download') - max_filesize: str = Field('10MB', description='Cancel download if filesize is greater than this size') - output_folder: str = Field('', description='Folder to download files to. If not specified, downloaded files will be deleted when the scan completes, to minimize disk usage.') + extensions: list[str] = Field( + [ + "bak", + "bash", + "bashrc", + "cfg", + "conf", + "crt", + "csv", + "db", + "dll", + "doc", + "docx", + "exe", + "ica", + "indd", + "ini", + "jar", + "json", + "key", + "log", + "markdown", + "md", + "msi", + "odg", + "odp", + "ods", + "odt", + "pdf", + "pem", + "pps", + "ppsx", + "ppt", + "pptx", + "ps1", + "pub", + "raw", + "rdp", + "rsa", + "sh", + "sql", + "sqlite", + "swp", + "sxw", + "tar.gz", + "tgz", + "tar", + "txt", + "vbs", + "war", + "wpd", + "xls", + "xlsx", + "xml", + "yaml", + "yml", + "zip", + "lzma", + "rar", + "7z", + "xz", + "bz2", + ], + description="File extensions to download", + ) + max_filesize: str = Field("10MB", description="Cancel download if filesize is greater than this size") + output_folder: str = Field( + "", + description="Folder to download files to. If not specified, downloaded files will be deleted when the scan completes, to minimize disk usage.", + ) scope_distance_modifier = 3 diff --git a/bbot/modules/fullhunt.py b/bbot/modules/fullhunt.py index 90839b195c..ba15bcf5c1 100644 --- a/bbot/modules/fullhunt.py +++ b/bbot/modules/fullhunt.py @@ -13,8 +13,9 @@ class fullhunt(subdomain_enum_apikey): "author": "@TheTechromancer", "auth_required": True, } + class Config(BaseModuleConfig): - api_key: str = Field('', description='FullHunt API Key') + api_key: str = Field("", description="FullHunt API Key") base_url = "https://fullhunt.io/api/v1" diff --git a/bbot/modules/git_clone.py b/bbot/modules/git_clone.py index 17a52c05ef..9f526d61e7 100644 --- a/bbot/modules/git_clone.py +++ b/bbot/modules/git_clone.py @@ -14,9 +14,13 @@ class git_clone(github): "created_date": "2024-03-08", "author": "@domwhewell-sage", } + class Config(BaseModuleConfig): - api_key: str = Field('', description='Github token') - output_folder: str = Field('', description='Folder to clone repositories to. If not specified, cloned repositories will be deleted when the scan completes, to minimize disk usage.') + api_key: str = Field("", description="Github token") + output_folder: str = Field( + "", + description="Folder to clone repositories to. If not specified, cloned repositories will be deleted when the scan completes, to minimize disk usage.", + ) deps_apt = ["git"] diff --git a/bbot/modules/gitdumper.py b/bbot/modules/gitdumper.py index ab837cc174..c611bdc050 100644 --- a/bbot/modules/gitdumper.py +++ b/bbot/modules/gitdumper.py @@ -15,10 +15,16 @@ class gitdumper(BaseModule): "created_date": "2025-02-11", "author": "@domwhewell-sage", } + class Config(BaseModuleConfig): - output_folder: str = Field('', description='Folder to download repositories to. If not specified, downloaded repositories will be deleted when the scan completes, to minimize disk usage.') - fuzz_tags: bool = Field(False, description='Fuzz for common git tag names (v0.0.1, 0.0.2, etc.) up to the max_semanic_version') - max_semanic_version: int = Field(10, description='Maximum version number to fuzz for (default < v10.10.10)') + output_folder: str = Field( + "", + description="Folder to download repositories to. If not specified, downloaded repositories will be deleted when the scan completes, to minimize disk usage.", + ) + fuzz_tags: bool = Field( + False, description="Fuzz for common git tag names (v0.0.1, 0.0.2, etc.) up to the max_semanic_version" + ) + max_semanic_version: int = Field(10, description="Maximum version number to fuzz for (default < v10.10.10)") scope_distance_modifier = 2 diff --git a/bbot/modules/github_codesearch.py b/bbot/modules/github_codesearch.py index 3fb15c7678..fbc4a253ef 100644 --- a/bbot/modules/github_codesearch.py +++ b/bbot/modules/github_codesearch.py @@ -14,9 +14,10 @@ class github_codesearch(github, subdomain_enum): "author": "@domwhewell-sage", "auth_required": True, } + class Config(BaseModuleConfig): - api_key: str = Field('', description='Github token') - limit: int = Field(100, description='Limit code search to this many results') + api_key: str = Field("", description="Github token") + limit: int = Field(100, description="Limit code search to this many results") github_raw_url = "https://raw.githubusercontent.com/" diff --git a/bbot/modules/github_org.py b/bbot/modules/github_org.py index 2e96fdad1e..bd84a55a77 100644 --- a/bbot/modules/github_org.py +++ b/bbot/modules/github_org.py @@ -12,9 +12,10 @@ class github_org(github): "created_date": "2023-12-14", "author": "@domwhewell-sage", } + class Config(BaseModuleConfig): - api_key: str = Field('', description='Github token') - include_members: bool = Field(True, description='Enumerate organization members') + api_key: str = Field("", description="Github token") + include_members: bool = Field(True, description="Enumerate organization members") include_member_repos: bool = Field(False, description="Also enumerate organization members' repositories") scope_distance_modifier = 2 diff --git a/bbot/modules/github_usersearch.py b/bbot/modules/github_usersearch.py index 47693c2764..9c6aac2370 100644 --- a/bbot/modules/github_usersearch.py +++ b/bbot/modules/github_usersearch.py @@ -14,8 +14,9 @@ class github_usersearch(github, subdomain_enum): "author": "@domwhewell-sage", "auth_required": True, } + class Config(BaseModuleConfig): - api_key: str = Field('', description='Github token') + api_key: str = Field("", description="Github token") async def handle_event(self, event): self.verbose("Searching for users with emails matching in scope domains") diff --git a/bbot/modules/github_workflows.py b/bbot/modules/github_workflows.py index 12931ffdd4..82e7344abe 100644 --- a/bbot/modules/github_workflows.py +++ b/bbot/modules/github_workflows.py @@ -17,10 +17,11 @@ class github_workflows(github): "author": "@domwhewell-sage", "auth_required": True, } + class Config(BaseModuleConfig): - api_key: str = Field('', description='Github token') - num_logs: int = Field(1, description='For each workflow fetch the last N successful runs logs (max 100)') - output_folder: str = Field('', description='Folder to download workflow logs and artifacts to') + api_key: str = Field("", description="Github token") + num_logs: int = Field(1, description="For each workflow fetch the last N successful runs logs (max 100)") + output_folder: str = Field("", description="Folder to download workflow logs and artifacts to") scope_distance_modifier = 2 diff --git a/bbot/modules/gitlab_com.py b/bbot/modules/gitlab_com.py index a08f645cc7..39ff302d41 100644 --- a/bbot/modules/gitlab_com.py +++ b/bbot/modules/gitlab_com.py @@ -16,7 +16,7 @@ class gitlab_com(GitLabBaseModule): } class Config(BaseModuleConfig): - api_key: str = Field('', description='GitLab access token (for gitlab.com/org only)') + api_key: str = Field("", description="GitLab access token (for gitlab.com/org only)") # This is needed because we are consuming SOCIAL events, which aren't in scope scope_distance_modifier = 2 diff --git a/bbot/modules/gitlab_onprem.py b/bbot/modules/gitlab_onprem.py index c76a0c91a2..03c51630ab 100644 --- a/bbot/modules/gitlab_onprem.py +++ b/bbot/modules/gitlab_onprem.py @@ -21,7 +21,7 @@ class gitlab_onprem(GitLabBaseModule): # Optional GitLab access token (only required for gitlab.com, but still # supported for on-prem installations that expose private projects). class Config(BaseModuleConfig): - api_key: str = Field('', description='GitLab access token (for self-hosted instances only)') + api_key: str = Field("", description="GitLab access token (for self-hosted instances only)") # Allow accepting events slightly beyond configured max distance so we can # discover repos on neighbouring infrastructure. diff --git a/bbot/modules/gowitness.py b/bbot/modules/gowitness.py index 46ef779b4e..f9eaa5337c 100644 --- a/bbot/modules/gowitness.py +++ b/bbot/modules/gowitness.py @@ -18,16 +18,20 @@ class gowitness(BaseModule): produced_events = ["WEBSCREENSHOT", "URL", "URL_UNVERIFIED", "TECHNOLOGY"] flags = ["safe", "active", "web-screenshots"] meta = {"description": "Take screenshots of webpages", "created_date": "2022-07-08", "author": "@TheTechromancer"} + class Config(BaseModuleConfig): - version: str = Field('3.1.1', description='Gowitness version') - threads: int = Field(0, description='How many gowitness threads to spawn (default is number of CPUs x 2)') - timeout: int = Field(10, description='Preflight check timeout') - resolution_x: int = Field(1440, description='Screenshot resolution x') - resolution_y: int = Field(900, description='Screenshot resolution y') - output_path: str = Field('', description='Where to save screenshots') - social: bool = Field(False, description='Whether to screenshot social media webpages') - idle_timeout: int = Field(1800, description='Skip the current gowitness batch if it stalls for longer than this many seconds') - chrome_path: str = Field('', description='Path to chrome executable') + version: str = Field("3.1.1", description="Gowitness version") + threads: int = Field(0, description="How many gowitness threads to spawn (default is number of CPUs x 2)") + timeout: int = Field(10, description="Preflight check timeout") + resolution_x: int = Field(1440, description="Screenshot resolution x") + resolution_y: int = Field(900, description="Screenshot resolution y") + output_path: str = Field("", description="Where to save screenshots") + social: bool = Field(False, description="Whether to screenshot social media webpages") + idle_timeout: int = Field( + 1800, description="Skip the current gowitness batch if it stalls for longer than this many seconds" + ) + chrome_path: str = Field("", description="Path to chrome executable") + deps_common = ["chromium"] deps_pip = ["aiosqlite"] deps_ansible = [ diff --git a/bbot/modules/graphql_introspection.py b/bbot/modules/graphql_introspection.py index 246e103abc..1f9e6e1a87 100644 --- a/bbot/modules/graphql_introspection.py +++ b/bbot/modules/graphql_introspection.py @@ -14,9 +14,12 @@ class graphql_introspection(BaseModule): "created_date": "2025-07-01", "author": "@mukesh-dream11", } + class Config(BaseModuleConfig): - graphql_endpoint_urls: list[str] = Field(['/', '/graphql', '/v1/graphql'], description='List of GraphQL endpoint to suffix to the target URL') - output_folder: str = Field('', description='Folder to save the GraphQL schemas to') + graphql_endpoint_urls: list[str] = Field( + ["/", "/graphql", "/v1/graphql"], description="List of GraphQL endpoint to suffix to the target URL" + ) + output_folder: str = Field("", description="Folder to save the GraphQL schemas to") async def setup(self): output_folder = self.config.get("output_folder", "") diff --git a/bbot/modules/httpx.py b/bbot/modules/httpx.py index a810d5edab..6267d5e38d 100644 --- a/bbot/modules/httpx.py +++ b/bbot/modules/httpx.py @@ -23,12 +23,13 @@ class httpx(BaseModule): } class Config(BaseModuleConfig): - threads: int = Field(50, description='Number of httpx threads to use') - in_scope_only: bool = Field(True, description='Only visit web reparents that are in scope.') - version: str = Field('1.2.5', description='httpx version') - max_response_size: int = Field(5242880, description='Max response size in bytes') - store_responses: bool = Field(False, description='Save raw HTTP responses to scan folder') - probe_all_ips: bool = Field(False, description='Probe all the ips associated with same host') + threads: int = Field(50, description="Number of httpx threads to use") + in_scope_only: bool = Field(True, description="Only visit web reparents that are in scope.") + version: str = Field("1.2.5", description="httpx version") + max_response_size: int = Field(5242880, description="Max response size in bytes") + store_responses: bool = Field(False, description="Save raw HTTP responses to scan folder") + probe_all_ips: bool = Field(False, description="Probe all the ips associated with same host") + deps_ansible = [ { "name": "Download httpx", diff --git a/bbot/modules/hunterio.py b/bbot/modules/hunterio.py index 832c3aa792..07c6b93cbb 100644 --- a/bbot/modules/hunterio.py +++ b/bbot/modules/hunterio.py @@ -13,8 +13,9 @@ class hunterio(subdomain_enum_apikey): "author": "@TheTechromancer", "auth_required": True, } + class Config(BaseModuleConfig): - api_key: str = Field('', description='Hunter.IO API key') + api_key: str = Field("", description="Hunter.IO API key") base_url = "https://api.hunter.io/v2" ping_url = f"{base_url}/account?api_key={{api_key}}" diff --git a/bbot/modules/iis_shortnames.py b/bbot/modules/iis_shortnames.py index 5be0e95793..b0b915e58a 100644 --- a/bbot/modules/iis_shortnames.py +++ b/bbot/modules/iis_shortnames.py @@ -24,10 +24,16 @@ class iis_shortnames(BaseModule): "created_date": "2022-04-15", "author": "@liquidsec", } + class Config(BaseModuleConfig): - detect_only: bool = Field(True, description='Only detect the vulnerability and do not run the shortname scanner') - max_node_count: int = Field(50, description='Limit how many nodes to attempt to resolve on any given recursion branch') + detect_only: bool = Field( + True, description="Only detect the vulnerability and do not run the shortname scanner" + ) + max_node_count: int = Field( + 50, description="Limit how many nodes to attempt to resolve on any given recursion branch" + ) speculate_magic_urls: bool = Field(True, description="Attempt to discover iis 'magic' special folders") + in_scope_only = True _module_threads = 8 diff --git a/bbot/modules/internal/excavate.py b/bbot/modules/internal/excavate.py index 4c84b1b870..b59a70e9e6 100644 --- a/bbot/modules/internal/excavate.py +++ b/bbot/modules/internal/excavate.py @@ -338,9 +338,14 @@ class excavateTestRule(ExcavateRule): } class Config(BaseModuleConfig): - yara_max_match_data: int = Field(2000, description='Sets the maximum amount of text that can extracted from a YARA regex') - custom_yara_rules: str = Field('', description='Include custom Yara rules') - speculate_params: bool = Field(False, description='Enable speculative parameter extraction from JSON and XML content') + yara_max_match_data: int = Field( + 2000, description="Sets the maximum amount of text that can extracted from a YARA regex" + ) + custom_yara_rules: str = Field("", description="Include custom Yara rules") + speculate_params: bool = Field( + False, description="Enable speculative parameter extraction from JSON and XML content" + ) + scope_distance_modifier = None accept_dupes = False diff --git a/bbot/modules/internal/speculate.py b/bbot/modules/internal/speculate.py index 2eae9124e5..ded0b834ee 100644 --- a/bbot/modules/internal/speculate.py +++ b/bbot/modules/internal/speculate.py @@ -35,9 +35,14 @@ class speculate(BaseInternalModule): } class Config(BaseModuleConfig): - ip_range_max_hosts: int = Field(65536, description='Max number of hosts an IP_RANGE can contain to allow conversion into IP_ADDRESS events') - ports: str = Field('80,443', description='The set of ports to speculate on') - essential_only: bool = Field(False, description='Only enable essential speculate features (no extra discovery)') + ip_range_max_hosts: int = Field( + 65536, description="Max number of hosts an IP_RANGE can contain to allow conversion into IP_ADDRESS events" + ) + ports: str = Field("80,443", description="The set of ports to speculate on") + essential_only: bool = Field( + False, description="Only enable essential speculate features (no extra discovery)" + ) + scope_distance_modifier = 1 _priority = 4 diff --git a/bbot/modules/ip2location.py b/bbot/modules/ip2location.py index 8abf7fcc69..08676fcfef 100644 --- a/bbot/modules/ip2location.py +++ b/bbot/modules/ip2location.py @@ -17,9 +17,14 @@ class IP2Location(BaseModule): "author": "@TheTechromancer", "auth_required": True, } + class Config(BaseModuleConfig): - api_key: str = Field('', description='IP2location.io API Key') - lang: str = Field('', description='Translation information(ISO639-1). The translation is only applicable for continent, country, region and city name.') + api_key: str = Field("", description="IP2location.io API Key") + lang: str = Field( + "", + description="Translation information(ISO639-1). The translation is only applicable for continent, country, region and city name.", + ) + scope_distance_modifier = 1 _priority = 2 suppress_dupes = False diff --git a/bbot/modules/ipneighbor.py b/bbot/modules/ipneighbor.py index badb48cd48..a69a602b03 100644 --- a/bbot/modules/ipneighbor.py +++ b/bbot/modules/ipneighbor.py @@ -14,8 +14,10 @@ class ipneighbor(BaseModule): "created_date": "2022-06-08", "author": "@TheTechromancer", } + class Config(BaseModuleConfig): - num_bits: int = Field(4, description='Netmask size (in CIDR notation) to check. Default is 4 bits (16 hosts)') + num_bits: int = Field(4, description="Netmask size (in CIDR notation) to check. Default is 4 bits (16 hosts)") + scope_distance_modifier = 1 async def setup(self): diff --git a/bbot/modules/ipstack.py b/bbot/modules/ipstack.py index f298c12b55..13e41b28b5 100644 --- a/bbot/modules/ipstack.py +++ b/bbot/modules/ipstack.py @@ -18,8 +18,10 @@ class Ipstack(BaseModule): "author": "@tycoonslive", "auth_required": True, } + class Config(BaseModuleConfig): - api_key: str = Field('', description='IPStack GeoIP API Key') + api_key: str = Field("", description="IPStack GeoIP API Key") + scope_distance_modifier = 1 _priority = 2 suppress_dupes = False diff --git a/bbot/modules/jadx.py b/bbot/modules/jadx.py index 44ae831e49..9da80079aa 100644 --- a/bbot/modules/jadx.py +++ b/bbot/modules/jadx.py @@ -14,8 +14,10 @@ class jadx(BaseModule): "created_date": "2024-11-04", "author": "@domwhewell-sage", } + class Config(BaseModuleConfig): threads: int = Field(4, description="Maximum jadx threads for extracting apk's, default: 4") + deps_common = ["java"] deps_ansible = [ { diff --git a/bbot/modules/kreuzberg.py b/bbot/modules/kreuzberg.py index a71ce524b5..d782958415 100644 --- a/bbot/modules/kreuzberg.py +++ b/bbot/modules/kreuzberg.py @@ -15,8 +15,58 @@ class kreuzberg(BaseModule): "created_date": "2024-06-03", "author": "@domwhewell-sage", } + class Config(BaseModuleConfig): - extensions: list[str] = Field(['bak', 'bash', 'bashrc', 'conf', 'cfg', 'crt', 'csv', 'db', 'sqlite', 'doc', 'docx', 'ica', 'indd', 'ini', 'json', 'key', 'pub', 'log', 'markdown', 'md', 'odg', 'odp', 'ods', 'odt', 'pdf', 'pem', 'pps', 'ppsx', 'ppt', 'pptx', 'ps1', 'rdp', 'rsa', 'sh', 'sql', 'swp', 'sxw', 'txt', 'vbs', 'wpd', 'xls', 'xlsx', 'xml', 'yml', 'yaml'], description='File extensions to parse') + extensions: list[str] = Field( + [ + "bak", + "bash", + "bashrc", + "conf", + "cfg", + "crt", + "csv", + "db", + "sqlite", + "doc", + "docx", + "ica", + "indd", + "ini", + "json", + "key", + "pub", + "log", + "markdown", + "md", + "odg", + "odp", + "ods", + "odt", + "pdf", + "pem", + "pps", + "ppsx", + "ppt", + "pptx", + "ps1", + "rdp", + "rsa", + "sh", + "sql", + "swp", + "sxw", + "txt", + "vbs", + "wpd", + "xls", + "xlsx", + "xml", + "yml", + "yaml", + ], + description="File extensions to parse", + ) deps_pip = ["kreuzberg>=4.3,<4.5", "pypdfium2~=5.0"] scope_distance_modifier = 1 diff --git a/bbot/modules/leakix.py b/bbot/modules/leakix.py index f4f44bbd09..5d98f83435 100644 --- a/bbot/modules/leakix.py +++ b/bbot/modules/leakix.py @@ -7,8 +7,10 @@ class leakix(subdomain_enum_apikey): watched_events = ["DNS_NAME"] produced_events = ["DNS_NAME"] flags = ["safe", "subdomain-enum", "passive"] + class Config(BaseModuleConfig): - api_key: str = Field('', description='LeakIX API Key') + api_key: str = Field("", description="LeakIX API Key") + # NOTE: API key is not required (but having one will get you more results) meta = { "description": "Query leakix.net for subdomains", diff --git a/bbot/modules/legba.py b/bbot/modules/legba.py index f76a73b038..8ad9eebd7e 100644 --- a/bbot/modules/legba.py +++ b/bbot/modules/legba.py @@ -31,17 +31,37 @@ class legba(BaseModule): scope_distance_modifier = None class Config(BaseModuleConfig): - ssh_wordlist: str = Field('https://raw.githubusercontent.com/danielmiessler/SecLists/refs/heads/master/Passwords/Default-Credentials/ssh-betterdefaultpasslist.txt', description='Wordlist URL for SSH combined username:password wordlist, newline separated') - ftp_wordlist: str = Field('https://raw.githubusercontent.com/danielmiessler/SecLists/refs/heads/master/Passwords/Default-Credentials/ftp-betterdefaultpasslist.txt', description='Wordlist URL for FTP combined username:password wordlist, newline separated') - telnet_wordlist: str = Field('https://raw.githubusercontent.com/danielmiessler/SecLists/refs/heads/master/Passwords/Default-Credentials/telnet-betterdefaultpasslist.txt', description='Wordlist URL for TELNET combined username:password wordlist, newline separated') - vnc_wordlist: str = Field('https://raw.githubusercontent.com/danielmiessler/SecLists/refs/heads/master/Passwords/Default-Credentials/vnc-betterdefaultpasslist.txt', description='Wordlist URL for VNC password wordlist, newline separated') - mssql_wordlist: str = Field('https://raw.githubusercontent.com/danielmiessler/SecLists/refs/heads/master/Passwords/Default-Credentials/mssql-betterdefaultpasslist.txt', description='Wordlist URL for MSSQL combined username:password wordlist, newline separated') - mysql_wordlist: str = Field('https://raw.githubusercontent.com/danielmiessler/SecLists/refs/heads/master/Passwords/Default-Credentials/mysql-betterdefaultpasslist.txt', description='Wordlist URL for MySQL combined username:password wordlist, newline separated') - postgresql_wordlist: str = Field('https://raw.githubusercontent.com/danielmiessler/SecLists/refs/heads/master/Passwords/Default-Credentials/postgres-betterdefaultpasslist.txt', description='Wordlist URL for PostgreSQL combined username:password wordlist, newline separated') - concurrency: int = Field(3, description='Number of concurrent workers, gets overridden for SSH') - rate_limit: int = Field(3, description='Limit the number of requests per second, gets overridden for SSH') - version: str = Field('1.1.1', description='legba version') - + ssh_wordlist: str = Field( + "https://raw.githubusercontent.com/danielmiessler/SecLists/refs/heads/master/Passwords/Default-Credentials/ssh-betterdefaultpasslist.txt", + description="Wordlist URL for SSH combined username:password wordlist, newline separated", + ) + ftp_wordlist: str = Field( + "https://raw.githubusercontent.com/danielmiessler/SecLists/refs/heads/master/Passwords/Default-Credentials/ftp-betterdefaultpasslist.txt", + description="Wordlist URL for FTP combined username:password wordlist, newline separated", + ) + telnet_wordlist: str = Field( + "https://raw.githubusercontent.com/danielmiessler/SecLists/refs/heads/master/Passwords/Default-Credentials/telnet-betterdefaultpasslist.txt", + description="Wordlist URL for TELNET combined username:password wordlist, newline separated", + ) + vnc_wordlist: str = Field( + "https://raw.githubusercontent.com/danielmiessler/SecLists/refs/heads/master/Passwords/Default-Credentials/vnc-betterdefaultpasslist.txt", + description="Wordlist URL for VNC password wordlist, newline separated", + ) + mssql_wordlist: str = Field( + "https://raw.githubusercontent.com/danielmiessler/SecLists/refs/heads/master/Passwords/Default-Credentials/mssql-betterdefaultpasslist.txt", + description="Wordlist URL for MSSQL combined username:password wordlist, newline separated", + ) + mysql_wordlist: str = Field( + "https://raw.githubusercontent.com/danielmiessler/SecLists/refs/heads/master/Passwords/Default-Credentials/mysql-betterdefaultpasslist.txt", + description="Wordlist URL for MySQL combined username:password wordlist, newline separated", + ) + postgresql_wordlist: str = Field( + "https://raw.githubusercontent.com/danielmiessler/SecLists/refs/heads/master/Passwords/Default-Credentials/postgres-betterdefaultpasslist.txt", + description="Wordlist URL for PostgreSQL combined username:password wordlist, newline separated", + ) + concurrency: int = Field(3, description="Number of concurrent workers, gets overridden for SSH") + rate_limit: int = Field(3, description="Limit the number of requests per second, gets overridden for SSH") + version: str = Field("1.1.1", description="legba version") deps_ansible = [ { diff --git a/bbot/modules/lightfuzz/lightfuzz.py b/bbot/modules/lightfuzz/lightfuzz.py index ba9989dd24..b8cc6b1f75 100644 --- a/bbot/modules/lightfuzz/lightfuzz.py +++ b/bbot/modules/lightfuzz/lightfuzz.py @@ -12,12 +12,25 @@ class lightfuzz(BaseModule): flags = ["active", "loud", "web-heavy", "invasive"] class Config(BaseModuleConfig): - force_common_headers: bool = Field(False, description='Force emit commonly exploitable parameters that may be difficult to detect') - enabled_submodules: list[str] = Field(['sqli', 'cmdi', 'xss', 'path', 'ssti', 'crypto', 'serial', 'esi', 'ssrf'], description='A list of submodules to enable. Empty list enabled all modules.') - disable_post: bool = Field(False, description='Disable processing of POST parameters, avoiding form submissions.') - try_post_as_get: bool = Field(False, description='For each POSTPARAM, also fuzz it as a GETPARAM (in addition to normal POST fuzzing).') - try_get_as_post: bool = Field(False, description='For each GETPARAM, also fuzz it as a POSTPARAM (in addition to normal GET fuzzing).') - avoid_wafs: bool = Field(True, description='Avoid running against confirmed WAFs, which are likely to block lightfuzz requests') + force_common_headers: bool = Field( + False, description="Force emit commonly exploitable parameters that may be difficult to detect" + ) + enabled_submodules: list[str] = Field( + ["sqli", "cmdi", "xss", "path", "ssti", "crypto", "serial", "esi", "ssrf"], + description="A list of submodules to enable. Empty list enabled all modules.", + ) + disable_post: bool = Field( + False, description="Disable processing of POST parameters, avoiding form submissions." + ) + try_post_as_get: bool = Field( + False, description="For each POSTPARAM, also fuzz it as a GETPARAM (in addition to normal POST fuzzing)." + ) + try_get_as_post: bool = Field( + False, description="For each GETPARAM, also fuzz it as a POSTPARAM (in addition to normal GET fuzzing)." + ) + avoid_wafs: bool = Field( + True, description="Avoid running against confirmed WAFs, which are likely to block lightfuzz requests" + ) meta = { "description": "Find Web Parameters and Lightly Fuzz them using a heuristic based scanner", diff --git a/bbot/modules/medusa.py b/bbot/modules/medusa.py index 5184eef871..b0315d7869 100644 --- a/bbot/modules/medusa.py +++ b/bbot/modules/medusa.py @@ -17,12 +17,20 @@ class medusa(BaseModule): scope_distance_modifier = None class Config(BaseModuleConfig): - snmp_wordlist: str = Field('https://raw.githubusercontent.com/danielmiessler/SecLists/refs/heads/master/Discovery/SNMP/common-snmp-community-strings.txt', description='Wordlist url for SNMP community strings, newline separated (default https://raw.githubusercontent.com/danielmiessler/SecLists/refs/heads/master/Discovery/SNMP/snmp.txt)') - snmp_versions: list[str] = Field(['1', '2C'], description="List of SNMP versions to attempt against the SNMP server (default ['1', '2C'])") - wait_microseconds: int = Field(200, description='Wait time after every SNMP request in microseconds (default 200)') - timeout_s: int = Field(5, description='Wait time for the SNMP response(s) once at the end of all attempts (default 5)') - threads: int = Field(5, description='Number of communities to be tested concurrently (default 5)') - + snmp_wordlist: str = Field( + "https://raw.githubusercontent.com/danielmiessler/SecLists/refs/heads/master/Discovery/SNMP/common-snmp-community-strings.txt", + description="Wordlist url for SNMP community strings, newline separated (default https://raw.githubusercontent.com/danielmiessler/SecLists/refs/heads/master/Discovery/SNMP/snmp.txt)", + ) + snmp_versions: list[str] = Field( + ["1", "2C"], description="List of SNMP versions to attempt against the SNMP server (default ['1', '2C'])" + ) + wait_microseconds: int = Field( + 200, description="Wait time after every SNMP request in microseconds (default 200)" + ) + timeout_s: int = Field( + 5, description="Wait time for the SNMP response(s) once at the end of all attempts (default 5)" + ) + threads: int = Field(5, description="Number of communities to be tested concurrently (default 5)") deps_ansible = [ { diff --git a/bbot/modules/ntlm.py b/bbot/modules/ntlm.py index 19d29af238..bd4a77ffa1 100644 --- a/bbot/modules/ntlm.py +++ b/bbot/modules/ntlm.py @@ -76,8 +76,9 @@ class ntlm(BaseModule): "created_date": "2022-07-25", "author": "@liquidsec", } + class Config(BaseModuleConfig): - try_all: bool = Field(False, description='Try every NTLM endpoint') + try_all: bool = Field(False, description="Try every NTLM endpoint") in_scope_only = True diff --git a/bbot/modules/nuclei.py b/bbot/modules/nuclei.py index 579f2dbf84..677889adec 100644 --- a/bbot/modules/nuclei.py +++ b/bbot/modules/nuclei.py @@ -18,12 +18,12 @@ class nuclei(BaseModule): } class Config(BaseModuleConfig): - version: str = Field('3.8.0', description='nuclei version') - tags: str = Field('', description='execute a subset of templates that contain the provided tags') - templates: str = Field('', description='template or template directory paths to include in the scan') - severity: str = Field('', description='Filter based on severity field available in the template.') - ratelimit: int = Field(150, description='maximum number of requests to send per second (default 150)') - concurrency: int = Field(25, description='maximum number of templates to be executed in parallel (default 25)') + version: str = Field("3.8.0", description="nuclei version") + tags: str = Field("", description="execute a subset of templates that contain the provided tags") + templates: str = Field("", description="template or template directory paths to include in the scan") + severity: str = Field("", description="Filter based on severity field available in the template.") + ratelimit: int = Field(150, description="maximum number of requests to send per second (default 150)") + concurrency: int = Field(25, description="maximum number of templates to be executed in parallel (default 25)") mode: Literal["manual", "technology", "severe", "budget"] = Field( "manual", description=( @@ -34,13 +34,14 @@ class Config(BaseModuleConfig): "Budget: Limit Nuclei to a specified number of HTTP requests" ), ) - etags: str = Field('', description='tags to exclude from the scan') - budget: int = Field(1, description='Used in budget mode to set the number of allowed requests per host') + etags: str = Field("", description="tags to exclude from the scan") + budget: int = Field(1, description="Used in budget mode to set the number of allowed requests per host") silent: bool = Field(False, description="Don't display nuclei's banner or status messages") directory_only: bool = Field(True, description="Filter out 'file' URL event (default True)") - retries: int = Field(0, description='number of times to retry a failed request (default 0)') - batch_size: int = Field(200, description='Number of targets to send to Nuclei per batch (default 200)') - module_timeout: int = Field(21600, description='Max time in seconds to spend handling each batch of events') + retries: int = Field(0, description="number of times to retry a failed request (default 0)") + batch_size: int = Field(200, description="Number of targets to send to Nuclei per batch (default 200)") + module_timeout: int = Field(21600, description="Max time in seconds to spend handling each batch of events") + deps_ansible = [ { "name": "Download nuclei", diff --git a/bbot/modules/oauth.py b/bbot/modules/oauth.py index 0843f25353..ac886d58ab 100644 --- a/bbot/modules/oauth.py +++ b/bbot/modules/oauth.py @@ -14,8 +14,9 @@ class OAUTH(BaseModule): "created_date": "2023-07-12", "author": "@TheTechromancer", } + class Config(BaseModuleConfig): - try_all: bool = Field(False, description='Check for OAUTH/IODC on every subdomain and URL.') + try_all: bool = Field(False, description="Check for OAUTH/IODC on every subdomain and URL.") in_scope_only = False scope_distance_modifier = 1 diff --git a/bbot/modules/otx.py b/bbot/modules/otx.py index 40100a81d3..8e2f91ea69 100644 --- a/bbot/modules/otx.py +++ b/bbot/modules/otx.py @@ -13,8 +13,9 @@ class otx(subdomain_enum_apikey): "author": "@TheTechromancer", "auth_required": True, } + class Config(BaseModuleConfig): - api_key: str = Field('', description='OTX API key') + api_key: str = Field("", description="OTX API key") base_url = "https://otx.alienvault.com" diff --git a/bbot/modules/output/asset_inventory.py b/bbot/modules/output/asset_inventory.py index 22d91e3542..380072b5de 100644 --- a/bbot/modules/output/asset_inventory.py +++ b/bbot/modules/output/asset_inventory.py @@ -39,11 +39,18 @@ class asset_inventory(CSV): "created_date": "2022-09-30", "author": "@liquidsec", } + class Config(BaseModuleConfig): - output_file: str = Field('', description='Set a custom output file') - use_previous: bool = Field(False, description='Emit previous asset inventory as new events (use in conjunction with -n )') - recheck: bool = Field(False, description="When use_previous=True, don't retain past details like open ports or findings. Instead, allow them to be rediscovered by the new scan") - summary_netmask: int = Field(16, description='Subnet mask to use when summarizing IP addresses at end of scan') + output_file: str = Field("", description="Set a custom output file") + use_previous: bool = Field( + False, + description="Emit previous asset inventory as new events (use in conjunction with -n )", + ) + recheck: bool = Field( + False, + description="When use_previous=True, don't retain past details like open ports or findings. Instead, allow them to be rediscovered by the new scan", + ) + summary_netmask: int = Field(16, description="Subnet mask to use when summarizing IP addresses at end of scan") header_row = [ "Host", diff --git a/bbot/modules/output/csv.py b/bbot/modules/output/csv.py index 677e04ebd9..237d2360bf 100644 --- a/bbot/modules/output/csv.py +++ b/bbot/modules/output/csv.py @@ -9,8 +9,9 @@ class CSV(BaseOutputModule): watched_events = ["*"] meta = {"description": "Output to CSV", "created_date": "2022-04-07", "author": "@TheTechromancer"} + class Config(BaseModuleConfig): - output_file: str = Field('', description='Output to CSV file') + output_file: str = Field("", description="Output to CSV file") header_row = [ "Event type", diff --git a/bbot/modules/output/discord.py b/bbot/modules/output/discord.py index f77ada86d8..145d1aefd0 100644 --- a/bbot/modules/output/discord.py +++ b/bbot/modules/output/discord.py @@ -10,8 +10,9 @@ class Discord(WebhookOutputModule): "created_date": "2023-08-14", "author": "@TheTechromancer", } + class Config(BaseModuleConfig): - webhook_url: str = Field('', description='Discord webhook URL') - event_types: list[str] = Field(['FINDING'], description='Types of events to send') - min_severity: str = Field('LOW', description='Only allow FINDING events of this severity or higher') - retries: int = Field(10, description='Number of times to retry sending the message before skipping the event') + webhook_url: str = Field("", description="Discord webhook URL") + event_types: list[str] = Field(["FINDING"], description="Types of events to send") + min_severity: str = Field("LOW", description="Only allow FINDING events of this severity or higher") + retries: int = Field(10, description="Number of times to retry sending the message before skipping the event") diff --git a/bbot/modules/output/elastic.py b/bbot/modules/output/elastic.py index b35e6ab19c..b2a5e19323 100644 --- a/bbot/modules/output/elastic.py +++ b/bbot/modules/output/elastic.py @@ -14,11 +14,15 @@ class Elastic(HTTP): "created_date": "2022-11-21", "author": "@TheTechromancer", } + class Config(BaseModuleConfig): - url: str = Field('https://localhost:9200/bbot_events/_doc', description='Elastic URL (e.g. https://localhost:9200//_doc)') - username: str = Field('elastic', description='Elastic username') - password: str = Field('bbotislife', description='Elastic password') - timeout: int = Field(10, description='HTTP timeout') + url: str = Field( + "https://localhost:9200/bbot_events/_doc", + description="Elastic URL (e.g. https://localhost:9200//_doc)", + ) + username: str = Field("elastic", description="Elastic username") + password: str = Field("bbotislife", description="Elastic password") + timeout: int = Field(10, description="HTTP timeout") async def cleanup(self): # refresh the index diff --git a/bbot/modules/output/emails.py b/bbot/modules/output/emails.py index 3fa9c1a839..5bbbc9358d 100644 --- a/bbot/modules/output/emails.py +++ b/bbot/modules/output/emails.py @@ -12,8 +12,10 @@ class Emails(TXT): "created_date": "2023-12-23", "author": "@domwhewell-sage", } + class Config(BaseModuleConfig): - output_file: str = Field('', description='Output to file') + output_file: str = Field("", description="Output to file") + in_scope_only = True accept_dupes = False diff --git a/bbot/modules/output/http.py b/bbot/modules/output/http.py index c79f4eb0dd..484c11d1d4 100644 --- a/bbot/modules/output/http.py +++ b/bbot/modules/output/http.py @@ -11,14 +11,15 @@ class HTTP(BaseOutputModule): "created_date": "2022-04-13", "author": "@TheTechromancer", } + class Config(BaseModuleConfig): - url: str = Field('', description='Web URL') - method: str = Field('POST', description='HTTP method') - bearer: str = Field('', description='Authorization Bearer token') - username: str = Field('', description='Username (basic auth)') - password: str = Field('', description='Password (basic auth)') - headers: dict = Field({}, description='Additional headers to send with the request') - timeout: int = Field(10, description='HTTP timeout') + url: str = Field("", description="Web URL") + method: str = Field("POST", description="HTTP method") + bearer: str = Field("", description="Authorization Bearer token") + username: str = Field("", description="Username (basic auth)") + password: str = Field("", description="Password (basic auth)") + headers: dict = Field({}, description="Additional headers to send with the request") + timeout: int = Field(10, description="HTTP timeout") async def setup(self): self.url = self.config.get("url", "") diff --git a/bbot/modules/output/json.py b/bbot/modules/output/json.py index c0e69f36bb..dbd2bb7b7d 100644 --- a/bbot/modules/output/json.py +++ b/bbot/modules/output/json.py @@ -13,8 +13,10 @@ class JSON(BaseOutputModule): "created_date": "2022-04-07", "author": "@TheTechromancer", } + class Config(BaseModuleConfig): - output_file: str = Field('', description='Output to file') + output_file: str = Field("", description="Output to file") + _preserve_graph = True async def setup(self): diff --git a/bbot/modules/output/kafka.py b/bbot/modules/output/kafka.py index 52dd95b11c..d0db260cbf 100644 --- a/bbot/modules/output/kafka.py +++ b/bbot/modules/output/kafka.py @@ -13,9 +13,13 @@ class Kafka(BaseOutputModule): "created_date": "2024-11-22", "author": "@TheTechromancer", } + class Config(BaseModuleConfig): - bootstrap_servers: str = Field('localhost:9092', description='A comma-separated list of Kafka server addresses') - topic: str = Field('bbot_events', description='The Kafka topic to publish events to') + bootstrap_servers: str = Field( + "localhost:9092", description="A comma-separated list of Kafka server addresses" + ) + topic: str = Field("bbot_events", description="The Kafka topic to publish events to") + deps_pip = ["aiokafka~=0.12.0"] async def setup(self): diff --git a/bbot/modules/output/mongo.py b/bbot/modules/output/mongo.py index f9c2ad185a..158f315d76 100644 --- a/bbot/modules/output/mongo.py +++ b/bbot/modules/output/mongo.py @@ -19,12 +19,14 @@ class Mongo(BaseOutputModule): "created_date": "2024-11-17", "author": "@TheTechromancer", } + class Config(BaseModuleConfig): - uri: str = Field('mongodb://localhost:27017', description='The URI of the MongoDB server') - database: str = Field('bbot', description='The name of the database to use') - username: str = Field('', description='The username to use to connect to the database') - password: str = Field('', description='The password to use to connect to the database') - collection_prefix: str = Field('', description='Prefix the name of each collection with this string') + uri: str = Field("mongodb://localhost:27017", description="The URI of the MongoDB server") + database: str = Field("bbot", description="The name of the database to use") + username: str = Field("", description="The username to use to connect to the database") + password: str = Field("", description="The password to use to connect to the database") + collection_prefix: str = Field("", description="Prefix the name of each collection with this string") + deps_pip = ["pymongo~=4.15"] async def setup(self): diff --git a/bbot/modules/output/mysql.py b/bbot/modules/output/mysql.py index f66fc68558..8fd1510acb 100644 --- a/bbot/modules/output/mysql.py +++ b/bbot/modules/output/mysql.py @@ -10,12 +10,14 @@ class MySQL(SQLTemplate): "created_date": "2024-11-13", "author": "@TheTechromancer", } + class Config(BaseModuleConfig): - username: str = Field('root', description='The username to connect to MySQL') - password: str = Field('bbotislife', description='The password to connect to MySQL') - host: str = Field('localhost', description='The server running MySQL') - port: int = Field(3306, description='The port to connect to MySQL') - database: str = Field('bbot', description='The database name to connect to') + username: str = Field("root", description="The username to connect to MySQL") + password: str = Field("bbotislife", description="The password to connect to MySQL") + host: str = Field("localhost", description="The server running MySQL") + port: int = Field(3306, description="The port to connect to MySQL") + database: str = Field("bbot", description="The database name to connect to") + deps_pip = ["sqlmodel", "aiomysql"] protocol = "mysql+aiomysql" diff --git a/bbot/modules/output/nats.py b/bbot/modules/output/nats.py index 75c3fdd741..036a9a5750 100644 --- a/bbot/modules/output/nats.py +++ b/bbot/modules/output/nats.py @@ -12,9 +12,11 @@ class NATS(BaseOutputModule): "created_date": "2024-11-22", "author": "@TheTechromancer", } + class Config(BaseModuleConfig): - servers: list = Field([], description='A list of NATS server addresses') - subject: str = Field('bbot_events', description='The NATS subject to publish events to') + servers: list = Field([], description="A list of NATS server addresses") + subject: str = Field("bbot_events", description="The NATS subject to publish events to") + deps_pip = ["nats-py"] async def setup(self): diff --git a/bbot/modules/output/neo4j.py b/bbot/modules/output/neo4j.py index 0641269799..21204f8f0e 100644 --- a/bbot/modules/output/neo4j.py +++ b/bbot/modules/output/neo4j.py @@ -35,10 +35,12 @@ class neo4j(BaseOutputModule): watched_events = ["*"] meta = {"description": "Output to Neo4j", "created_date": "2022-04-07", "author": "@TheTechromancer"} + class Config(BaseModuleConfig): - uri: str = Field('bolt://localhost:7687', description='Neo4j server + port') - username: str = Field('neo4j', description='Neo4j username') - password: str = Field('bbotislife', description='Neo4j password') + uri: str = Field("bolt://localhost:7687", description="Neo4j server + port") + username: str = Field("neo4j", description="Neo4j username") + password: str = Field("bbotislife", description="Neo4j password") + deps_pip = ["neo4j"] _batch_size = 500 _preserve_graph = True diff --git a/bbot/modules/output/postgres.py b/bbot/modules/output/postgres.py index 61694b2f4b..ffbbe72afe 100644 --- a/bbot/modules/output/postgres.py +++ b/bbot/modules/output/postgres.py @@ -10,12 +10,14 @@ class Postgres(SQLTemplate): "created_date": "2024-11-08", "author": "@TheTechromancer", } + class Config(BaseModuleConfig): - username: str = Field('postgres', description='The username to connect to Postgres') - password: str = Field('bbotislife', description='The password to connect to Postgres') - host: str = Field('localhost', description='The server running Postgres') - port: int = Field(5432, description='The port to connect to Postgres') - database: str = Field('bbot', description='The database name to connect to') + username: str = Field("postgres", description="The username to connect to Postgres") + password: str = Field("bbotislife", description="The password to connect to Postgres") + host: str = Field("localhost", description="The server running Postgres") + port: int = Field(5432, description="The port to connect to Postgres") + database: str = Field("bbot", description="The database name to connect to") + deps_pip = ["sqlmodel", "asyncpg"] protocol = "postgresql+asyncpg" diff --git a/bbot/modules/output/rabbitmq.py b/bbot/modules/output/rabbitmq.py index 8e28e4c649..c5bff3a78c 100644 --- a/bbot/modules/output/rabbitmq.py +++ b/bbot/modules/output/rabbitmq.py @@ -13,9 +13,11 @@ class RabbitMQ(BaseOutputModule): "created_date": "2024-11-22", "author": "@TheTechromancer", } + class Config(BaseModuleConfig): - url: str = Field('amqp://guest:guest@localhost/', description='The RabbitMQ connection URL') - queue: str = Field('bbot_events', description='The RabbitMQ queue to publish events to') + url: str = Field("amqp://guest:guest@localhost/", description="The RabbitMQ connection URL") + queue: str = Field("bbot_events", description="The RabbitMQ queue to publish events to") + deps_pip = ["aio_pika~=9.5.0"] async def setup(self): diff --git a/bbot/modules/output/slack.py b/bbot/modules/output/slack.py index 056bed4389..b1109ac99d 100644 --- a/bbot/modules/output/slack.py +++ b/bbot/modules/output/slack.py @@ -12,11 +12,13 @@ class Slack(WebhookOutputModule): "created_date": "2023-08-14", "author": "@TheTechromancer", } + class Config(BaseModuleConfig): - webhook_url: str = Field('', description='Discord webhook URL') - event_types: list[str] = Field(['FINDING'], description='Types of events to send') - min_severity: str = Field('LOW', description='Only allow FINDING events of this severity or higher') - retries: int = Field(10, description='Number of times to retry sending the message before skipping the event') + webhook_url: str = Field("", description="Discord webhook URL") + event_types: list[str] = Field(["FINDING"], description="Types of events to send") + min_severity: str = Field("LOW", description="Only allow FINDING events of this severity or higher") + retries: int = Field(10, description="Number of times to retry sending the message before skipping the event") + content_key = "text" def format_message_str(self, event): diff --git a/bbot/modules/output/splunk.py b/bbot/modules/output/splunk.py index 97d33a6bc1..c0d45b89e6 100644 --- a/bbot/modules/output/splunk.py +++ b/bbot/modules/output/splunk.py @@ -11,12 +11,13 @@ class Splunk(BaseOutputModule): "created_date": "2024-02-17", "author": "@w0Tx", } + class Config(BaseModuleConfig): - url: str = Field('', description='Web URL') - hectoken: str = Field('', description='HEC Token') - index: str = Field('', description='Index to send data to') - source: str = Field('', description='Source path to be added to the metadata') - timeout: int = Field(10, description='HTTP timeout') + url: str = Field("", description="Web URL") + hectoken: str = Field("", description="HEC Token") + index: str = Field("", description="Index to send data to") + source: str = Field("", description="Source path to be added to the metadata") + timeout: int = Field(10, description="HTTP timeout") async def setup(self): self.url = self.config.get("url", "") diff --git a/bbot/modules/output/sqlite.py b/bbot/modules/output/sqlite.py index b6a0e11d37..e306664354 100644 --- a/bbot/modules/output/sqlite.py +++ b/bbot/modules/output/sqlite.py @@ -12,8 +12,10 @@ class SQLite(SQLTemplate): "created_date": "2024-11-07", "author": "@TheTechromancer", } + class Config(BaseModuleConfig): - database: str = Field('', description='The path to the sqlite database file') + database: str = Field("", description="The path to the sqlite database file") + deps_pip = ["sqlmodel", "aiosqlite"] async def setup(self): diff --git a/bbot/modules/output/stdout.py b/bbot/modules/output/stdout.py index 73d59370dc..9c43ed06e3 100644 --- a/bbot/modules/output/stdout.py +++ b/bbot/modules/output/stdout.py @@ -9,12 +9,14 @@ class Stdout(BaseOutputModule): watched_events = ["*"] meta = {"description": "Output to text", "created_date": "2024-04-03", "author": "@TheTechromancer"} + class Config(BaseModuleConfig): - format: str = Field('text', description='Which text format to display, choices: text,json') - event_types: list = Field([], description='Which events to display, default all event types') - event_fields: list = Field([], description='Which event fields to display') - in_scope_only: bool = Field(False, description='Whether to only show in-scope events') - accept_dupes: bool = Field(True, description='Whether to show duplicate events, default True') + format: str = Field("text", description="Which text format to display, choices: text,json") + event_types: list = Field([], description="Which events to display, default all event types") + event_fields: list = Field([], description="Which event fields to display") + in_scope_only: bool = Field(False, description="Whether to only show in-scope events") + accept_dupes: bool = Field(True, description="Whether to show duplicate events, default True") + vuln_severity_map = { "INFO": "HUGEINFO", "LOW": "HUGEWARNING", diff --git a/bbot/modules/output/subdomains.py b/bbot/modules/output/subdomains.py index 0e58d3e842..eebda26720 100644 --- a/bbot/modules/output/subdomains.py +++ b/bbot/modules/output/subdomains.py @@ -12,9 +12,11 @@ class Subdomains(TXT): "created_date": "2023-07-31", "author": "@TheTechromancer", } + class Config(BaseModuleConfig): - output_file: str = Field('', description='Output to file') - include_unresolved: bool = Field(False, description='Include unresolved subdomains in output') + output_file: str = Field("", description="Output to file") + include_unresolved: bool = Field(False, description="Include unresolved subdomains in output") + accept_dupes = False in_scope_only = True diff --git a/bbot/modules/output/teams.py b/bbot/modules/output/teams.py index c4fd7a3bac..31db1ceef9 100644 --- a/bbot/modules/output/teams.py +++ b/bbot/modules/output/teams.py @@ -10,11 +10,12 @@ class Teams(WebhookOutputModule): "created_date": "2023-08-14", "author": "@TheTechromancer", } + class Config(BaseModuleConfig): - webhook_url: str = Field('', description='Teams webhook URL') - event_types: list[str] = Field(['FINDING'], description='Types of events to send') - min_severity: str = Field('LOW', description='Only allow FINDING events of this severity or higher') - retries: int = Field(10, description='Number of times to retry sending the message before skipping the event') + webhook_url: str = Field("", description="Teams webhook URL") + event_types: list[str] = Field(["FINDING"], description="Types of events to send") + min_severity: str = Field("LOW", description="Only allow FINDING events of this severity or higher") + retries: int = Field(10, description="Number of times to retry sending the message before skipping the event") async def handle_event(self, event): data = self.format_message(event) diff --git a/bbot/modules/output/txt.py b/bbot/modules/output/txt.py index 42fd12d1fe..b4c123168f 100644 --- a/bbot/modules/output/txt.py +++ b/bbot/modules/output/txt.py @@ -8,8 +8,9 @@ class TXT(BaseOutputModule): watched_events = ["*"] meta = {"description": "Output to text", "created_date": "2024-04-03", "author": "@TheTechromancer"} + class Config(BaseModuleConfig): - output_file: str = Field('', description='Output to file') + output_file: str = Field("", description="Output to file") output_filename = "output.txt" diff --git a/bbot/modules/output/web_parameters.py b/bbot/modules/output/web_parameters.py index 1edf5a1b7a..0e4a6b625c 100644 --- a/bbot/modules/output/web_parameters.py +++ b/bbot/modules/output/web_parameters.py @@ -13,9 +13,10 @@ class Web_parameters(BaseOutputModule): "created_date": "2025-01-25", "author": "@liquidsec", } + class Config(BaseModuleConfig): - output_file: str = Field('', description='Output to file') - include_count: bool = Field(False, description='Include the count of each parameter in the output') + output_file: str = Field("", description="Output to file") + include_count: bool = Field(False, description="Include the count of each parameter in the output") output_filename = "web_parameters.txt" diff --git a/bbot/modules/output/web_report.py b/bbot/modules/output/web_report.py index 81f984540e..d6dcfbbd95 100644 --- a/bbot/modules/output/web_report.py +++ b/bbot/modules/output/web_report.py @@ -12,9 +12,14 @@ class web_report(BaseOutputModule): "created_date": "2023-02-08", "author": "@liquidsec", } + class Config(BaseModuleConfig): - output_file: str = Field('', description='Output to file') - css_theme_file: str = Field('https://cdnjs.cloudflare.com/ajax/libs/github-markdown-css/5.1.0/github-markdown.min.css', description='CSS theme URL for HTML output') + output_file: str = Field("", description="Output to file") + css_theme_file: str = Field( + "https://cdnjs.cloudflare.com/ajax/libs/github-markdown-css/5.1.0/github-markdown.min.css", + description="CSS theme URL for HTML output", + ) + deps_pip = ["markdown~=3.4.3"] async def setup(self): diff --git a/bbot/modules/output/websocket.py b/bbot/modules/output/websocket.py index d5f3d92c8f..1ca60a36ca 100644 --- a/bbot/modules/output/websocket.py +++ b/bbot/modules/output/websocket.py @@ -11,11 +11,16 @@ class Websocket(BaseOutputModule): watched_events = ["*"] meta = {"description": "Output to websockets", "created_date": "2022-04-15", "author": "@TheTechromancer"} + class Config(BaseModuleConfig): - url: str = Field('', description='Web URL') - token: str = Field('', description='Authorization Bearer token') - preserve_graph: bool = Field(True, description='Preserve full chains of events in the graph (prevents orphans)') - ignore_ssl: bool = Field(False, description='Ignores all Websocket SSL related errors (like Self-Signed Certificates, etc.)') + url: str = Field("", description="Web URL") + token: str = Field("", description="Authorization Bearer token") + preserve_graph: bool = Field( + True, description="Preserve full chains of events in the graph (prevents orphans)" + ) + ignore_ssl: bool = Field( + False, description="Ignores all Websocket SSL related errors (like Self-Signed Certificates, etc.)" + ) async def setup(self): self.url = self.config.get("url", "") diff --git a/bbot/modules/output/zeromq.py b/bbot/modules/output/zeromq.py index c9a7e4059f..b481fa1e7c 100644 --- a/bbot/modules/output/zeromq.py +++ b/bbot/modules/output/zeromq.py @@ -13,8 +13,11 @@ class ZeroMQ(BaseOutputModule): "created_date": "2024-11-22", "author": "@TheTechromancer", } + class Config(BaseModuleConfig): - zmq_address: str = Field('', description='The ZeroMQ socket address to publish events to (e.g. tcp://localhost:5555)') + zmq_address: str = Field( + "", description="The ZeroMQ socket address to publish events to (e.g. tcp://localhost:5555)" + ) async def setup(self): self.zmq_address = self.config.get("zmq_address", "") diff --git a/bbot/modules/paramminer_cookies.py b/bbot/modules/paramminer_cookies.py index c85b8e92f5..fe20d75ba5 100644 --- a/bbot/modules/paramminer_cookies.py +++ b/bbot/modules/paramminer_cookies.py @@ -16,14 +16,14 @@ class paramminer_cookies(paramminer_headers): "created_date": "2022-06-27", "author": "@liquidsec", } + class Config(BaseModuleConfig): wordlist: str = Field("", description="Define the wordlist to be used to derive cookies") recycle_words: bool = Field( False, description="Attempt to use words found during the scan on all other endpoints" ) - skip_boring_words: bool = Field( - True, description="Remove commonly uninteresting words from the wordlist" - ) + skip_boring_words: bool = Field(True, description="Remove commonly uninteresting words from the wordlist") + scanned_hosts = [] boring_words = set() _module_threads = 12 diff --git a/bbot/modules/paramminer_getparams.py b/bbot/modules/paramminer_getparams.py index 98e9497f69..6471822168 100644 --- a/bbot/modules/paramminer_getparams.py +++ b/bbot/modules/paramminer_getparams.py @@ -17,10 +17,14 @@ class paramminer_getparams(paramminer_headers): "author": "@liquidsec", } scanned_hosts = [] + class Config(BaseModuleConfig): - wordlist: str = Field('', description='Define the wordlist to be used to derive headers') - recycle_words: bool = Field(False, description='Attempt to use words found during the scan on all other endpoints') - skip_boring_words: bool = Field(True, description='Remove commonly uninteresting words from the wordlist') + wordlist: str = Field("", description="Define the wordlist to be used to derive headers") + recycle_words: bool = Field( + False, description="Attempt to use words found during the scan on all other endpoints" + ) + skip_boring_words: bool = Field(True, description="Remove commonly uninteresting words from the wordlist") + boring_words = {"utm_source", "utm_campaign", "utm_medium", "utm_term", "utm_content"} in_scope_only = True compare_mode = "getparam" diff --git a/bbot/modules/paramminer_headers.py b/bbot/modules/paramminer_headers.py index c99388ffae..0d0a8cf84a 100644 --- a/bbot/modules/paramminer_headers.py +++ b/bbot/modules/paramminer_headers.py @@ -19,10 +19,14 @@ class paramminer_headers(BaseModule): "created_date": "2022-04-15", "author": "@liquidsec", } + class Config(BaseModuleConfig): - wordlist: str = Field('', description='Define the wordlist to be used to derive headers') - recycle_words: bool = Field(False, description='Attempt to use words found during the scan on all other endpoints') - skip_boring_words: bool = Field(True, description='Remove commonly uninteresting words from the wordlist') + wordlist: str = Field("", description="Define the wordlist to be used to derive headers") + recycle_words: bool = Field( + False, description="Attempt to use words found during the scan on all other endpoints" + ) + skip_boring_words: bool = Field(True, description="Remove commonly uninteresting words from the wordlist") + scanned_hosts = [] boring_words = { "accept", diff --git a/bbot/modules/passivetotal.py b/bbot/modules/passivetotal.py index d5d5f26611..8758693adb 100644 --- a/bbot/modules/passivetotal.py +++ b/bbot/modules/passivetotal.py @@ -13,8 +13,9 @@ class passivetotal(subdomain_enum_apikey): "author": "@TheTechromancer", "auth_required": True, } + class Config(BaseModuleConfig): - api_key: str = Field('', description="PassiveTotal API Key in the format of 'username:api_key'") + api_key: str = Field("", description="PassiveTotal API Key in the format of 'username:api_key'") base_url = "https://api.passivetotal.org/v2" diff --git a/bbot/modules/pgp.py b/bbot/modules/pgp.py index 8e53c4e48c..61f596c478 100644 --- a/bbot/modules/pgp.py +++ b/bbot/modules/pgp.py @@ -12,8 +12,17 @@ class pgp(subdomain_enum): "created_date": "2022-08-10", "author": "@TheTechromancer", } + class Config(BaseModuleConfig): - search_urls: list[str] = Field(['https://keyserver.ubuntu.com/pks/lookup?fingerprint=on&op=vindex&search=', 'http://the.earth.li:11371/pks/lookup?fingerprint=on&op=vindex&search=', 'https://pgpkeys.eu/pks/lookup?search=&op=index', 'https://pgp.mit.edu/pks/lookup?search=&op=index'], description='PGP key servers to search') + search_urls: list[str] = Field( + [ + "https://keyserver.ubuntu.com/pks/lookup?fingerprint=on&op=vindex&search=", + "http://the.earth.li:11371/pks/lookup?fingerprint=on&op=vindex&search=", + "https://pgpkeys.eu/pks/lookup?search=&op=index", + "https://pgp.mit.edu/pks/lookup?search=&op=index", + ], + description="PGP key servers to search", + ) async def handle_event(self, event): query = self.make_query(event) diff --git a/bbot/modules/portfilter.py b/bbot/modules/portfilter.py index 9d006b2af5..25d943c595 100644 --- a/bbot/modules/portfilter.py +++ b/bbot/modules/portfilter.py @@ -11,9 +11,12 @@ class portfilter(BaseInterceptModule): "created_date": "2025-01-06", "author": "@TheTechromancer", } + class Config(BaseModuleConfig): - cdn_tags: str = Field('cdn,waf', description="Comma-separated list of tags to skip, e.g. 'cdn,waf'") - allowed_cdn_ports: str = Field('80,443', description='Comma-separated list of ports that are allowed to be scanned for CDNs') + cdn_tags: str = Field("cdn,waf", description="Comma-separated list of tags to skip, e.g. 'cdn,waf'") + allowed_cdn_ports: str = Field( + "80,443", description="Comma-separated list of ports that are allowed to be scanned for CDNs" + ) _priority = 4 # we consume URLs but we don't want to automatically enable httpx diff --git a/bbot/modules/portscan.py b/bbot/modules/portscan.py index 1d991e403f..d9749e854a 100644 --- a/bbot/modules/portscan.py +++ b/bbot/modules/portscan.py @@ -20,18 +20,31 @@ class portscan(BaseModule): "created_date": "2024-05-15", "author": "@TheTechromancer", } + class Config(BaseModuleConfig): top_ports: int = Field(100, description="Top ports to scan (default 100) (to override, specify 'ports')") - ports: str = Field('', description='Ports to scan') - rate: int = Field(300, description='Rate in packets per second') - wait: int = Field(5, description='Seconds to wait for replies after scan is complete') - ping_first: bool = Field(False, description='Only portscan hosts that reply to pings') - ping_only: bool = Field(False, description='Ping sweep only, no portscan') - adapter: str = Field('', description='Manually specify a network interface, such as "eth0" or "tun0". If not specified, the first network interface found with a default gateway will be used.') - adapter_ip: str = Field('', description="Send packets using this IP address. Not needed unless masscan's autodetection fails") - adapter_mac: str = Field('', description="Send packets using this as the source MAC address. Not needed unless masscan's autodetection fails") - router_mac: str = Field('', description="Send packets to this MAC address as the destination. Not needed unless masscan's autodetection fails") - module_timeout: int = Field(259200, description='Max time in seconds to spend handling each batch of events') + ports: str = Field("", description="Ports to scan") + rate: int = Field(300, description="Rate in packets per second") + wait: int = Field(5, description="Seconds to wait for replies after scan is complete") + ping_first: bool = Field(False, description="Only portscan hosts that reply to pings") + ping_only: bool = Field(False, description="Ping sweep only, no portscan") + adapter: str = Field( + "", + description='Manually specify a network interface, such as "eth0" or "tun0". If not specified, the first network interface found with a default gateway will be used.', + ) + adapter_ip: str = Field( + "", description="Send packets using this IP address. Not needed unless masscan's autodetection fails" + ) + adapter_mac: str = Field( + "", + description="Send packets using this as the source MAC address. Not needed unless masscan's autodetection fails", + ) + router_mac: str = Field( + "", + description="Send packets to this MAC address as the destination. Not needed unless masscan's autodetection fails", + ) + module_timeout: int = Field(259200, description="Max time in seconds to spend handling each batch of events") + deps_common = ["masscan"] batch_size = 1000000 _shuffle_incoming_queue = False diff --git a/bbot/modules/postman.py b/bbot/modules/postman.py index 076e763c1f..a182894514 100644 --- a/bbot/modules/postman.py +++ b/bbot/modules/postman.py @@ -12,8 +12,10 @@ class postman(postman): "created_date": "2024-09-07", "author": "@domwhewell-sage", } + class Config(BaseModuleConfig): - api_key: str = Field('', description='Postman API Key') + api_key: str = Field("", description="Postman API Key") + reject_wildcards = False async def handle_event(self, event): diff --git a/bbot/modules/postman_download.py b/bbot/modules/postman_download.py index 34fdf8c6bd..7f063303ec 100644 --- a/bbot/modules/postman_download.py +++ b/bbot/modules/postman_download.py @@ -15,9 +15,14 @@ class postman_download(postman): "created_date": "2024-09-07", "author": "@domwhewell-sage", } + class Config(BaseModuleConfig): - output_folder: str = Field('', description='Folder to download postman workspaces to. If not specified, downloaded workspaces will be deleted when the scan completes, to minimize disk usage.') - api_key: str = Field('', description='Postman API Key') + output_folder: str = Field( + "", + description="Folder to download postman workspaces to. If not specified, downloaded workspaces will be deleted when the scan completes, to minimize disk usage.", + ) + api_key: str = Field("", description="Postman API Key") + scope_distance_modifier = 2 async def setup(self): diff --git a/bbot/modules/retirejs.py b/bbot/modules/retirejs.py index d07dbeb7b0..b2d2d8323d 100644 --- a/bbot/modules/retirejs.py +++ b/bbot/modules/retirejs.py @@ -29,10 +29,13 @@ class retirejs(BaseModule): "created_date": "2025-08-19", "author": "@liquidsec", } + class Config(BaseModuleConfig): - version: str = Field('5.3.0', description='retire.js version') - node_version: str = Field('18.19.1', description='Node.js version to install locally') - severity: str = Field('medium', description='Minimum severity level to report (none, low, medium, high, critical)') + version: str = Field("5.3.0", description="retire.js version") + node_version: str = Field("18.19.1", description="Node.js version to install locally") + severity: str = Field( + "medium", description="Minimum severity level to report (none, low, medium, high, critical)" + ) deps_ansible = [ # Download Node.js binary (Linux x64) diff --git a/bbot/modules/securitytrails.py b/bbot/modules/securitytrails.py index c686f4c120..092e478972 100644 --- a/bbot/modules/securitytrails.py +++ b/bbot/modules/securitytrails.py @@ -13,8 +13,9 @@ class securitytrails(subdomain_enum_apikey): "author": "@TheTechromancer", "auth_required": True, } + class Config(BaseModuleConfig): - api_key: str = Field('', description='SecurityTrails API key') + api_key: str = Field("", description="SecurityTrails API key") base_url = "https://api.securitytrails.com/v1" ping_url = f"{base_url}/ping?apikey={{api_key}}" diff --git a/bbot/modules/securitytxt.py b/bbot/modules/securitytxt.py index fb84e5d1b2..e32c3f92b2 100644 --- a/bbot/modules/securitytxt.py +++ b/bbot/modules/securitytxt.py @@ -66,9 +66,10 @@ class securitytxt(BaseModule): "author": "@colin-stubbs", "created_date": "2024-05-26", } + class Config(BaseModuleConfig): - emails: bool = Field(True, description='emit EMAIL_ADDRESS events') - urls: bool = Field(True, description='emit URL_UNVERIFIED events') + emails: bool = Field(True, description="emit EMAIL_ADDRESS events") + urls: bool = Field(True, description="emit URL_UNVERIFIED events") async def setup(self): self._emails = self.config.get("emails", True) diff --git a/bbot/modules/shodan_dns.py b/bbot/modules/shodan_dns.py index f3654fabf1..7a023b47ef 100644 --- a/bbot/modules/shodan_dns.py +++ b/bbot/modules/shodan_dns.py @@ -13,8 +13,9 @@ class shodan_dns(shodan): "author": "@TheTechromancer", "auth_required": True, } + class Config(BaseModuleConfig): - api_key: str = Field('', description='Shodan API key') + api_key: str = Field("", description="Shodan API key") base_url = "https://api.shodan.io" diff --git a/bbot/modules/shodan_enterprise.py b/bbot/modules/shodan_enterprise.py index 3cd732e0a4..46f9fb9511 100644 --- a/bbot/modules/shodan_enterprise.py +++ b/bbot/modules/shodan_enterprise.py @@ -13,9 +13,13 @@ class shodan_enterprise(BaseModule): "description": "Shodan Enterprise API integration module.", "auth_required": True, } + class Config(BaseModuleConfig): - api_key: str = Field('', description='Shodan API Key') - in_scope_only: bool = Field(True, description='Only query in-scope IPs. If False, will query up to distance 1.') + api_key: str = Field("", description="Shodan API Key") + in_scope_only: bool = Field( + True, description="Only query in-scope IPs. If False, will query up to distance 1." + ) + in_scope_only = True base_url = "https://api.shodan.io" diff --git a/bbot/modules/shodan_idb.py b/bbot/modules/shodan_idb.py index 31d8fbda36..b2c1208b4f 100644 --- a/bbot/modules/shodan_idb.py +++ b/bbot/modules/shodan_idb.py @@ -50,8 +50,12 @@ class shodan_idb(BaseModule): "created_date": "2023-12-22", "author": "@TheTechromancer", } + class Config(BaseModuleConfig): - retries: Optional[str] = Field(None, description='How many times to retry API requests (e.g. after a 429 error). Overrides the global web.api_retries setting.') + retries: Optional[str] = Field( + None, + description="How many times to retry API requests (e.g. after a 429 error). Overrides the global web.api_retries setting.", + ) # we typically don't want to abort this module _api_failure_abort_threshold = 9999999999 diff --git a/bbot/modules/sslcert.py b/bbot/modules/sslcert.py index ffc0890f2e..62911d1b3f 100644 --- a/bbot/modules/sslcert.py +++ b/bbot/modules/sslcert.py @@ -19,9 +19,11 @@ class sslcert(BaseModule): "created_date": "2022-03-30", "author": "@TheTechromancer", } + class Config(BaseModuleConfig): - timeout: float = Field(5.0, description='Socket connect timeout in seconds') + timeout: float = Field(5.0, description="Socket connect timeout in seconds") skip_non_ssl: bool = Field(True, description="Don't try common non-SSL ports") + deps_apt = ["openssl"] deps_pip = ["pyOpenSSL~=25.3.0"] _module_threads = 25 diff --git a/bbot/modules/subdomainradar.py b/bbot/modules/subdomainradar.py index 5d98a7a7fd..f8fada1714 100644 --- a/bbot/modules/subdomainradar.py +++ b/bbot/modules/subdomainradar.py @@ -16,10 +16,11 @@ class SubdomainRadar(subdomain_enum_apikey): "author": "@TheTechromancer", "auth_required": True, } + class Config(BaseModuleConfig): - api_key: str = Field('', description='SubDomainRadar.io API key') - group: str = Field('fast', description='The enumeration group to use. Choose from fast, medium, deep') - timeout: int = Field(120, description='Timeout in seconds') + api_key: str = Field("", description="SubDomainRadar.io API key") + group: str = Field("fast", description="The enumeration group to use. Choose from fast, medium, deep") + timeout: int = Field(120, description="Timeout in seconds") base_url = "https://api.subdomainradar.io" ping_url = f"{base_url}/profile" diff --git a/bbot/modules/telerik.py b/bbot/modules/telerik.py index 70c168bea2..62ae34404b 100644 --- a/bbot/modules/telerik.py +++ b/bbot/modules/telerik.py @@ -156,8 +156,8 @@ class telerik(BaseModule): RAUConfirmed = [] class Config(BaseModuleConfig): - exploit_RAU_crypto: bool = Field(False, description='Attempt to confirm any RAU AXD detections are vulnerable') - include_subdirs: bool = Field(False, description='Include subdirectories in the scan (off by default)') + exploit_RAU_crypto: bool = Field(False, description="Attempt to confirm any RAU AXD detections are vulnerable") + include_subdirs: bool = Field(False, description="Include subdirectories in the scan (off by default)") in_scope_only = True diff --git a/bbot/modules/trajan.py b/bbot/modules/trajan.py index a8400b48d7..c7cf634f65 100644 --- a/bbot/modules/trajan.py +++ b/bbot/modules/trajan.py @@ -18,14 +18,14 @@ class trajan(BaseModule): # Configuration options class Config(BaseModuleConfig): - version: str = Field('1.0.0', description='Trajan version to download and use') - github_token: str = Field('', description='GitHub API token for rate-limiting and private repo access') - gitlab_token: str = Field('', description='GitLab API token for private repo access') - ado_token: str = Field('', description='Azure DevOps Personal Access Token (PAT)') - jfrog_token: str = Field('', description='JFrog API token') - jenkins_username: str = Field('', description='Jenkins username for basic auth') - jenkins_password: str = Field('', description='Jenkins password for basic auth') - jenkins_token: str = Field('', description='Jenkins API token') + version: str = Field("1.0.0", description="Trajan version to download and use") + github_token: str = Field("", description="GitHub API token for rate-limiting and private repo access") + gitlab_token: str = Field("", description="GitLab API token for private repo access") + ado_token: str = Field("", description="Azure DevOps Personal Access Token (PAT)") + jfrog_token: str = Field("", description="JFrog API token") + jenkins_username: str = Field("", description="Jenkins username for basic auth") + jenkins_password: str = Field("", description="Jenkins password for basic auth") + jenkins_token: str = Field("", description="Jenkins API token") deps_ansible = [ { diff --git a/bbot/modules/trickest.py b/bbot/modules/trickest.py index c008a89bd5..12f835ab24 100644 --- a/bbot/modules/trickest.py +++ b/bbot/modules/trickest.py @@ -13,8 +13,9 @@ class Trickest(subdomain_enum_apikey): "created_date": "2024-07-27", "auth_required": True, } + class Config(BaseModuleConfig): - api_key: str = Field('', description='Trickest API key') + api_key: str = Field("", description="Trickest API key") base_url = "https://api.trickest.io/solutions/v1/public/solution/a7cba1f1-df07-4a5c-876a-953f178996be" ping_url = f"{base_url}/dataset" diff --git a/bbot/modules/trufflehog.py b/bbot/modules/trufflehog.py index 95cfd6d016..3baa6503b4 100644 --- a/bbot/modules/trufflehog.py +++ b/bbot/modules/trufflehog.py @@ -16,11 +16,15 @@ class trufflehog(BaseModule): } class Config(BaseModuleConfig): - version: str = Field('3.90.8', description='trufflehog version') - config: str = Field('', description='File path or URL to YAML trufflehog config') - only_verified: bool = Field(True, description='Only report credentials that have been verified') - concurrency: int = Field(8, description='Number of concurrent workers') - deleted_forks: bool = Field(False, description='Scan for deleted github forks. WARNING: This is SLOW. For a smaller repository, this process can take 20 minutes. For a larger repository, it could take hours.') + version: str = Field("3.90.8", description="trufflehog version") + config: str = Field("", description="File path or URL to YAML trufflehog config") + only_verified: bool = Field(True, description="Only report credentials that have been verified") + concurrency: int = Field(8, description="Number of concurrent workers") + deleted_forks: bool = Field( + False, + description="Scan for deleted github forks. WARNING: This is SLOW. For a smaller repository, this process can take 20 minutes. For a larger repository, it could take hours.", + ) + deps_ansible = [ { "name": "Download trufflehog", diff --git a/bbot/modules/url_manipulation.py b/bbot/modules/url_manipulation.py index cca6effc88..6f56c4e5e7 100644 --- a/bbot/modules/url_manipulation.py +++ b/bbot/modules/url_manipulation.py @@ -16,7 +16,10 @@ class url_manipulation(BaseModule): in_scope_only = True class Config(BaseModuleConfig): - allow_redirects: bool = Field(True, description='Allowing redirects will sometimes create false positives. Disallowing will sometimes create false negatives. Allowed by default.') + allow_redirects: bool = Field( + True, + description="Allowing redirects will sometimes create false positives. Disallowing will sometimes create false negatives. Allowed by default.", + ) async def setup(self): # ([string]method,[string]path,[bool]strip trailing slash) diff --git a/bbot/modules/urlscan.py b/bbot/modules/urlscan.py index c433e9a0d1..adaf72af73 100644 --- a/bbot/modules/urlscan.py +++ b/bbot/modules/urlscan.py @@ -12,8 +12,9 @@ class urlscan(subdomain_enum): "created_date": "2022-06-09", "author": "@TheTechromancer", } + class Config(BaseModuleConfig): - urls: bool = Field(False, description='Emit URLs in addition to DNS_NAMEs') + urls: bool = Field(False, description="Emit URLs in addition to DNS_NAMEs") base_url = "https://urlscan.io/api/v1" diff --git a/bbot/modules/virustotal.py b/bbot/modules/virustotal.py index ec7e1e3ebd..ac69dea3d1 100644 --- a/bbot/modules/virustotal.py +++ b/bbot/modules/virustotal.py @@ -13,8 +13,9 @@ class virustotal(subdomain_enum_apikey): "author": "@TheTechromancer", "auth_required": True, } + class Config(BaseModuleConfig): - api_key: str = Field('', description='VirusTotal API Key') + api_key: str = Field("", description="VirusTotal API Key") base_url = "https://www.virustotal.com/api/v3" api_page_iter_kwargs = {"json": False, "next_key": lambda r: r.json().get("links", {}).get("next", "")} diff --git a/bbot/modules/wafw00f.py b/bbot/modules/wafw00f.py index 58d2085c62..499fae3c0c 100644 --- a/bbot/modules/wafw00f.py +++ b/bbot/modules/wafw00f.py @@ -27,7 +27,9 @@ class wafw00f(BaseModule): deps_pip = ["wafw00f~=2.3.1"] class Config(BaseModuleConfig): - generic_detect: bool = Field(True, description='When no specific WAF detections are made, try to perform a generic detect') + generic_detect: bool = Field( + True, description="When no specific WAF detections are made, try to perform a generic detect" + ) in_scope_only = True per_hostport_only = True diff --git a/bbot/modules/wayback.py b/bbot/modules/wayback.py index c07b3ff827..203ba7edc2 100644 --- a/bbot/modules/wayback.py +++ b/bbot/modules/wayback.py @@ -14,9 +14,14 @@ class wayback(subdomain_enum): "created_date": "2022-04-01", "author": "@liquidsec", } + class Config(BaseModuleConfig): - urls: bool = Field(False, description='emit URLs in addition to DNS_NAMEs') - garbage_threshold: int = Field(10, description='Dedupe similar urls if they are in a group of this size or higher (lower values == less garbage data)') + urls: bool = Field(False, description="emit URLs in addition to DNS_NAMEs") + garbage_threshold: int = Field( + 10, + description="Dedupe similar urls if they are in a group of this size or higher (lower values == less garbage data)", + ) + in_scope_only = True base_url = "http://web.archive.org" diff --git a/bbot/modules/wpscan.py b/bbot/modules/wpscan.py index 4fc1c0ddc1..e6953bd08a 100644 --- a/bbot/modules/wpscan.py +++ b/bbot/modules/wpscan.py @@ -15,13 +15,18 @@ class wpscan(BaseModule): } class Config(BaseModuleConfig): - api_key: str = Field('', description='WPScan API Key') - enumerate: str = Field('vp,vt,cb,dbe', description='Enumeration Process see wpscan help documentation (default: vp,vt,cb,dbe)') - threads: int = Field(5, description='How many wpscan threads to spawn (default is 5)') - request_timeout: int = Field(5, description='The request timeout in seconds (default 5)') - connection_timeout: int = Field(2, description='The connection timeout in seconds (default 2)') - disable_tls_checks: bool = Field(True, description='Disables the SSL/TLS certificate verification (Default True)') - force: bool = Field(False, description='Do not check if the target is running WordPress or returns a 403') + api_key: str = Field("", description="WPScan API Key") + enumerate: str = Field( + "vp,vt,cb,dbe", description="Enumeration Process see wpscan help documentation (default: vp,vt,cb,dbe)" + ) + threads: int = Field(5, description="How many wpscan threads to spawn (default is 5)") + request_timeout: int = Field(5, description="The request timeout in seconds (default 5)") + connection_timeout: int = Field(2, description="The connection timeout in seconds (default 2)") + disable_tls_checks: bool = Field( + True, description="Disables the SSL/TLS certificate verification (Default True)" + ) + force: bool = Field(False, description="Do not check if the target is running WordPress or returns a 403") + deps_apt = ["curl", "make", "gcc"] deps_ansible = [ { diff --git a/bbot/scanner/preset/validate.py b/bbot/scanner/preset/validate.py index a50e3a0523..008c29728b 100644 --- a/bbot/scanner/preset/validate.py +++ b/bbot/scanner/preset/validate.py @@ -26,6 +26,8 @@ from pydantic import ValidationError +from bbot.core.helpers.misc import get_closest_match + log = logging.getLogger("bbot.presets.validate") @@ -70,7 +72,7 @@ def _classify_loc(loc: tuple) -> tuple[str, str]: return ("preset", ".".join(parts)) -def _format_msg(err: dict) -> str: +def _format_msg(err: dict, known_modules: set | None = None) -> str: kind = err["type"] input_value = err.get("input") loc = err["loc"] @@ -78,10 +80,10 @@ def _format_msg(err: dict) -> str: path = ".".join(str(p) for p in loc) if kind == "extra_forbidden": - # Special-case the unknown-module-name error so users get - # "Unknown module: 'nucleii'" instead of "Unknown option: 'nucleii'". + # Special-case unknown module name (config.modules.) so users get + # a suggestion rather than "Unknown option". if len(loc) == 3 and loc[0] == "config" and loc[1] == "modules": - return f'Unknown module: "{field}"' + return get_closest_match(field, known_modules or set(), msg="module") msg = f"Unknown option: {field!r}" if isinstance(input_value, (str, int, bool, float)): msg += f" (value: {input_value!r})" @@ -112,11 +114,11 @@ def _format_msg(err: dict) -> str: return err["msg"] if err.get("msg") else f"validation error at {path}" -def _format_errors(exc: ValidationError) -> list[PresetValidationError]: +def _format_errors(exc: ValidationError, known_modules: set | None = None) -> list[PresetValidationError]: out: list[PresetValidationError] = [] for err in exc.errors(): where, path = _classify_loc(err["loc"]) - out.append(PresetValidationError(where=where, path=path, message=_format_msg(err))) + out.append(PresetValidationError(where=where, path=path, message=_format_msg(err, known_modules))) return out @@ -125,8 +127,14 @@ def validate_preset(preset_dict: Any, module_loader=None) -> list[PresetValidati Validate a preset dict against BBOT's composite schema. Returns a list of `PresetValidationError` objects. An empty list means - the preset is valid. Errors from all layers are aggregated in a single - pass, so a user with multiple typos sees them all at once. + the preset is valid. Errors from all layers are aggregated, so a user + with multiple typos sees them all at once. + + **Side effect**: any custom `module_dirs` declared in the preset (either + at top level or inside `config`) are registered with `module_loader` so + their modules become known. Without this, modules from a custom dir + would be falsely reported as unknown. Idempotent — `add_module_dir` + short-circuits on already-loaded directories. Args: preset_dict: Preset as a plain dict (e.g. from `yaml.safe_load`). @@ -146,28 +154,37 @@ def validate_preset(preset_dict: Any, module_loader=None) -> list[PresetValidati module_loader = MODULE_LOADER + # Pre-load any custom module_dirs the preset declares so the composite + # schema includes their modules. Defensive iteration — bad shape gets + # surfaced by the schema pass below. + config_dict = preset_dict.get("config") + for source in ( + preset_dict.get("module_dirs"), + config_dict.get("module_dirs") if isinstance(config_dict, dict) else None, + ): + for d in source or []: + if isinstance(d, str): + module_loader.add_module_dir(d) + errors: list[PresetValidationError] = [] + known_modules = set(module_loader.all_module_choices) - # Single-pass validation against the composite schema + # Validate against the composite schema (rebuilt automatically if new + # module_dirs were just preloaded above). Closest-match suggestions + # for unknown module names are produced inside the formatter. try: module_loader.validation_schema.model_validate(preset_dict) except ValidationError as e: - errors.extend(_format_errors(e)) + errors.extend(_format_errors(e, known_modules=known_modules)) # Module names listed in top-level `modules`/`output_modules`/`exclude_modules` # aren't covered by the composite schema (they're a list of strings, not a - # nested mapping). Check them here. - known_modules = set(module_loader.all_module_choices) + # nested mapping). Check them explicitly, with the same closest-match hint. for key in ("modules", "output_modules", "exclude_modules"): for name in preset_dict.get(key) or []: if name not in known_modules: - errors.append( - PresetValidationError( - where="preset", - path=key, - message=f'Unknown module: "{name}"', - ) - ) + hint = get_closest_match(name, known_modules, msg="module") + errors.append(PresetValidationError(where="preset", path=key, message=hint)) return errors From 58c159d52e081e11f7bdf4ebd7081ee03014b32b Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Tue, 28 Apr 2026 13:48:49 -0400 Subject: [PATCH 05/15] preset tests --- bbot/test/test_step_1/test_validate_preset.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/bbot/test/test_step_1/test_validate_preset.py b/bbot/test/test_step_1/test_validate_preset.py index 128a5c1979..4fe9ca4121 100644 --- a/bbot/test/test_step_1/test_validate_preset.py +++ b/bbot/test/test_step_1/test_validate_preset.py @@ -33,7 +33,8 @@ def test_validate_preset_wrong_type(): def test_validate_preset_unknown_module(): errs = validate_preset({"modules": ["nucleii"]}) - assert any('Unknown module: "nucleii"' in str(e) for e in errs) + # closest-match suggestion: "Could not find module 'nucleii'. Did you mean 'nuclei'?" + assert any('"nucleii"' in str(e) and "nuclei" in str(e) for e in errs) def test_validate_preset_unknown_module_option(): @@ -57,7 +58,8 @@ def test_validate_preset_wrong_type_on_module_option(): def test_validate_preset_unknown_module_in_config(): """Unknown module name nested under config.modules gets a clean error.""" errs = validate_preset({"config": {"modules": {"nucleii": {"tgas": "x"}}}}) - assert any('Unknown module: "nucleii"' in str(e) for e in errs) + # closest-match: "Could not find module 'nucleii'. Did you mean 'nuclei'?" + assert any('"nucleii"' in str(e) and "nuclei" in str(e) for e in errs) def test_validate_preset_multiple_errors(): From 39c7f2c1202431a816b59ee8a04176c909b126af Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Wed, 29 Apr 2026 12:56:36 -0400 Subject: [PATCH 06/15] fix tests --- bbot/modules/lightfuzz/lightfuzz.py | 2 +- bbot/modules/templates/gitlab.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/bbot/modules/lightfuzz/lightfuzz.py b/bbot/modules/lightfuzz/lightfuzz.py index b8cc6b1f75..06b735ebb8 100644 --- a/bbot/modules/lightfuzz/lightfuzz.py +++ b/bbot/modules/lightfuzz/lightfuzz.py @@ -246,7 +246,7 @@ def help_text(self): import importlib submodules = {} - for submodule_name in self.options.get("enabled_submodules", []): + for submodule_name in self.config.get("enabled_submodules", []): try: submodule_module = importlib.import_module(f"bbot.modules.lightfuzz.submodules.{submodule_name}") submodule_class = getattr(submodule_module, submodule_name) diff --git a/bbot/modules/templates/gitlab.py b/bbot/modules/templates/gitlab.py index d53ac12596..a40832bc3a 100644 --- a/bbot/modules/templates/gitlab.py +++ b/bbot/modules/templates/gitlab.py @@ -16,8 +16,7 @@ class GitLabBaseModule(BaseModule): saas_domains = ["gitlab.com", "gitlab.org"] async def setup(self): - if self.options.get("api_key") is not None: - await self.require_api_key() + await self.require_api_key() return True async def handle_social(self, event): From 20af84e2d7aae2bc2f5959b47d965e0609c9a0ee Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Wed, 29 Apr 2026 15:24:03 -0400 Subject: [PATCH 07/15] cleanup, test fixes --- bbot/core/config/models.py | 15 ++++++++-- bbot/scanner/preset/args.py | 49 +++++++++++++------------------ bbot/scanner/preset/validate.py | 34 ++++++++++++++++----- bbot/scripts/docs.py | 8 ++--- bbot/test/test_step_1/test_cli.py | 8 +++-- 5 files changed, 67 insertions(+), 47 deletions(-) diff --git a/bbot/core/config/models.py b/bbot/core/config/models.py index 815e66d8c6..3ac3e83d62 100644 --- a/bbot/core/config/models.py +++ b/bbot/core/config/models.py @@ -108,9 +108,18 @@ class BaseModuleConfig(BaseModel): model_config = STRICT - batch_size: Optional[int] = None - module_threads: Optional[int] = None - module_timeout: Optional[int] = None + batch_size: Optional[int] = Field( + default=None, + description="The number of events to process in a single batch (only applies to batch modules)", + ) + module_threads: Optional[int] = Field( + default=None, + description="How many event handlers to run in parallel", + ) + module_timeout: Optional[int] = Field( + default=None, + description="Max time in seconds to spend handling each event or batch of events", + ) class BBOTConfig(BaseSettings): diff --git a/bbot/scanner/preset/args.py b/bbot/scanner/preset/args.py index ae51491d8f..3a0da5d248 100644 --- a/bbot/scanner/preset/args.py +++ b/bbot/scanner/preset/args.py @@ -1,11 +1,10 @@ -import re import yaml import logging import argparse from bbot.errors import * -from bbot.core.config.merge import dotted_get, dotted_set -from bbot.core.helpers.misc import chain_lists, get_closest_match, get_keys_in_dot_syntax +from bbot.core.config.merge import dotted_set +from bbot.core.helpers.misc import chain_lists def _parse_cli_value(raw: str): @@ -14,8 +13,12 @@ def _parse_cli_value(raw: str): YAML safe_load handles `true`/`false`/`null`/ints/floats and quoted strings the way users expect when they write `web.spider_distance=2` or - `modules.stdout.event_fields='[type, data]'`. + `modules.stdout.event_fields='[type, data]'`. An empty RHS (`-c key=`) is + treated as an empty string rather than None — matching the "clear this + value" intent users normally have. """ + if raw == "": + return "" try: return yaml.safe_load(raw) except yaml.YAMLError: @@ -47,19 +50,7 @@ def parse_dotted_cli(entries): log = logging.getLogger("bbot.presets.args") -universal_module_options = { - "batch_size": "The number of events to process in a single batch (only applies to batch modules)", - "module_threads": "How many event handlers to run in parallel", - "module_timeout": "Max time in seconds to spend handling each event or batch of events", -} - - class BBOTArgs: - # module config options to exclude from validation - exclude_from_validation = re.compile( - r".*modules\.[a-z0-9_]+\.(?:" + "|".join(universal_module_options.keys()) + ")$" - ) - scan_examples = [ ( "Subdomains", @@ -491,16 +482,16 @@ def sanitize_args(self): self.parsed.preset += ["fast"] def validate(self): - # validate config options - sentinel = object() - all_options = set(get_keys_in_dot_syntax(self.preset.core.default_config)) - for c in self.parsed.config: - c = c.split("=")[0].strip() - v = dotted_get(self.preset.core.default_config, c, default=sentinel) - # if option isn't in the default config - if v is sentinel: - # skip if it's excluded from validation - if self.exclude_from_validation.match(c): - continue - # otherwise, ensure it exists as a module option - raise ValidationError(get_closest_match(c, all_options, msg="config option")) + """ + Validate the CLI `-c key=value` arguments against the composite + preset schema. Catches typos like `bbot -c modules.shoudn.api_key=x` + with a closest-match suggestion. + """ + from .validate import validate_preset + + if not self.parsed.config: + return + cli_dict = parse_dotted_cli(self.parsed.config) + errs = validate_preset({"config": cli_dict}, module_loader=self.preset.module_loader) + if errs: + raise ValidationError("\n".join(str(e) for e in errs)) diff --git a/bbot/scanner/preset/validate.py b/bbot/scanner/preset/validate.py index 008c29728b..7f0a9e6afd 100644 --- a/bbot/scanner/preset/validate.py +++ b/bbot/scanner/preset/validate.py @@ -26,7 +26,7 @@ from pydantic import ValidationError -from bbot.core.helpers.misc import get_closest_match +from bbot.core.helpers.misc import get_closest_match, get_keys_in_dot_syntax log = logging.getLogger("bbot.presets.validate") @@ -72,7 +72,7 @@ def _classify_loc(loc: tuple) -> tuple[str, str]: return ("preset", ".".join(parts)) -def _format_msg(err: dict, known_modules: set | None = None) -> str: +def _format_msg(err: dict, known_modules: set | None = None, known_paths: set | None = None) -> str: kind = err["type"] input_value = err.get("input") loc = err["loc"] @@ -80,10 +80,17 @@ def _format_msg(err: dict, known_modules: set | None = None) -> str: path = ".".join(str(p) for p in loc) if kind == "extra_forbidden": - # Special-case unknown module name (config.modules.) so users get - # a suggestion rather than "Unknown option". + # Special-case unknown module name (config.modules.) — users get + # a suggestion drawn from the set of known module names. if len(loc) == 3 and loc[0] == "config" and loc[1] == "modules": return get_closest_match(field, known_modules or set(), msg="module") + # For everything else, suggest from the known dotted-path universe + # (`web.spier_distance` → `web.spider_distance`). + if known_paths: + # strip the leading "config." prefix when matching, since + # default_config dotted paths don't include it + lookup_path = ".".join(str(p) for p in loc[1:]) if loc and loc[0] == "config" else path + return get_closest_match(lookup_path, known_paths, msg="config option") msg = f"Unknown option: {field!r}" if isinstance(input_value, (str, int, bool, float)): msg += f" (value: {input_value!r})" @@ -114,11 +121,19 @@ def _format_msg(err: dict, known_modules: set | None = None) -> str: return err["msg"] if err.get("msg") else f"validation error at {path}" -def _format_errors(exc: ValidationError, known_modules: set | None = None) -> list[PresetValidationError]: +def _format_errors( + exc: ValidationError, + known_modules: set | None = None, + known_paths: set | None = None, +) -> list[PresetValidationError]: out: list[PresetValidationError] = [] for err in exc.errors(): where, path = _classify_loc(err["loc"]) - out.append(PresetValidationError(where=where, path=path, message=_format_msg(err, known_modules))) + out.append( + PresetValidationError( + where=where, path=path, message=_format_msg(err, known_modules, known_paths) + ) + ) return out @@ -168,14 +183,17 @@ def validate_preset(preset_dict: Any, module_loader=None) -> list[PresetValidati errors: list[PresetValidationError] = [] known_modules = set(module_loader.all_module_choices) + # Universe of valid dotted config paths, used for "did you mean ...?" + # suggestions on unknown global-config keys. + known_paths = set(get_keys_in_dot_syntax(module_loader.core.default_config)) # Validate against the composite schema (rebuilt automatically if new # module_dirs were just preloaded above). Closest-match suggestions - # for unknown module names are produced inside the formatter. + # for unknown module names + config options are produced inside the formatter. try: module_loader.validation_schema.model_validate(preset_dict) except ValidationError as e: - errors.extend(_format_errors(e, known_modules=known_modules)) + errors.extend(_format_errors(e, known_modules=known_modules, known_paths=known_paths)) # Module names listed in top-level `modules`/`output_modules`/`exclude_modules` # aren't covered by the composite schema (they're a list of strings, not a diff --git a/bbot/scripts/docs.py b/bbot/scripts/docs.py index 8da02e83bd..2b52f69316 100755 --- a/bbot/scripts/docs.py +++ b/bbot/scripts/docs.py @@ -182,12 +182,12 @@ def update_individual_module_options(): assert len(bbot_output_module_table.splitlines()) > 10 update_md_files("BBOT OUTPUT MODULES", bbot_output_module_table) - # BBOT universal module options - from bbot.scanner.preset.args import universal_module_options + # BBOT universal module options (sourced from BaseModuleConfig) + from bbot.core.config.models import BaseModuleConfig universal_module_options_table = "" - for option, description in universal_module_options.items(): - universal_module_options_table += f"**{option}**: {description}\n" + for name, field in BaseModuleConfig.model_fields.items(): + universal_module_options_table += f"**{name}**: {field.description}\n" update_md_files("BBOT UNIVERSAL MODULE OPTIONS", universal_module_options_table) # BBOT module options diff --git a/bbot/test/test_step_1/test_cli.py b/bbot/test/test_step_1/test_cli.py index 9824bd3fee..b10ad49974 100644 --- a/bbot/test/test_step_1/test_cli.py +++ b/bbot/test/test_step_1/test_cli.py @@ -466,13 +466,15 @@ def test_cli_config_validation(monkeypatch, caplog): monkeypatch.setattr(sys, "exit", lambda *args, **kwargs: True) monkeypatch.setattr(os, "_exit", lambda *args, **kwargs: True) - # incorrect module option + # incorrect module name nested under modules.* — surfaces as an unknown + # module with a closest-match suggestion (more useful than the legacy + # "Could not find config option ..." phrasing) caplog.clear() assert not caplog.text monkeypatch.setattr("sys.argv", ["bbot", "-c", "modules.ipnegibhor.num_bits=4"]) cli.main() - assert 'Could not find config option "modules.ipnegibhor.num_bits"' in caplog.text - assert 'Did you mean "modules.ipneighbor.num_bits"?' in caplog.text + assert 'Could not find module "ipnegibhor"' in caplog.text + assert 'Did you mean "ipneighbor"?' in caplog.text # incorrect global option caplog.clear() From 4972c121b837cd0073e40b54586bc3ecf350a2d9 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Thu, 30 Apr 2026 11:11:55 -0400 Subject: [PATCH 08/15] fix tests, again --- bbot/modules/output/neo4j.py | 7 ++----- bbot/scanner/preset/validate.py | 6 +----- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/bbot/modules/output/neo4j.py b/bbot/modules/output/neo4j.py index 21204f8f0e..3296111766 100644 --- a/bbot/modules/output/neo4j.py +++ b/bbot/modules/output/neo4j.py @@ -48,11 +48,8 @@ class Config(BaseModuleConfig): async def setup(self): try: self.driver = AsyncGraphDatabase.driver( - uri=self.config.get("uri", self.options["uri"]), - auth=( - self.config.get("username", self.options["username"]), - self.config.get("password", self.options["password"]), - ), + uri=self.config.get("uri"), + auth=(self.config.get("username"), self.config.get("password")), ) self.session = self.driver.session() await self.session.run("Match () Return 1 Limit 1") diff --git a/bbot/scanner/preset/validate.py b/bbot/scanner/preset/validate.py index 7f0a9e6afd..e81b8366ba 100644 --- a/bbot/scanner/preset/validate.py +++ b/bbot/scanner/preset/validate.py @@ -129,11 +129,7 @@ def _format_errors( out: list[PresetValidationError] = [] for err in exc.errors(): where, path = _classify_loc(err["loc"]) - out.append( - PresetValidationError( - where=where, path=path, message=_format_msg(err, known_modules, known_paths) - ) - ) + out.append(PresetValidationError(where=where, path=path, message=_format_msg(err, known_modules, known_paths))) return out From 9c1ea268cddd2cd70bdb59ca2a9e4030ef885a6f Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Thu, 30 Apr 2026 11:58:55 -0400 Subject: [PATCH 09/15] small improvements, rename deep_update -> deep_merge --- bbot/core/config/files.py | 4 ++-- bbot/core/config/merge.py | 25 ++++++++++++++++------ bbot/core/core.py | 8 +++---- bbot/test/bbot_fixtures.py | 4 ++-- bbot/test/test_step_2/module_tests/base.py | 6 +++--- 5 files changed, 29 insertions(+), 18 deletions(-) diff --git a/bbot/core/config/files.py b/bbot/core/config/files.py index b056fdd500..7814626f11 100644 --- a/bbot/core/config/files.py +++ b/bbot/core/config/files.py @@ -2,7 +2,7 @@ import yaml from pathlib import Path -from .merge import deep_update +from .merge import deep_merge from ...logger import log_to_stderr from ...errors import ConfigLoadError @@ -41,7 +41,7 @@ def _get_config(self, filename, name="config") -> dict: raise ConfigLoadError(f"Error parsing config at {filename}:\n\n{e}") def get_custom_config(self) -> dict: - return deep_update( + return deep_merge( self._get_config(self.config_filename, name="config"), self._get_config(self.secrets_filename, name="secrets"), ) diff --git a/bbot/core/config/merge.py b/bbot/core/config/merge.py index 735f803429..a59964995e 100644 --- a/bbot/core/config/merge.py +++ b/bbot/core/config/merge.py @@ -1,30 +1,34 @@ """ Deep-merge helpers replacing omegaconf's merge semantics. -`deep_update(a, b)` returns a new dict that is `a` with `b` merged in: nested +`deep_merge(a, b)` returns a new dict that is `a` with `b` merged in: nested dicts are merged recursively, leaf values (and lists) from `b` replace those in `a`. This matches `OmegaConf.merge(a, b)` for BBOT's preset layering use case. """ from __future__ import annotations +from copy import deepcopy from typing import Any -def deep_update(base: dict[str, Any], *updates: dict[str, Any]) -> dict[str, Any]: +def deep_merge(base: dict[str, Any] | None, *updates: dict[str, Any] | None) -> dict[str, Any]: """ Deep-merge one or more update dicts into a copy of `base`. Last wins on leaf conflicts; lists are replaced wholesale (not concatenated). + + The returned dict shares no mutable state with the inputs — nested dicts, + lists, and other mutable values are deep-copied as they're carried over. """ - result: dict[str, Any] = dict(base) if base else {} + result: dict[str, Any] = deepcopy(base) if base else {} for update in updates: if not update: continue for k, v in update.items(): if k in result and isinstance(result[k], dict) and isinstance(v, dict): - result[k] = deep_update(result[k], v) + result[k] = deep_merge(result[k], v) else: - result[k] = v + result[k] = deepcopy(v) return result @@ -32,6 +36,8 @@ def dotted_get(data: dict[str, Any], path: str, default: Any = None) -> Any: """ Look up a dotted path in a nested dict. + Note: keys containing literal `.` are not addressable (no escape syntax). + >>> dotted_get({"a": {"b": {"c": 1}}}, "a.b.c") 1 >>> dotted_get({"a": 1}, "a.b.c", default="x") @@ -49,6 +55,10 @@ def dotted_set(data: dict[str, Any], path: str, value: Any) -> None: """ Set a dotted path in a nested dict, creating intermediate dicts as needed. + Non-dict intermediates are silently replaced. This is intentional — + callers (CLI parsing) feed the result through pydantic validation, which + surfaces any resulting type mismatch. + >>> d = {} >>> dotted_set(d, "a.b.c", 1) >>> d @@ -65,7 +75,8 @@ def dotted_set(data: dict[str, Any], path: str, value: Any) -> None: def iter_dotted_paths(data: dict[str, Any], prefix: str = "") -> list[str]: """ - Yield every dotted leaf path in a nested dict. + Return every dotted leaf path in a nested dict. Empty dicts are treated + as leaves (so they round-trip through dotted_get/dotted_set). >>> iter_dotted_paths({"a": 1, "b": {"c": 2}}) ['a', 'b.c'] @@ -80,4 +91,4 @@ def iter_dotted_paths(data: dict[str, Any], prefix: str = "") -> list[str]: return paths -__all__ = ["deep_update", "dotted_get", "dotted_set", "iter_dotted_paths"] +__all__ = ["deep_merge", "dotted_get", "dotted_set", "iter_dotted_paths"] diff --git a/bbot/core/core.py b/bbot/core/core.py index 3084b16da6..26eda685ba 100644 --- a/bbot/core/core.py +++ b/bbot/core/core.py @@ -4,7 +4,7 @@ from pathlib import Path from bbot.errors import BBOTError -from .config.merge import deep_update +from .config.merge import deep_merge from .multiprocess import SHARED_INTERPRETER_STATE @@ -91,7 +91,7 @@ def config(self) -> dict: Any new values should be added to custom_config. """ if self._config is None: - self._config = deep_update(self.default_config, self.custom_config) + self._config = deep_merge(self.default_config, self.custom_config) return self._config @property @@ -158,11 +158,11 @@ def secrets_only_config(self, config): def merge_custom(self, config): """Merge a config dict into the custom config.""" - self.custom_config = deep_update(self.custom_config, dict(config) if config else {}) + self.custom_config = deep_merge(self.custom_config, dict(config) if config else {}) def merge_default(self, config): """Merge a config dict into the default config.""" - self.default_config = deep_update(self.default_config, dict(config) if config else {}) + self.default_config = deep_merge(self.default_config, dict(config) if config else {}) def copy(self): """ diff --git a/bbot/test/bbot_fixtures.py b/bbot/test/bbot_fixtures.py index afdb24c016..eb1ec3c61c 100644 --- a/bbot/test/bbot_fixtures.py +++ b/bbot/test/bbot_fixtures.py @@ -12,7 +12,7 @@ from bbot.errors import * # noqa: F401 from bbot.core import CORE -from bbot.core.config.merge import deep_update +from bbot.core.config.merge import deep_merge from bbot.scanner import Preset from bbot.core.helpers.misc import mkdir, rand_string @@ -49,7 +49,7 @@ def tempapkfile(): @pytest.fixture def clean_default_config(monkeypatch): - clean_config = deep_update( + clean_config = deep_merge( CORE.files_config.get_default_config(), {"modules": DEFAULT_PRESET.module_loader.configs()}, ) diff --git a/bbot/test/test_step_2/module_tests/base.py b/bbot/test/test_step_2/module_tests/base.py index 0a30fc0054..0ee6971320 100644 --- a/bbot/test/test_step_2/module_tests/base.py +++ b/bbot/test/test_step_2/module_tests/base.py @@ -5,7 +5,7 @@ from ...bbot_fixtures import * from bbot.scanner import Scanner -from bbot.core.config.merge import deep_update +from bbot.core.config.merge import deep_merge from bbot.core.helpers.misc import rand_string log = logging.getLogger("bbot.test.modules") @@ -28,7 +28,7 @@ def __init__( self, module_test_base, httpx_mock, httpserver, httpserver_ssl, monkeypatch, request, caplog, capsys ): self.name = module_test_base.name - self.config = deep_update(dict(CORE.config), dict(module_test_base.config_overrides)) + self.config = deep_merge(dict(CORE.config), dict(module_test_base.config_overrides)) self.caplog = caplog self.capsys = capsys @@ -51,7 +51,7 @@ def __init__( if module_type == "output": output_modules.append(module) elif module_type == "internal" and not module == "dnsresolve": - self.config = deep_update(self.config, {module: True}) + self.config = deep_merge(self.config, {module: True}) seeds = module_test_base.seeds or None From bce3e4b939221654d61640120f2d76e74a045563 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Thu, 30 Apr 2026 13:52:42 -0400 Subject: [PATCH 10/15] remove env var interpolation docs --- docs/scanning/presets.md | 26 +------------------------- 1 file changed, 1 insertion(+), 25 deletions(-) diff --git a/docs/scanning/presets.md b/docs/scanning/presets.md index fc5e9426f8..dd176ede2f 100644 --- a/docs/scanning/presets.md +++ b/docs/scanning/presets.md @@ -122,7 +122,7 @@ bbot -p ./mypreset.yml --current-preset ## Advanced Usage -BBOT Presets support advanced features like file-based targets, environment variable substitution, and custom conditions. +BBOT Presets support advanced features like file-based targets, custom modules, and custom conditions. ### Files as Targets @@ -154,30 +154,6 @@ module_dirs: - /home/user/custom_modules ``` -### Environment Variables - -You can insert environment variables into your preset like this: `${env:}`: - -```yaml title="my_nuclei.yml" -description: Do a nuclei scan - -target: - - evilcorp.com - -modules: - - nuclei - -config: - modules: - nuclei: - # allow the nuclei templates to be specified at runtime via an environment variable - tags: ${env:NUCLEI_TAGS} -``` - -```bash -NUCLEI_TAGS=apache,nginx bbot -p ./my_nuclei.yml -``` - ### Conditions Sometimes, you might need to add custom logic to a preset. BBOT supports this via `conditions`. The `conditions` attribute allows you to specify a list of custom conditions that will be evaluated before the scan starts. This is useful for performing last-minute sanity checks, or changing the behavior of the scan based on custom criteria. From e48e570f642867219046210b15a61a2451b3ccd4 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Fri, 1 May 2026 11:37:30 -0400 Subject: [PATCH 11/15] bring back baddns case insensitivity --- bbot/modules/baddns.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/bbot/modules/baddns.py b/bbot/modules/baddns.py index 45d2f28a93..713590ed9d 100644 --- a/bbot/modules/baddns.py +++ b/bbot/modules/baddns.py @@ -4,7 +4,7 @@ from .base import BaseModule import logging -from pydantic import Field +from pydantic import Field, field_validator from bbot.core.config.models import BaseModuleConfig SEVERITY_LEVELS = ("INFO", "LOW", "MEDIUM", "HIGH", "CRITICAL") @@ -64,6 +64,11 @@ class Config(BaseModuleConfig): description="A list of submodules to enable. Empty list (default) enables CNAME, TXT and MX Only", ) + @field_validator("min_severity", "min_confidence", mode="before") + @classmethod + def _normalize_case(cls, v): + return v.upper() if isinstance(v, str) else v + module_threads = 8 deps_pip = ["baddns~=2.1.0"] From eca363bb10e3ce6a1258d9e554bbda8de368f113 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Fri, 1 May 2026 12:11:38 -0400 Subject: [PATCH 12/15] allow field validators --- bbot/core/modules.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/bbot/core/modules.py b/bbot/core/modules.py index adeca4862f..5756848d92 100644 --- a/bbot/core/modules.py +++ b/bbot/core/modules.py @@ -61,15 +61,15 @@ def _exec_config_class(source: str, module_name: str): """ Execute a `class Config(BaseModuleConfig):` snippet in a controlled namespace and return the resulting class. The namespace provides exactly - what a Config block is allowed to reference: the typing primitives, the - pydantic `Field` factory, and `BaseModuleConfig`. + what a Config block is allowed to reference: the typing primitives, + pydantic's `Field` factory and validator decorators, and `BaseModuleConfig`. This replaces parsing annotations as strings: pydantic handles every valid type expression (`Optional[str]`, `Literal["a", "b"]`, `list[Union[int, str]]`, …) without any hand-rolled resolver. """ from typing import Any, Dict, List, Literal, Optional, Set, Tuple, Union - from pydantic import Field + from pydantic import AfterValidator, BeforeValidator, Field, field_validator, model_validator from bbot.core.config.models import BaseModuleConfig namespace: dict = { @@ -82,6 +82,10 @@ def _exec_config_class(source: str, module_name: str): "Tuple": Tuple, "Union": Union, "Field": Field, + "field_validator": field_validator, + "model_validator": model_validator, + "BeforeValidator": BeforeValidator, + "AfterValidator": AfterValidator, "BaseModuleConfig": BaseModuleConfig, } try: @@ -90,7 +94,8 @@ def _exec_config_class(source: str, module_name: str): raise BBOTError( f'module "{module_name}" has an invalid Config class ({type(e).__name__}: {e}). ' "Config blocks may only reference: Optional, Union, Literal, Any, List, Dict, Tuple, Set, " - "Field, BaseModuleConfig, and Python builtins." + "Field, field_validator, model_validator, BeforeValidator, AfterValidator, BaseModuleConfig, " + "and Python builtins." ) from e cfg = namespace.get("Config") if cfg is None: From 6bc7928fa817645774463900dc297b6015f4d26a Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Mon, 4 May 2026 10:31:13 -0400 Subject: [PATCH 13/15] preset schema --- bbot/scanner/__init__.py | 34 ++++++++++++++++++++++++++++++--- bbot/scanner/preset/__init__.py | 28 ++++++++++++++++++++++++++- 2 files changed, 58 insertions(+), 4 deletions(-) diff --git a/bbot/scanner/__init__.py b/bbot/scanner/__init__.py index 2f5970db48..24bedbe1a4 100644 --- a/bbot/scanner/__init__.py +++ b/bbot/scanner/__init__.py @@ -1,5 +1,33 @@ -from .preset import Preset -from .preset import PresetValidationError, validate_preset, validate_preset_file +from .preset import ( + BaseModuleConfig, + BBOTConfig, + DepsConfig, + DepsToolConfig, + DnsConfig, + EngineConfig, + Preset, + PresetSchema, + PresetValidationError, + ScopeConfig, + WebConfig, + validate_preset, + validate_preset_file, +) from .scanner import Scanner -__all__ = ["Preset", "PresetValidationError", "Scanner", "validate_preset", "validate_preset_file"] +__all__ = [ + "BBOTConfig", + "BaseModuleConfig", + "DepsConfig", + "DepsToolConfig", + "DnsConfig", + "EngineConfig", + "Preset", + "PresetSchema", + "PresetValidationError", + "Scanner", + "ScopeConfig", + "WebConfig", + "validate_preset", + "validate_preset_file", +] diff --git a/bbot/scanner/preset/__init__.py b/bbot/scanner/preset/__init__.py index e80a0f5683..fb2e423dfd 100644 --- a/bbot/scanner/preset/__init__.py +++ b/bbot/scanner/preset/__init__.py @@ -1,4 +1,30 @@ +from bbot.core.config.models import ( + BaseModuleConfig, + BBOTConfig, + DepsConfig, + DepsToolConfig, + DnsConfig, + EngineConfig, + PresetSchema, + ScopeConfig, + WebConfig, +) + from .preset import Preset from .validate import PresetValidationError, validate_preset, validate_preset_file -__all__ = ["Preset", "PresetValidationError", "validate_preset", "validate_preset_file"] +__all__ = [ + "BBOTConfig", + "BaseModuleConfig", + "DepsConfig", + "DepsToolConfig", + "DnsConfig", + "EngineConfig", + "Preset", + "PresetSchema", + "PresetValidationError", + "ScopeConfig", + "WebConfig", + "validate_preset", + "validate_preset_file", +] From d04ef70bdc3e692f5228be11a60e931ee25db998 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Tue, 5 May 2026 10:03:36 -0400 Subject: [PATCH 14/15] mandatory and sensitive fields --- bbot/core/config/models.py | 140 +++++++++++++++++++++- bbot/core/core.py | 57 ++++----- bbot/core/modules.py | 65 ++++++++-- bbot/modules/anubisdb.py | 3 +- bbot/modules/apkpure.py | 3 +- bbot/modules/baddns_direct.py | 3 +- bbot/modules/baddns_zone.py | 3 +- bbot/modules/badsecrets.py | 3 +- bbot/modules/base.py | 12 +- bbot/modules/bevigil.py | 6 +- bbot/modules/bucket_amazon.py | 3 +- bbot/modules/bucket_digitalocean.py | 3 +- bbot/modules/bucket_file_enum.py | 3 +- bbot/modules/bucket_firebase.py | 3 +- bbot/modules/bucket_google.py | 3 +- bbot/modules/bucket_microsoft.py | 3 +- bbot/modules/bufferoverrun.py | 6 +- bbot/modules/builtwith.py | 6 +- bbot/modules/c99.py | 6 +- bbot/modules/censys_dns.py | 8 +- bbot/modules/censys_ip.py | 8 +- bbot/modules/chaos.py | 6 +- bbot/modules/credshed.py | 10 +- bbot/modules/dehashed.py | 6 +- bbot/modules/dnsbimi.py | 3 +- bbot/modules/dnsbrute.py | 3 +- bbot/modules/dnsbrute_mutations.py | 3 +- bbot/modules/dnscaa.py | 3 +- bbot/modules/dnscommonsrv.py | 3 +- bbot/modules/dnstlsrpt.py | 3 +- bbot/modules/docker_pull.py | 3 +- bbot/modules/ffuf.py | 3 +- bbot/modules/ffuf_shortnames.py | 3 +- bbot/modules/filedownload.py | 3 +- bbot/modules/fingerprintx.py | 3 +- bbot/modules/fullhunt.py | 6 +- bbot/modules/git_clone.py | 5 +- bbot/modules/gitdumper.py | 3 +- bbot/modules/github_codesearch.py | 6 +- bbot/modules/github_org.py | 5 +- bbot/modules/github_usersearch.py | 6 +- bbot/modules/github_workflows.py | 6 +- bbot/modules/gitlab_com.py | 5 +- bbot/modules/gitlab_onprem.py | 5 +- bbot/modules/gowitness.py | 3 +- bbot/modules/graphql_introspection.py | 3 +- bbot/modules/httpx.py | 3 +- bbot/modules/hunterio.py | 12 +- bbot/modules/iis_shortnames.py | 3 +- bbot/modules/internal/excavate.py | 3 +- bbot/modules/internal/speculate.py | 3 +- bbot/modules/ip2location.py | 6 +- bbot/modules/ipneighbor.py | 3 +- bbot/modules/ipstack.py | 12 +- bbot/modules/jadx.py | 3 +- bbot/modules/kreuzberg.py | 3 +- bbot/modules/leakix.py | 5 +- bbot/modules/legba.py | 3 +- bbot/modules/lightfuzz/lightfuzz.py | 3 +- bbot/modules/medusa.py | 3 +- bbot/modules/ntlm.py | 3 +- bbot/modules/nuclei.py | 3 +- bbot/modules/oauth.py | 3 +- bbot/modules/otx.py | 6 +- bbot/modules/output/asset_inventory.py | 3 +- bbot/modules/output/csv.py | 3 +- bbot/modules/output/discord.py | 5 +- bbot/modules/output/elastic.py | 7 +- bbot/modules/output/emails.py | 3 +- bbot/modules/output/http.py | 9 +- bbot/modules/output/json.py | 3 +- bbot/modules/output/kafka.py | 3 +- bbot/modules/output/mongo.py | 9 +- bbot/modules/output/mysql.py | 7 +- bbot/modules/output/nats.py | 3 +- bbot/modules/output/neo4j.py | 7 +- bbot/modules/output/postgres.py | 7 +- bbot/modules/output/rabbitmq.py | 5 +- bbot/modules/output/slack.py | 5 +- bbot/modules/output/splunk.py | 5 +- bbot/modules/output/sqlite.py | 3 +- bbot/modules/output/subdomains.py | 3 +- bbot/modules/output/teams.py | 5 +- bbot/modules/output/txt.py | 3 +- bbot/modules/output/web_parameters.py | 3 +- bbot/modules/output/web_report.py | 3 +- bbot/modules/output/websocket.py | 5 +- bbot/modules/output/zeromq.py | 3 +- bbot/modules/paramminer_cookies.py | 3 +- bbot/modules/paramminer_getparams.py | 3 +- bbot/modules/paramminer_headers.py | 3 +- bbot/modules/passivetotal.py | 8 +- bbot/modules/pgp.py | 3 +- bbot/modules/portfilter.py | 3 +- bbot/modules/portscan.py | 3 +- bbot/modules/postman.py | 5 +- bbot/modules/postman_download.py | 5 +- bbot/modules/robots.py | 3 +- bbot/modules/securitytrails.py | 6 +- bbot/modules/securitytxt.py | 3 +- bbot/modules/shodan_dns.py | 12 +- bbot/modules/shodan_enterprise.py | 6 +- bbot/modules/shodan_idb.py | 3 +- bbot/modules/sslcert.py | 3 +- bbot/modules/subdomainradar.py | 3 +- bbot/modules/telerik.py | 3 +- bbot/modules/templates/postman.py | 1 - bbot/modules/templates/subdomain_enum.py | 2 +- bbot/modules/trajan.py | 17 +-- bbot/modules/trickest.py | 12 +- bbot/modules/trufflehog.py | 3 +- bbot/modules/url_manipulation.py | 3 +- bbot/modules/urlscan.py | 3 +- bbot/modules/virustotal.py | 6 +- bbot/modules/wafw00f.py | 3 +- bbot/modules/wayback.py | 3 +- bbot/modules/wpscan.py | 5 +- bbot/test/test_step_1/test_presets.py | 10 +- docs/data/chord_graph/entities.json | 4 +- docs/data/chord_graph/rels.json | 5 + docs/dev/preset_validation.md | 146 +++++++++++++++++++++++ mkdocs.yml | 1 + 122 files changed, 574 insertions(+), 375 deletions(-) create mode 100644 docs/dev/preset_validation.md diff --git a/bbot/core/config/models.py b/bbot/core/config/models.py index 3ac3e83d62..5f69456b42 100644 --- a/bbot/core/config/models.py +++ b/bbot/core/config/models.py @@ -15,13 +15,142 @@ from typing import Any, Literal, Optional -from pydantic import BaseModel, ConfigDict, Field +from pydantic import BaseModel, ConfigDict +from pydantic import Field as _PydanticField +from pydantic_core import PydanticUndefined from pydantic_settings import BaseSettings, SettingsConfigDict STRICT = ConfigDict(extra="forbid") +def Field(default=PydanticUndefined, *, sensitive: bool = False, mandatory: bool = False, **kwargs): + """ + Drop-in replacement for `pydantic.Field` that records two BBOT-specific + flags as field metadata: + + - `sensitive=True`: value should be redacted when serializing configs + (api keys, passwords, http cookies, …). + - `mandatory=True`: option must be supplied for the module to function; + drives the "Needs API Key" column in `bbot -l` and the + `BaseModule.auth_required` property. + + Both flags are stashed under `json_schema_extra` so pydantic preserves + them on `FieldInfo.json_schema_extra` (and in any generated JSON schema) + without affecting validation. All other arguments pass through unchanged. + """ + extra = dict(kwargs.pop("json_schema_extra", None) or {}) + if sensitive: + extra["sensitive"] = True + if mandatory: + extra["mandatory"] = True + if extra: + kwargs["json_schema_extra"] = extra + return _PydanticField(default, **kwargs) + + +def field_flags(field) -> dict: + """Return the BBOT flags dict for a pydantic FieldInfo (empty if none).""" + extra = getattr(field, "json_schema_extra", None) + return dict(extra) if isinstance(extra, dict) else {} + + +def is_sensitive(field) -> bool: + return bool(field_flags(field).get("sensitive")) + + +def is_mandatory(field) -> bool: + return bool(field_flags(field).get("mandatory")) + + +def _unwrap_optional(annotation): + """Strip a single `Optional[X]` / `Union[X, None]` wrapper, return X. Pass-through otherwise.""" + import typing + + origin = typing.get_origin(annotation) + if origin is typing.Union: + args = [a for a in typing.get_args(annotation) if a is not type(None)] + if len(args) == 1: + return args[0] + return annotation + + +def _resolve_field(model, key): + """Resolve `key` against `model.model_fields`, honoring `Field(alias=...)`.""" + fields = getattr(model, "model_fields", None) + if not fields: + return None + if key in fields: + return fields[key] + for f in fields.values(): + if getattr(f, "alias", None) == key: + return f + return None + + +def _field_submodel(field): + """If `field`'s annotation is a `BaseModel` subclass, return it; else None.""" + if field is None: + return None + ann = _unwrap_optional(field.annotation) + if isinstance(ann, type) and issubclass(ann, BaseModel): + return ann + return None + + +def partition_sensitive_config(config, model, *, keep_sensitive: bool): + """ + Walk `config` (a dict) alongside the pydantic `model`, returning a copy + that either drops or extracts every `sensitive=True` field. + + - `keep_sensitive=False` -> caller wants the public, non-secret view (used + by `BBOTCore.no_secrets_config`). + - `keep_sensitive=True` -> caller wants only the secrets (used by + `BBOTCore.secrets_only_config` to materialize `~/.config/bbot/secrets.yml`). + + Unknown keys (no matching field in the schema) pass through unchanged when + redacting and are dropped when extracting secrets-only. + """ + import copy as _copy + + if not isinstance(config, dict) or model is None: + return _copy.deepcopy(config) if keep_sensitive is False else {} + + out: dict = {} + for key, val in config.items(): + field = _resolve_field(model, key) + sub_model = _field_submodel(field) + + if sub_model is not None and isinstance(val, dict): + child = partition_sensitive_config(val, sub_model, keep_sensitive=keep_sensitive) + # When redacting, preserve the parent key even if its body was + # entirely sensitive — matches the prior `clean_dict` behavior. + # When extracting secrets-only, drop empty branches so the result + # is just the secrets that exist. + if keep_sensitive: + if child: + out[key] = child + else: + out[key] = child + continue + + if field is None: + # No matching schema field — pass through unchanged when redacting, + # drop when extracting secrets-only. + if not keep_sensitive: + out[key] = _copy.deepcopy(val) + continue + + sensitive = is_sensitive(field) + if keep_sensitive: + if sensitive: + out[key] = _copy.deepcopy(val) + else: + if not sensitive: + out[key] = _copy.deepcopy(val) + return out + + class ScopeConfig(BaseModel): model_config = STRICT @@ -64,7 +193,7 @@ class WebConfig(BaseModel): http_timeout: Optional[int] = None httpx_timeout: Optional[int] = None http_headers: Optional[dict[str, str]] = None - http_cookies: Optional[dict[str, str]] = None + http_cookies: Optional[dict[str, str]] = Field(default=None, sensitive=True) api_retries: Optional[int] = None http_retries: Optional[int] = None httpx_retries: Optional[int] = None @@ -184,7 +313,7 @@ class BBOTConfig(BaseSettings): # Interactsh interactsh_server: Optional[str] = None - interactsh_token: Optional[str] = None + interactsh_token: Optional[str] = Field(default=None, sensitive=True) interactsh_disable: Optional[bool] = None # Per-module configs — validated separately, per-module, against each @@ -243,7 +372,12 @@ class PresetSchema(BaseModel): "DepsToolConfig", "DnsConfig", "EngineConfig", + "Field", "PresetSchema", "ScopeConfig", "WebConfig", + "field_flags", + "is_mandatory", + "is_sensitive", + "partition_sensitive_config", ] diff --git a/bbot/core/core.py b/bbot/core/core.py index 26eda685ba..feeccac0f5 100644 --- a/bbot/core/core.py +++ b/bbot/core/core.py @@ -25,11 +25,6 @@ class BBOTCore: - load quickly """ - # used for filtering out sensitive config values - secrets_strings = ["api_key", "username", "password", "token", "secret", "_id"] - # don't filter/remove entries under this key - secrets_exclude_keys = ["modules"] - def __init__(self): self._logger = None self._files_config = None @@ -131,30 +126,38 @@ def custom_config(self, value: dict): self._custom_config = dict(value) if value else {} def no_secrets_config(self, config): - """Return a copy of the config with secret-looking keys removed.""" - from .helpers.misc import clean_dict - - if not isinstance(config, dict): - config = deepcopy(config) - return clean_dict( - config, - *self.secrets_strings, - fuzzy=True, - exclude_keys=self.secrets_exclude_keys, - ) + """Return a copy of `config` with every `sensitive=True` field removed. + + Sensitivity is read from the per-field `json_schema_extra["sensitive"]` + flag declared on `BBOTConfig` (and each module's `class Config`). + Module-level redaction uses the composite schema built lazily by + `MODULE_LOADER.config_schema`; if a key isn't covered by any schema + (e.g. an unknown module), it passes through unchanged. + """ + from .config.models import partition_sensitive_config + + return partition_sensitive_config(config, self._config_schema(), keep_sensitive=False) def secrets_only_config(self, config): - """Return a copy of the config containing only secret-looking keys.""" - from .helpers.misc import filter_dict - - if not isinstance(config, dict): - config = deepcopy(config) - return filter_dict( - config, - *self.secrets_strings, - fuzzy=True, - exclude_keys=self.secrets_exclude_keys, - ) + """Return a copy of `config` containing only `sensitive=True` fields. + + Inverse of `no_secrets_config()`. Useful for splitting a merged config + into a public `bbot.yml` and a private `secrets.yml`. + """ + from .config.models import partition_sensitive_config + + return partition_sensitive_config(config, self._config_schema(), keep_sensitive=True) + + def _config_schema(self): + """Resolve the runtime BBOTConfig schema (with per-module configs).""" + try: + from bbot.core.modules import MODULE_LOADER + + return MODULE_LOADER.config_schema + except Exception: + from .config.models import BBOTConfig + + return BBOTConfig def merge_custom(self, config): """Merge a config dict into the custom config.""" diff --git a/bbot/core/modules.py b/bbot/core/modules.py index 5756848d92..e0b1825644 100644 --- a/bbot/core/modules.py +++ b/bbot/core/modules.py @@ -69,8 +69,8 @@ def _exec_config_class(source: str, module_name: str): `list[Union[int, str]]`, …) without any hand-rolled resolver. """ from typing import Any, Dict, List, Literal, Optional, Set, Tuple, Union - from pydantic import AfterValidator, BeforeValidator, Field, field_validator, model_validator - from bbot.core.config.models import BaseModuleConfig + from pydantic import AfterValidator, BeforeValidator, field_validator, model_validator + from bbot.core.config.models import BaseModuleConfig, Field namespace: dict = { "Any": Any, @@ -167,17 +167,20 @@ def _build_validation_schema(preloaded: dict): return FullPresetSchema -def _extract_pydantic_config(config_class: ast.ClassDef) -> tuple[dict, dict]: +def _extract_pydantic_config(config_class: ast.ClassDef) -> tuple[dict, dict, set, set]: """ Walk a `class Config(BaseModuleConfig):` block and extract - `(defaults, descriptions)` dicts — cheap metadata used for `bbot -l` and - similar listing paths, without importing or exec-ing anything. + `(defaults, descriptions, sensitive, mandatory)` — cheap metadata used + for `bbot -l` and similar listing paths, without importing or exec-ing + anything. The actual typed pydantic class is built later via `_exec_config_class` on the captured source text. """ defaults: dict = {} descriptions: dict = {} + sensitive: set = set() + mandatory: set = set() for node in config_class.body: # `model_config = ConfigDict(...)` etc. are plain assigns, not typed — skip. if not isinstance(node, ast.AnnAssign) or not isinstance(node.target, ast.Name): @@ -188,10 +191,12 @@ def _extract_pydantic_config(config_class: ast.ClassDef) -> tuple[dict, dict]: default = _UNEVALUATED description = "" + is_sensitive = False + is_mandatory = False value = node.value if isinstance(value, ast.Call) and isinstance(value.func, ast.Name) and value.func.id == "Field": - # Field(default, description="...", default_factory=..., ...) + # Field(default, description="...", default_factory=..., sensitive=..., mandatory=..., ...) # - first positional arg is the default, if given if value.args: default = _eval_ast_default(value.args[0]) @@ -203,6 +208,18 @@ def _extract_pydantic_config(config_class: ast.ClassDef) -> tuple[dict, dict]: elif kw.arg == "description": with suppress(ValueError, TypeError, SyntaxError): description = ast.literal_eval(kw.value) + elif kw.arg == "sensitive": + with suppress(ValueError, TypeError, SyntaxError): + is_sensitive = bool(ast.literal_eval(kw.value)) + elif kw.arg == "mandatory": + with suppress(ValueError, TypeError, SyntaxError): + is_mandatory = bool(ast.literal_eval(kw.value)) + elif kw.arg == "json_schema_extra": + with suppress(ValueError, TypeError, SyntaxError): + extra = ast.literal_eval(kw.value) + if isinstance(extra, dict): + is_sensitive = is_sensitive or bool(extra.get("sensitive")) + is_mandatory = is_mandatory or bool(extra.get("mandatory")) elif value is not None: default = _eval_ast_default(value) @@ -211,7 +228,11 @@ def _extract_pydantic_config(config_class: ast.ClassDef) -> tuple[dict, dict]: default = None defaults[name] = default descriptions[name] = description - return defaults, descriptions + if is_sensitive: + sensitive.add(name) + if is_mandatory: + mandatory.add(name) + return defaults, descriptions, sensitive, mandatory class ModuleLoader: @@ -452,6 +473,25 @@ def validation_schema(self): self._validation_schema = _build_validation_schema(self._preloaded) return self._validation_schema + @property + def config_schema(self): + """ + The runtime `BBOTConfig` schema with per-module configs grafted in. + + This is `validation_schema.config` (i.e. `FullBBOTConfig`) and is the + right model to walk a config dict against — used by + `BBOTCore.no_secrets_config()` and `BBOTCore.secrets_only_config()` to + partition sensitive fields. + """ + from bbot.core.config.models import _unwrap_optional + + field = self.validation_schema.model_fields.get("config") + if field is None: + from bbot.core.config.models import BBOTConfig + + return BBOTConfig + return _unwrap_optional(field.annotation) + def find_and_replace(self, **kwargs): self.__preloaded = search_format_dict(self.__preloaded, **kwargs) self._shared_deps = search_format_dict(self._shared_deps, **kwargs) @@ -520,6 +560,8 @@ def preload_module(self, module_file): ansible_tasks = [] config = {} options_desc = {} + options_sensitive: set = set() + options_mandatory: set = set() config_source: str | None = None disable_auto_module_deps = False with open(module_file) as f: @@ -549,9 +591,11 @@ def preload_module(self, module_file): for class_attr in root_element.body: # nested `class Config(BaseModuleConfig): ...` — the module's config schema if type(class_attr) == ast.ClassDef and class_attr.name == "Config": - config_fields, config_descs = _extract_pydantic_config(class_attr) + config_fields, config_descs, config_sens, config_mand = _extract_pydantic_config(class_attr) config.update(config_fields) options_desc.update(config_descs) + options_sensitive.update(config_sens) + options_mandatory.update(config_mand) # capture the class source verbatim; schema build re-execs it config_source = ast.get_source_segment(python_code, class_attr) continue @@ -635,6 +679,8 @@ def preload_module(self, module_file): "meta": meta, "config": config, "options_desc": options_desc, + "options_sensitive": sorted(options_sensitive), + "options_mandatory": sorted(options_mandatory), "config_source": config_source, "hash": module_hash, "deps": { @@ -770,9 +816,8 @@ def modules_table(self, modules=None, mod_type=None, include_author=False, inclu consumed_events = sorted(preloaded.get("watched_events", [])) produced_events = sorted(preloaded.get("produced_events", [])) flags = sorted(preloaded.get("flags", [])) - api_key_required = "" meta = preloaded.get("meta", {}) - api_key_required = "Yes" if meta.get("auth_required", False) else "No" + api_key_required = "Yes" if preloaded.get("options_mandatory") else "No" description = meta.get("description", "") row = [ module_name, diff --git a/bbot/modules/anubisdb.py b/bbot/modules/anubisdb.py index 2e1688a489..4e4b869ad9 100644 --- a/bbot/modules/anubisdb.py +++ b/bbot/modules/anubisdb.py @@ -1,6 +1,5 @@ from bbot.modules.templates.subdomain_enum import subdomain_enum -from pydantic import Field -from bbot.core.config.models import BaseModuleConfig +from bbot.core.config.models import BaseModuleConfig, Field class anubisdb(subdomain_enum): diff --git a/bbot/modules/apkpure.py b/bbot/modules/apkpure.py index 602c048725..41fbda3da3 100644 --- a/bbot/modules/apkpure.py +++ b/bbot/modules/apkpure.py @@ -1,8 +1,7 @@ import re from pathlib import Path from bbot.modules.base import BaseModule -from pydantic import Field -from bbot.core.config.models import BaseModuleConfig +from bbot.core.config.models import BaseModuleConfig, Field class apkpure(BaseModule): diff --git a/bbot/modules/baddns_direct.py b/bbot/modules/baddns_direct.py index 6f9a7ede3a..82fea18488 100644 --- a/bbot/modules/baddns_direct.py +++ b/bbot/modules/baddns_direct.py @@ -1,6 +1,5 @@ from .baddns import baddns as baddns_module -from pydantic import Field -from bbot.core.config.models import BaseModuleConfig +from bbot.core.config.models import BaseModuleConfig, Field class baddns_direct(baddns_module): diff --git a/bbot/modules/baddns_zone.py b/bbot/modules/baddns_zone.py index 25eda1695d..76be6cc692 100644 --- a/bbot/modules/baddns_zone.py +++ b/bbot/modules/baddns_zone.py @@ -1,6 +1,5 @@ from .baddns import baddns as baddns_module -from pydantic import Field -from bbot.core.config.models import BaseModuleConfig +from bbot.core.config.models import BaseModuleConfig, Field class baddns_zone(baddns_module): diff --git a/bbot/modules/badsecrets.py b/bbot/modules/badsecrets.py index f9847e8fb1..59b9436019 100644 --- a/bbot/modules/badsecrets.py +++ b/bbot/modules/badsecrets.py @@ -2,8 +2,7 @@ from pathlib import Path from .base import BaseModule from badsecrets.base import carve_all_modules -from pydantic import Field -from bbot.core.config.models import BaseModuleConfig +from bbot.core.config.models import BaseModuleConfig, Field from typing import Optional diff --git a/bbot/modules/base.py b/bbot/modules/base.py index f561fdcf19..0d427b8c8f 100644 --- a/bbot/modules/base.py +++ b/bbot/modules/base.py @@ -18,7 +18,7 @@ class BaseModule: produced_events (List): Event types to produce. - meta (Dict): Metadata about the module, such as whether authentication is required and a description. + meta (Dict): Metadata about the module — description, author, created_date, etc. flags (List): Flags indicating the type of module (must have at least "passive" or "active"). @@ -86,7 +86,7 @@ class BaseModule: watched_events = [] produced_events = [] - meta = {"auth_required": False, "description": "Base module"} + meta = {"description": "Base module"} flags = [] options = {} options_desc = {} @@ -1503,7 +1503,13 @@ def priority(self): @property def auth_required(self): - return self.meta.get("auth_required", False) + """True iff this module's `class Config` declares any `mandatory=True` field.""" + cfg = getattr(type(self), "Config", None) + if cfg is None: + return False + from bbot.core.config.models import is_mandatory + + return any(is_mandatory(f) for f in getattr(cfg, "model_fields", {}).values()) @property def http_timeout(self): diff --git a/bbot/modules/bevigil.py b/bbot/modules/bevigil.py index 9103fe872b..c657038da1 100644 --- a/bbot/modules/bevigil.py +++ b/bbot/modules/bevigil.py @@ -1,6 +1,5 @@ from bbot.modules.templates.subdomain_enum import subdomain_enum_apikey -from pydantic import Field -from bbot.core.config.models import BaseModuleConfig +from bbot.core.config.models import BaseModuleConfig, Field class bevigil(subdomain_enum_apikey): @@ -15,11 +14,10 @@ class bevigil(subdomain_enum_apikey): "description": "Retrieve OSINT data from mobile applications using BeVigil", "created_date": "2022-10-26", "author": "@alt-glitch", - "auth_required": True, } class Config(BaseModuleConfig): - api_key: str = Field("", description="BeVigil OSINT API Key") + api_key: str = Field("", description="BeVigil OSINT API Key", sensitive=True, mandatory=True) urls: bool = Field(False, description="Emit URLs in addition to DNS_NAMEs") base_url = "https://osint.bevigil.com/api" diff --git a/bbot/modules/bucket_amazon.py b/bbot/modules/bucket_amazon.py index 98c0f74846..94008fa7bb 100644 --- a/bbot/modules/bucket_amazon.py +++ b/bbot/modules/bucket_amazon.py @@ -1,6 +1,5 @@ from bbot.modules.templates.bucket import bucket_template -from pydantic import Field -from bbot.core.config.models import BaseModuleConfig +from bbot.core.config.models import BaseModuleConfig, Field class bucket_amazon(bucket_template): diff --git a/bbot/modules/bucket_digitalocean.py b/bbot/modules/bucket_digitalocean.py index 187e71f2ba..1c462bd727 100644 --- a/bbot/modules/bucket_digitalocean.py +++ b/bbot/modules/bucket_digitalocean.py @@ -1,6 +1,5 @@ from bbot.modules.templates.bucket import bucket_template -from pydantic import Field -from bbot.core.config.models import BaseModuleConfig +from bbot.core.config.models import BaseModuleConfig, Field class bucket_digitalocean(bucket_template): diff --git a/bbot/modules/bucket_file_enum.py b/bbot/modules/bucket_file_enum.py index b3d3d854b3..a464316d80 100644 --- a/bbot/modules/bucket_file_enum.py +++ b/bbot/modules/bucket_file_enum.py @@ -1,7 +1,6 @@ from bbot.modules.base import BaseModule import xml.etree.ElementTree as ET -from pydantic import Field -from bbot.core.config.models import BaseModuleConfig +from bbot.core.config.models import BaseModuleConfig, Field class bucket_file_enum(BaseModule): diff --git a/bbot/modules/bucket_firebase.py b/bbot/modules/bucket_firebase.py index ce93f713ef..dbe64bf475 100644 --- a/bbot/modules/bucket_firebase.py +++ b/bbot/modules/bucket_firebase.py @@ -1,6 +1,5 @@ from bbot.modules.templates.bucket import bucket_template -from pydantic import Field -from bbot.core.config.models import BaseModuleConfig +from bbot.core.config.models import BaseModuleConfig, Field class bucket_firebase(bucket_template): diff --git a/bbot/modules/bucket_google.py b/bbot/modules/bucket_google.py index 9185f2ce6b..edaf864f0c 100644 --- a/bbot/modules/bucket_google.py +++ b/bbot/modules/bucket_google.py @@ -1,6 +1,5 @@ from bbot.modules.templates.bucket import bucket_template -from pydantic import Field -from bbot.core.config.models import BaseModuleConfig +from bbot.core.config.models import BaseModuleConfig, Field class bucket_google(bucket_template): diff --git a/bbot/modules/bucket_microsoft.py b/bbot/modules/bucket_microsoft.py index 51e4fdb1c2..a565fdfe01 100644 --- a/bbot/modules/bucket_microsoft.py +++ b/bbot/modules/bucket_microsoft.py @@ -1,6 +1,5 @@ from bbot.modules.templates.bucket import bucket_template -from pydantic import Field -from bbot.core.config.models import BaseModuleConfig +from bbot.core.config.models import BaseModuleConfig, Field class bucket_microsoft(bucket_template): diff --git a/bbot/modules/bufferoverrun.py b/bbot/modules/bufferoverrun.py index bbcd1bc0a5..4744c72b1a 100644 --- a/bbot/modules/bufferoverrun.py +++ b/bbot/modules/bufferoverrun.py @@ -1,6 +1,5 @@ from bbot.modules.templates.subdomain_enum import subdomain_enum_apikey -from pydantic import Field -from bbot.core.config.models import BaseModuleConfig +from bbot.core.config.models import BaseModuleConfig, Field class BufferOverrun(subdomain_enum_apikey): @@ -11,11 +10,10 @@ class BufferOverrun(subdomain_enum_apikey): "description": "Query BufferOverrun's TLS API for subdomains", "created_date": "2024-10-23", "author": "@TheTechromancer", - "auth_required": True, } class Config(BaseModuleConfig): - api_key: str = Field("", description="BufferOverrun API key") + api_key: str = Field("", description="BufferOverrun API key", sensitive=True, mandatory=True) commercial: bool = Field(False, description="Use commercial API") base_url = "https://tls.bufferover.run/dns" diff --git a/bbot/modules/builtwith.py b/bbot/modules/builtwith.py index 38102ed57d..1b34944d55 100644 --- a/bbot/modules/builtwith.py +++ b/bbot/modules/builtwith.py @@ -11,8 +11,7 @@ ############################################################ from bbot.modules.templates.subdomain_enum import subdomain_enum_apikey -from pydantic import Field -from bbot.core.config.models import BaseModuleConfig +from bbot.core.config.models import BaseModuleConfig, Field class builtwith(subdomain_enum_apikey): @@ -23,11 +22,10 @@ class builtwith(subdomain_enum_apikey): "description": "Query Builtwith.com for subdomains", "created_date": "2022-08-23", "author": "@TheTechromancer", - "auth_required": True, } class Config(BaseModuleConfig): - api_key: str = Field("", description="Builtwith API key") + api_key: str = Field("", description="Builtwith API key", sensitive=True, mandatory=True) redirects: bool = Field(True, description="Also look up inbound and outbound redirects") base_url = "https://api.builtwith.com" diff --git a/bbot/modules/c99.py b/bbot/modules/c99.py index 1676117779..bc0c16f8e2 100644 --- a/bbot/modules/c99.py +++ b/bbot/modules/c99.py @@ -1,6 +1,5 @@ from bbot.modules.templates.subdomain_enum import subdomain_enum_apikey -from pydantic import Field -from bbot.core.config.models import BaseModuleConfig +from bbot.core.config.models import BaseModuleConfig, Field class c99(subdomain_enum_apikey): @@ -11,11 +10,10 @@ class c99(subdomain_enum_apikey): "description": "Query the C99 API for subdomains", "created_date": "2022-07-08", "author": "@TheTechromancer", - "auth_required": True, } class Config(BaseModuleConfig): - api_key: str = Field("", description="c99.nl API key") + api_key: str = Field("", description="c99.nl API key", sensitive=True, mandatory=True) base_url = "https://api.c99.nl" ping_url = f"{base_url}/randomnumber?key={{api_key}}&between=1,100&json" diff --git a/bbot/modules/censys_dns.py b/bbot/modules/censys_dns.py index f3a4645a52..f9ba270ac1 100644 --- a/bbot/modules/censys_dns.py +++ b/bbot/modules/censys_dns.py @@ -1,6 +1,5 @@ from bbot.modules.templates.censys import censys -from pydantic import Field -from bbot.core.config.models import BaseModuleConfig +from bbot.core.config.models import BaseModuleConfig, Field class censys_dns(censys): @@ -16,11 +15,12 @@ class censys_dns(censys): "description": "Query the Censys API for subdomains", "created_date": "2022-08-04", "author": "@TheTechromancer", - "auth_required": True, } class Config(BaseModuleConfig): - api_key: str = Field("", description="Censys.io API Key in the format of 'key:secret'") + api_key: str = Field( + "", description="Censys.io API Key in the format of 'key:secret'", sensitive=True, mandatory=True + ) max_pages: int = Field(5, description="Maximum number of pages to fetch (100 results per page)") async def setup(self): diff --git a/bbot/modules/censys_ip.py b/bbot/modules/censys_ip.py index 9a798dbf7c..14d35cbf13 100644 --- a/bbot/modules/censys_ip.py +++ b/bbot/modules/censys_ip.py @@ -1,6 +1,5 @@ from bbot.modules.templates.censys import censys -from pydantic import Field -from bbot.core.config.models import BaseModuleConfig +from bbot.core.config.models import BaseModuleConfig, Field class censys_ip(censys): @@ -23,11 +22,12 @@ class censys_ip(censys): "description": "Query the Censys API for hosts by IP address", "created_date": "2026-01-26", "author": "@TheTechromancer", - "auth_required": True, } class Config(BaseModuleConfig): - api_key: str = Field("", description="Censys.io API Key in the format of 'key:secret'") + api_key: str = Field( + "", description="Censys.io API Key in the format of 'key:secret'", sensitive=True, mandatory=True + ) dns_names_limit: int = Field( 100, description="Maximum number of DNS names to extract from dns.names (default 100)" ) diff --git a/bbot/modules/chaos.py b/bbot/modules/chaos.py index b9e803f8e9..82e72f8f26 100644 --- a/bbot/modules/chaos.py +++ b/bbot/modules/chaos.py @@ -1,6 +1,5 @@ from bbot.modules.templates.subdomain_enum import subdomain_enum_apikey -from pydantic import Field -from bbot.core.config.models import BaseModuleConfig +from bbot.core.config.models import BaseModuleConfig, Field class chaos(subdomain_enum_apikey): @@ -11,11 +10,10 @@ class chaos(subdomain_enum_apikey): "description": "Query ProjectDiscovery's Chaos API for subdomains", "created_date": "2022-08-14", "author": "@TheTechromancer", - "auth_required": True, } class Config(BaseModuleConfig): - api_key: str = Field("", description="Chaos API key") + api_key: str = Field("", description="Chaos API key", sensitive=True, mandatory=True) base_url = "https://dns.projectdiscovery.io/dns" ping_url = f"{base_url}/example.com" diff --git a/bbot/modules/credshed.py b/bbot/modules/credshed.py index 1f0f13f74f..c2c477990e 100644 --- a/bbot/modules/credshed.py +++ b/bbot/modules/credshed.py @@ -1,8 +1,7 @@ from contextlib import suppress from bbot.modules.templates.subdomain_enum import subdomain_enum -from pydantic import Field -from bbot.core.config.models import BaseModuleConfig +from bbot.core.config.models import BaseModuleConfig, Field class credshed(subdomain_enum): @@ -13,13 +12,12 @@ class credshed(subdomain_enum): "description": "Send queries to your own credshed server to check for known credentials of your targets", "created_date": "2023-10-12", "author": "@SpamFaux", - "auth_required": True, } class Config(BaseModuleConfig): - username: str = Field("", description="Credshed username") - password: str = Field("", description="Credshed password") - credshed_url: str = Field("", description="URL of credshed server") + username: str = Field("", description="Credshed username", sensitive=True, mandatory=True) + password: str = Field("", description="Credshed password", sensitive=True, mandatory=True) + credshed_url: str = Field("", description="URL of credshed server", mandatory=True) target_only = True diff --git a/bbot/modules/dehashed.py b/bbot/modules/dehashed.py index 27903b86b3..805316f808 100644 --- a/bbot/modules/dehashed.py +++ b/bbot/modules/dehashed.py @@ -1,8 +1,7 @@ from contextlib import suppress from bbot.modules.templates.subdomain_enum import subdomain_enum -from pydantic import Field -from bbot.core.config.models import BaseModuleConfig +from bbot.core.config.models import BaseModuleConfig, Field class dehashed(subdomain_enum): @@ -13,11 +12,10 @@ class dehashed(subdomain_enum): "description": "Execute queries against dehashed.com for exposed credentials", "created_date": "2023-10-12", "author": "@SpamFaux", - "auth_required": True, } class Config(BaseModuleConfig): - api_key: str = Field("", description="DeHashed API Key") + api_key: str = Field("", description="DeHashed API Key", sensitive=True, mandatory=True) target_only = True diff --git a/bbot/modules/dnsbimi.py b/bbot/modules/dnsbimi.py index 9463966a81..3a5c38f38e 100644 --- a/bbot/modules/dnsbimi.py +++ b/bbot/modules/dnsbimi.py @@ -28,8 +28,7 @@ from bbot.core.helpers.dns.helpers import record_to_text, service_record import re -from pydantic import Field -from bbot.core.config.models import BaseModuleConfig +from bbot.core.config.models import BaseModuleConfig, Field # Handle "v=BIMI1; l=; a=;" == RFC conformant explicit declination to publish, e.g. useful on a sub-domain if you don't want the sub-domain to have a BIMI logo, yet your registered domain does? # Handle "v=BIMI1; l=; a=" == RFC non-conformant explicit declination to publish diff --git a/bbot/modules/dnsbrute.py b/bbot/modules/dnsbrute.py index 5bd0e55516..013226e072 100644 --- a/bbot/modules/dnsbrute.py +++ b/bbot/modules/dnsbrute.py @@ -1,6 +1,5 @@ from bbot.modules.templates.subdomain_enum import subdomain_enum -from pydantic import Field -from bbot.core.config.models import BaseModuleConfig +from bbot.core.config.models import BaseModuleConfig, Field class dnsbrute(subdomain_enum): diff --git a/bbot/modules/dnsbrute_mutations.py b/bbot/modules/dnsbrute_mutations.py index 7927af3c75..efc68c88ba 100644 --- a/bbot/modules/dnsbrute_mutations.py +++ b/bbot/modules/dnsbrute_mutations.py @@ -1,8 +1,7 @@ import time from bbot.modules.base import BaseModule -from pydantic import Field -from bbot.core.config.models import BaseModuleConfig +from bbot.core.config.models import BaseModuleConfig, Field class dnsbrute_mutations(BaseModule): diff --git a/bbot/modules/dnscaa.py b/bbot/modules/dnscaa.py index 412ed9d6ff..c132f6af5e 100644 --- a/bbot/modules/dnscaa.py +++ b/bbot/modules/dnscaa.py @@ -22,8 +22,7 @@ from bbot.modules.base import BaseModule from bbot.core.helpers.regexes import dns_name_extraction_regex, email_regex, url_regexes -from pydantic import Field -from bbot.core.config.models import BaseModuleConfig +from bbot.core.config.models import BaseModuleConfig, Field class dnscaa(BaseModule): diff --git a/bbot/modules/dnscommonsrv.py b/bbot/modules/dnscommonsrv.py index 855663a28a..db61011ee8 100644 --- a/bbot/modules/dnscommonsrv.py +++ b/bbot/modules/dnscommonsrv.py @@ -1,7 +1,6 @@ from bbot.core.helpers.dns.helpers import common_srvs from bbot.modules.templates.subdomain_enum import subdomain_enum -from pydantic import Field -from bbot.core.config.models import BaseModuleConfig +from bbot.core.config.models import BaseModuleConfig, Field class dnscommonsrv(subdomain_enum): diff --git a/bbot/modules/dnstlsrpt.py b/bbot/modules/dnstlsrpt.py index 2513ebe71e..ead85169c5 100644 --- a/bbot/modules/dnstlsrpt.py +++ b/bbot/modules/dnstlsrpt.py @@ -20,8 +20,7 @@ import re from bbot.core.helpers.regexes import email_regex, url_regexes -from pydantic import Field -from bbot.core.config.models import BaseModuleConfig +from bbot.core.config.models import BaseModuleConfig, Field _tlsrpt_regex = r"^v=(?PTLSRPTv[0-9]+); *(?P.*)$" tlsrpt_regex = re.compile(_tlsrpt_regex, re.I) diff --git a/bbot/modules/docker_pull.py b/bbot/modules/docker_pull.py index 839455f9f8..28180499aa 100644 --- a/bbot/modules/docker_pull.py +++ b/bbot/modules/docker_pull.py @@ -3,8 +3,7 @@ import tarfile from pathlib import Path from bbot.modules.base import BaseModule -from pydantic import Field -from bbot.core.config.models import BaseModuleConfig +from bbot.core.config.models import BaseModuleConfig, Field class docker_pull(BaseModule): diff --git a/bbot/modules/ffuf.py b/bbot/modules/ffuf.py index 89078d7d85..9e2296f71d 100644 --- a/bbot/modules/ffuf.py +++ b/bbot/modules/ffuf.py @@ -4,8 +4,7 @@ import string import json import base64 -from pydantic import Field -from bbot.core.config.models import BaseModuleConfig +from bbot.core.config.models import BaseModuleConfig, Field class ffuf(BaseModule): diff --git a/bbot/modules/ffuf_shortnames.py b/bbot/modules/ffuf_shortnames.py index 293fc1402d..869b6649a9 100644 --- a/bbot/modules/ffuf_shortnames.py +++ b/bbot/modules/ffuf_shortnames.py @@ -4,8 +4,7 @@ import string from bbot.modules.ffuf import ffuf -from pydantic import Field -from bbot.core.config.models import BaseModuleConfig +from bbot.core.config.models import BaseModuleConfig, Field class ffuf_shortnames(ffuf): diff --git a/bbot/modules/filedownload.py b/bbot/modules/filedownload.py index 761193f14d..b43e58be0c 100644 --- a/bbot/modules/filedownload.py +++ b/bbot/modules/filedownload.py @@ -2,8 +2,7 @@ from pathlib import Path from bbot.modules.base import BaseModule -from pydantic import Field -from bbot.core.config.models import BaseModuleConfig +from bbot.core.config.models import BaseModuleConfig, Field class filedownload(BaseModule): diff --git a/bbot/modules/fingerprintx.py b/bbot/modules/fingerprintx.py index b815f438b8..4bbac86970 100644 --- a/bbot/modules/fingerprintx.py +++ b/bbot/modules/fingerprintx.py @@ -1,8 +1,7 @@ import json import subprocess from bbot.modules.base import BaseModule -from pydantic import Field -from bbot.core.config.models import BaseModuleConfig +from bbot.core.config.models import BaseModuleConfig, Field class fingerprintx(BaseModule): diff --git a/bbot/modules/fullhunt.py b/bbot/modules/fullhunt.py index ba15bcf5c1..ae7996c7d1 100644 --- a/bbot/modules/fullhunt.py +++ b/bbot/modules/fullhunt.py @@ -1,6 +1,5 @@ from bbot.modules.templates.subdomain_enum import subdomain_enum_apikey -from pydantic import Field -from bbot.core.config.models import BaseModuleConfig +from bbot.core.config.models import BaseModuleConfig, Field class fullhunt(subdomain_enum_apikey): @@ -11,11 +10,10 @@ class fullhunt(subdomain_enum_apikey): "description": "Query the fullhunt.io API for subdomains", "created_date": "2022-08-24", "author": "@TheTechromancer", - "auth_required": True, } class Config(BaseModuleConfig): - api_key: str = Field("", description="FullHunt API Key") + api_key: str = Field("", description="FullHunt API Key", sensitive=True, mandatory=True) base_url = "https://fullhunt.io/api/v1" diff --git a/bbot/modules/git_clone.py b/bbot/modules/git_clone.py index 9f526d61e7..38e4fffba2 100644 --- a/bbot/modules/git_clone.py +++ b/bbot/modules/git_clone.py @@ -1,8 +1,7 @@ from pathlib import Path from subprocess import CalledProcessError from bbot.modules.templates.github import github -from pydantic import Field -from bbot.core.config.models import BaseModuleConfig +from bbot.core.config.models import BaseModuleConfig, Field class git_clone(github): @@ -16,7 +15,7 @@ class git_clone(github): } class Config(BaseModuleConfig): - api_key: str = Field("", description="Github token") + api_key: str = Field("", description="Github token", sensitive=True) output_folder: str = Field( "", description="Folder to clone repositories to. If not specified, cloned repositories will be deleted when the scan completes, to minimize disk usage.", diff --git a/bbot/modules/gitdumper.py b/bbot/modules/gitdumper.py index c611bdc050..3024bd2a37 100644 --- a/bbot/modules/gitdumper.py +++ b/bbot/modules/gitdumper.py @@ -2,8 +2,7 @@ from pathlib import Path from subprocess import CalledProcessError from bbot.modules.base import BaseModule -from pydantic import Field -from bbot.core.config.models import BaseModuleConfig +from bbot.core.config.models import BaseModuleConfig, Field class gitdumper(BaseModule): diff --git a/bbot/modules/github_codesearch.py b/bbot/modules/github_codesearch.py index fbc4a253ef..ea4dc984b9 100644 --- a/bbot/modules/github_codesearch.py +++ b/bbot/modules/github_codesearch.py @@ -1,7 +1,6 @@ from bbot.modules.templates.github import github from bbot.modules.templates.subdomain_enum import subdomain_enum -from pydantic import Field -from bbot.core.config.models import BaseModuleConfig +from bbot.core.config.models import BaseModuleConfig, Field class github_codesearch(github, subdomain_enum): @@ -12,11 +11,10 @@ class github_codesearch(github, subdomain_enum): "description": "Query Github's API for code containing the target domain name", "created_date": "2023-12-14", "author": "@domwhewell-sage", - "auth_required": True, } class Config(BaseModuleConfig): - api_key: str = Field("", description="Github token") + api_key: str = Field("", description="Github token", sensitive=True, mandatory=True) limit: int = Field(100, description="Limit code search to this many results") github_raw_url = "https://raw.githubusercontent.com/" diff --git a/bbot/modules/github_org.py b/bbot/modules/github_org.py index bd84a55a77..4585ebe3e4 100644 --- a/bbot/modules/github_org.py +++ b/bbot/modules/github_org.py @@ -1,6 +1,5 @@ from bbot.modules.templates.github import github -from pydantic import Field -from bbot.core.config.models import BaseModuleConfig +from bbot.core.config.models import BaseModuleConfig, Field class github_org(github): @@ -14,7 +13,7 @@ class github_org(github): } class Config(BaseModuleConfig): - api_key: str = Field("", description="Github token") + api_key: str = Field("", description="Github token", sensitive=True) include_members: bool = Field(True, description="Enumerate organization members") include_member_repos: bool = Field(False, description="Also enumerate organization members' repositories") diff --git a/bbot/modules/github_usersearch.py b/bbot/modules/github_usersearch.py index 9c6aac2370..5662ff1cf9 100644 --- a/bbot/modules/github_usersearch.py +++ b/bbot/modules/github_usersearch.py @@ -1,7 +1,6 @@ from bbot.modules.templates.github import github from bbot.modules.templates.subdomain_enum import subdomain_enum -from pydantic import Field -from bbot.core.config.models import BaseModuleConfig +from bbot.core.config.models import BaseModuleConfig, Field class github_usersearch(github, subdomain_enum): @@ -12,11 +11,10 @@ class github_usersearch(github, subdomain_enum): "description": "Query Github's API for users with emails matching in scope domains that may not be discoverable by listing members of the organization.", "created_date": "2025-05-10", "author": "@domwhewell-sage", - "auth_required": True, } class Config(BaseModuleConfig): - api_key: str = Field("", description="Github token") + api_key: str = Field("", description="Github token", sensitive=True, mandatory=True) async def handle_event(self, event): self.verbose("Searching for users with emails matching in scope domains") diff --git a/bbot/modules/github_workflows.py b/bbot/modules/github_workflows.py index 3ccea39be5..c17c81583d 100644 --- a/bbot/modules/github_workflows.py +++ b/bbot/modules/github_workflows.py @@ -3,8 +3,7 @@ from pathlib import Path from bbot.modules.templates.github import github -from pydantic import Field -from bbot.core.config.models import BaseModuleConfig +from bbot.core.config.models import BaseModuleConfig, Field class github_workflows(github): @@ -15,11 +14,10 @@ class github_workflows(github): "description": "Download a github repositories workflow logs and workflow artifacts", "created_date": "2024-04-29", "author": "@domwhewell-sage", - "auth_required": True, } class Config(BaseModuleConfig): - api_key: str = Field("", description="Github token") + api_key: str = Field("", description="Github token", sensitive=True, mandatory=True) num_logs: int = Field(1, description="For each workflow fetch the last N successful runs logs (max 100)") output_folder: str = Field("", description="Folder to download workflow logs and artifacts to") diff --git a/bbot/modules/gitlab_com.py b/bbot/modules/gitlab_com.py index 39ff302d41..68417c255b 100644 --- a/bbot/modules/gitlab_com.py +++ b/bbot/modules/gitlab_com.py @@ -1,6 +1,5 @@ from bbot.modules.templates.gitlab import GitLabBaseModule -from pydantic import Field -from bbot.core.config.models import BaseModuleConfig +from bbot.core.config.models import BaseModuleConfig, Field class gitlab_com(GitLabBaseModule): @@ -16,7 +15,7 @@ class gitlab_com(GitLabBaseModule): } class Config(BaseModuleConfig): - api_key: str = Field("", description="GitLab access token (for gitlab.com/org only)") + api_key: str = Field("", description="GitLab access token (for gitlab.com/org only)", sensitive=True) # This is needed because we are consuming SOCIAL events, which aren't in scope scope_distance_modifier = 2 diff --git a/bbot/modules/gitlab_onprem.py b/bbot/modules/gitlab_onprem.py index 03c51630ab..7e659a6723 100644 --- a/bbot/modules/gitlab_onprem.py +++ b/bbot/modules/gitlab_onprem.py @@ -1,6 +1,5 @@ from bbot.modules.templates.gitlab import GitLabBaseModule -from pydantic import Field -from bbot.core.config.models import BaseModuleConfig +from bbot.core.config.models import BaseModuleConfig, Field class gitlab_onprem(GitLabBaseModule): @@ -21,7 +20,7 @@ class gitlab_onprem(GitLabBaseModule): # Optional GitLab access token (only required for gitlab.com, but still # supported for on-prem installations that expose private projects). class Config(BaseModuleConfig): - api_key: str = Field("", description="GitLab access token (for self-hosted instances only)") + api_key: str = Field("", description="GitLab access token (for self-hosted instances only)", sensitive=True) # Allow accepting events slightly beyond configured max distance so we can # discover repos on neighbouring infrastructure. diff --git a/bbot/modules/gowitness.py b/bbot/modules/gowitness.py index f9eaa5337c..b4df850919 100644 --- a/bbot/modules/gowitness.py +++ b/bbot/modules/gowitness.py @@ -9,8 +9,7 @@ from shutil import copyfile, copymode from bbot.modules.base import BaseModule -from pydantic import Field -from bbot.core.config.models import BaseModuleConfig +from bbot.core.config.models import BaseModuleConfig, Field class gowitness(BaseModule): diff --git a/bbot/modules/graphql_introspection.py b/bbot/modules/graphql_introspection.py index 1f9e6e1a87..58552651d1 100644 --- a/bbot/modules/graphql_introspection.py +++ b/bbot/modules/graphql_introspection.py @@ -1,8 +1,7 @@ import json from pathlib import Path from bbot.modules.base import BaseModule -from pydantic import Field -from bbot.core.config.models import BaseModuleConfig +from bbot.core.config.models import BaseModuleConfig, Field class graphql_introspection(BaseModule): diff --git a/bbot/modules/httpx.py b/bbot/modules/httpx.py index 6267d5e38d..14a618a0b0 100644 --- a/bbot/modules/httpx.py +++ b/bbot/modules/httpx.py @@ -7,8 +7,7 @@ from http.cookies import SimpleCookie from bbot.modules.base import BaseModule -from pydantic import Field -from bbot.core.config.models import BaseModuleConfig +from bbot.core.config.models import BaseModuleConfig, Field class httpx(BaseModule): diff --git a/bbot/modules/hunterio.py b/bbot/modules/hunterio.py index 07c6b93cbb..7fd08a870e 100644 --- a/bbot/modules/hunterio.py +++ b/bbot/modules/hunterio.py @@ -1,21 +1,15 @@ from bbot.modules.templates.subdomain_enum import subdomain_enum_apikey -from pydantic import Field -from bbot.core.config.models import BaseModuleConfig +from bbot.core.config.models import BaseModuleConfig, Field class hunterio(subdomain_enum_apikey): watched_events = ["DNS_NAME"] produced_events = ["EMAIL_ADDRESS", "DNS_NAME", "URL_UNVERIFIED"] flags = ["safe", "passive", "email-enum", "subdomain-enum"] - meta = { - "description": "Query hunter.io for emails", - "created_date": "2022-04-25", - "author": "@TheTechromancer", - "auth_required": True, - } + meta = {"description": "Query hunter.io for emails", "created_date": "2022-04-25", "author": "@TheTechromancer"} class Config(BaseModuleConfig): - api_key: str = Field("", description="Hunter.IO API key") + api_key: str = Field("", description="Hunter.IO API key", sensitive=True, mandatory=True) base_url = "https://api.hunter.io/v2" ping_url = f"{base_url}/account?api_key={{api_key}}" diff --git a/bbot/modules/iis_shortnames.py b/bbot/modules/iis_shortnames.py index b0b915e58a..99cef5457f 100644 --- a/bbot/modules/iis_shortnames.py +++ b/bbot/modules/iis_shortnames.py @@ -1,8 +1,7 @@ import re from bbot.modules.base import BaseModule -from pydantic import Field -from bbot.core.config.models import BaseModuleConfig +from bbot.core.config.models import BaseModuleConfig, Field valid_chars = "ETAONRISHDLFCMUGYPWBVKJXQZ0123456789_-$~()&!#%'@^`{}]]" diff --git a/bbot/modules/internal/excavate.py b/bbot/modules/internal/excavate.py index 937e3c7257..fb4b8f45b7 100644 --- a/bbot/modules/internal/excavate.py +++ b/bbot/modules/internal/excavate.py @@ -10,8 +10,7 @@ from bbot.modules.base import BaseInterceptModule from bbot.modules.internal.base import BaseInternalModule from urllib.parse import urlparse, urljoin, parse_qs, urlunparse, urldefrag -from pydantic import Field -from bbot.core.config.models import BaseModuleConfig +from bbot.core.config.models import BaseModuleConfig, Field def find_subclasses(obj, base_class): diff --git a/bbot/modules/internal/speculate.py b/bbot/modules/internal/speculate.py index ded0b834ee..8caf4bab1f 100644 --- a/bbot/modules/internal/speculate.py +++ b/bbot/modules/internal/speculate.py @@ -3,8 +3,7 @@ from bbot.core.helpers import validators from bbot.modules.internal.base import BaseInternalModule -from pydantic import Field -from bbot.core.config.models import BaseModuleConfig +from bbot.core.config.models import BaseModuleConfig, Field class speculate(BaseInternalModule): diff --git a/bbot/modules/ip2location.py b/bbot/modules/ip2location.py index 08676fcfef..2001a04269 100644 --- a/bbot/modules/ip2location.py +++ b/bbot/modules/ip2location.py @@ -1,6 +1,5 @@ from bbot.modules.base import BaseModule -from pydantic import Field -from bbot.core.config.models import BaseModuleConfig +from bbot.core.config.models import BaseModuleConfig, Field class IP2Location(BaseModule): @@ -15,11 +14,10 @@ class IP2Location(BaseModule): "description": "Query IP2location.io's API for geolocation information. ", "created_date": "2023-09-12", "author": "@TheTechromancer", - "auth_required": True, } class Config(BaseModuleConfig): - api_key: str = Field("", description="IP2location.io API Key") + api_key: str = Field("", description="IP2location.io API Key", sensitive=True, mandatory=True) lang: str = Field( "", description="Translation information(ISO639-1). The translation is only applicable for continent, country, region and city name.", diff --git a/bbot/modules/ipneighbor.py b/bbot/modules/ipneighbor.py index 1384f5ca76..518d5510f6 100644 --- a/bbot/modules/ipneighbor.py +++ b/bbot/modules/ipneighbor.py @@ -1,8 +1,7 @@ import ipaddress from bbot.modules.base import BaseModule -from pydantic import Field -from bbot.core.config.models import BaseModuleConfig +from bbot.core.config.models import BaseModuleConfig, Field class ipneighbor(BaseModule): diff --git a/bbot/modules/ipstack.py b/bbot/modules/ipstack.py index 13e41b28b5..c1d41b5102 100644 --- a/bbot/modules/ipstack.py +++ b/bbot/modules/ipstack.py @@ -1,6 +1,5 @@ from bbot.modules.base import BaseModule -from pydantic import Field -from bbot.core.config.models import BaseModuleConfig +from bbot.core.config.models import BaseModuleConfig, Field class Ipstack(BaseModule): @@ -12,15 +11,10 @@ class Ipstack(BaseModule): watched_events = ["IP_ADDRESS"] produced_events = ["GEOLOCATION"] flags = ["safe", "passive"] - meta = { - "description": "Query IPStack's GeoIP API", - "created_date": "2022-11-26", - "author": "@tycoonslive", - "auth_required": True, - } + meta = {"description": "Query IPStack's GeoIP API", "created_date": "2022-11-26", "author": "@tycoonslive"} class Config(BaseModuleConfig): - api_key: str = Field("", description="IPStack GeoIP API Key") + api_key: str = Field("", description="IPStack GeoIP API Key", sensitive=True, mandatory=True) scope_distance_modifier = 1 _priority = 2 diff --git a/bbot/modules/jadx.py b/bbot/modules/jadx.py index 9da80079aa..b3a1d2cb12 100644 --- a/bbot/modules/jadx.py +++ b/bbot/modules/jadx.py @@ -1,8 +1,7 @@ from pathlib import Path from subprocess import CalledProcessError from bbot.modules.internal.base import BaseModule -from pydantic import Field -from bbot.core.config.models import BaseModuleConfig +from bbot.core.config.models import BaseModuleConfig, Field class jadx(BaseModule): diff --git a/bbot/modules/kreuzberg.py b/bbot/modules/kreuzberg.py index d782958415..ea30b2b453 100644 --- a/bbot/modules/kreuzberg.py +++ b/bbot/modules/kreuzberg.py @@ -2,8 +2,7 @@ from kreuzberg import extract_file from bbot.modules.base import BaseModule -from pydantic import Field -from bbot.core.config.models import BaseModuleConfig +from bbot.core.config.models import BaseModuleConfig, Field class kreuzberg(BaseModule): diff --git a/bbot/modules/leakix.py b/bbot/modules/leakix.py index 5d98f83435..099a7eee2c 100644 --- a/bbot/modules/leakix.py +++ b/bbot/modules/leakix.py @@ -1,6 +1,5 @@ from bbot.modules.templates.subdomain_enum import subdomain_enum_apikey -from pydantic import Field -from bbot.core.config.models import BaseModuleConfig +from bbot.core.config.models import BaseModuleConfig, Field class leakix(subdomain_enum_apikey): @@ -9,7 +8,7 @@ class leakix(subdomain_enum_apikey): flags = ["safe", "subdomain-enum", "passive"] class Config(BaseModuleConfig): - api_key: str = Field("", description="LeakIX API Key") + api_key: str = Field("", description="LeakIX API Key", sensitive=True) # NOTE: API key is not required (but having one will get you more results) meta = { diff --git a/bbot/modules/legba.py b/bbot/modules/legba.py index 8ad9eebd7e..44305f2a4f 100644 --- a/bbot/modules/legba.py +++ b/bbot/modules/legba.py @@ -2,8 +2,7 @@ from pathlib import Path from bbot.errors import WordlistError from bbot.modules.base import BaseModule -from pydantic import Field -from bbot.core.config.models import BaseModuleConfig +from bbot.core.config.models import BaseModuleConfig, Field # key: value: # List with `legba -L` diff --git a/bbot/modules/lightfuzz/lightfuzz.py b/bbot/modules/lightfuzz/lightfuzz.py index 06b735ebb8..63af8ff275 100644 --- a/bbot/modules/lightfuzz/lightfuzz.py +++ b/bbot/modules/lightfuzz/lightfuzz.py @@ -2,8 +2,7 @@ from bbot.modules.base import BaseModule from bbot.errors import InteractshError -from pydantic import Field -from bbot.core.config.models import BaseModuleConfig +from bbot.core.config.models import BaseModuleConfig, Field class lightfuzz(BaseModule): diff --git a/bbot/modules/medusa.py b/bbot/modules/medusa.py index b0315d7869..f0e16cc879 100644 --- a/bbot/modules/medusa.py +++ b/bbot/modules/medusa.py @@ -1,7 +1,6 @@ import re from bbot.modules.base import BaseModule -from pydantic import Field -from bbot.core.config.models import BaseModuleConfig +from bbot.core.config.models import BaseModuleConfig, Field class medusa(BaseModule): diff --git a/bbot/modules/ntlm.py b/bbot/modules/ntlm.py index bd4a77ffa1..621b86182a 100644 --- a/bbot/modules/ntlm.py +++ b/bbot/modules/ntlm.py @@ -1,7 +1,6 @@ from bbot.errors import NTLMError from bbot.modules.base import BaseModule -from pydantic import Field -from bbot.core.config.models import BaseModuleConfig +from bbot.core.config.models import BaseModuleConfig, Field ntlm_discovery_endpoints = [ "", diff --git a/bbot/modules/nuclei.py b/bbot/modules/nuclei.py index e330ddf085..1152a974d9 100644 --- a/bbot/modules/nuclei.py +++ b/bbot/modules/nuclei.py @@ -3,8 +3,7 @@ from typing import Literal from itertools import islice from bbot.modules.base import BaseModule -from pydantic import Field -from bbot.core.config.models import BaseModuleConfig +from bbot.core.config.models import BaseModuleConfig, Field class nuclei(BaseModule): diff --git a/bbot/modules/oauth.py b/bbot/modules/oauth.py index ac886d58ab..4033d84e77 100644 --- a/bbot/modules/oauth.py +++ b/bbot/modules/oauth.py @@ -1,8 +1,7 @@ from bbot.core.helpers.regexes import url_regexes from .base import BaseModule -from pydantic import Field -from bbot.core.config.models import BaseModuleConfig +from bbot.core.config.models import BaseModuleConfig, Field class OAUTH(BaseModule): diff --git a/bbot/modules/otx.py b/bbot/modules/otx.py index 8e2f91ea69..53e3e189c2 100644 --- a/bbot/modules/otx.py +++ b/bbot/modules/otx.py @@ -1,6 +1,5 @@ from bbot.modules.templates.subdomain_enum import subdomain_enum_apikey -from pydantic import Field -from bbot.core.config.models import BaseModuleConfig +from bbot.core.config.models import BaseModuleConfig, Field class otx(subdomain_enum_apikey): @@ -11,11 +10,10 @@ class otx(subdomain_enum_apikey): "description": "Query otx.alienvault.com for subdomains", "created_date": "2022-08-24", "author": "@TheTechromancer", - "auth_required": True, } class Config(BaseModuleConfig): - api_key: str = Field("", description="OTX API key") + api_key: str = Field("", description="OTX API key", sensitive=True, mandatory=True) base_url = "https://otx.alienvault.com" diff --git a/bbot/modules/output/asset_inventory.py b/bbot/modules/output/asset_inventory.py index 380072b5de..4046a9a741 100644 --- a/bbot/modules/output/asset_inventory.py +++ b/bbot/modules/output/asset_inventory.py @@ -4,8 +4,7 @@ from .csv import CSV from bbot.core.helpers.misc import make_ip_type, is_ip, is_port, best_http_status -from pydantic import Field -from bbot.core.config.models import BaseModuleConfig +from bbot.core.config.models import BaseModuleConfig, Field severity_map = { "INFO": 0, diff --git a/bbot/modules/output/csv.py b/bbot/modules/output/csv.py index 237d2360bf..3a30650b53 100644 --- a/bbot/modules/output/csv.py +++ b/bbot/modules/output/csv.py @@ -2,8 +2,7 @@ from contextlib import suppress from bbot.modules.output.base import BaseOutputModule -from pydantic import Field -from bbot.core.config.models import BaseModuleConfig +from bbot.core.config.models import BaseModuleConfig, Field class CSV(BaseOutputModule): diff --git a/bbot/modules/output/discord.py b/bbot/modules/output/discord.py index 145d1aefd0..82656e0e30 100644 --- a/bbot/modules/output/discord.py +++ b/bbot/modules/output/discord.py @@ -1,6 +1,5 @@ from bbot.modules.templates.webhook import WebhookOutputModule -from pydantic import Field -from bbot.core.config.models import BaseModuleConfig +from bbot.core.config.models import BaseModuleConfig, Field class Discord(WebhookOutputModule): @@ -12,7 +11,7 @@ class Discord(WebhookOutputModule): } class Config(BaseModuleConfig): - webhook_url: str = Field("", description="Discord webhook URL") + webhook_url: str = Field("", description="Discord webhook URL", sensitive=True) event_types: list[str] = Field(["FINDING"], description="Types of events to send") min_severity: str = Field("LOW", description="Only allow FINDING events of this severity or higher") retries: int = Field(10, description="Number of times to retry sending the message before skipping the event") diff --git a/bbot/modules/output/elastic.py b/bbot/modules/output/elastic.py index b2a5e19323..a582464b9c 100644 --- a/bbot/modules/output/elastic.py +++ b/bbot/modules/output/elastic.py @@ -1,6 +1,5 @@ from .http import HTTP -from pydantic import Field -from bbot.core.config.models import BaseModuleConfig +from bbot.core.config.models import BaseModuleConfig, Field class Elastic(HTTP): @@ -20,8 +19,8 @@ class Config(BaseModuleConfig): "https://localhost:9200/bbot_events/_doc", description="Elastic URL (e.g. https://localhost:9200//_doc)", ) - username: str = Field("elastic", description="Elastic username") - password: str = Field("bbotislife", description="Elastic password") + username: str = Field("elastic", description="Elastic username", sensitive=True) + password: str = Field("bbotislife", description="Elastic password", sensitive=True) timeout: int = Field(10, description="HTTP timeout") async def cleanup(self): diff --git a/bbot/modules/output/emails.py b/bbot/modules/output/emails.py index 5bbbc9358d..1eb1718181 100644 --- a/bbot/modules/output/emails.py +++ b/bbot/modules/output/emails.py @@ -1,7 +1,6 @@ from bbot.modules.output.txt import TXT from bbot.modules.base import BaseModule -from pydantic import Field -from bbot.core.config.models import BaseModuleConfig +from bbot.core.config.models import BaseModuleConfig, Field class Emails(TXT): diff --git a/bbot/modules/output/http.py b/bbot/modules/output/http.py index 484c11d1d4..c77b8f2ac4 100644 --- a/bbot/modules/output/http.py +++ b/bbot/modules/output/http.py @@ -1,7 +1,6 @@ from bbot.models.pydantic import Event from bbot.modules.output.base import BaseOutputModule -from pydantic import Field -from bbot.core.config.models import BaseModuleConfig +from bbot.core.config.models import BaseModuleConfig, Field class HTTP(BaseOutputModule): @@ -15,10 +14,10 @@ class HTTP(BaseOutputModule): class Config(BaseModuleConfig): url: str = Field("", description="Web URL") method: str = Field("POST", description="HTTP method") - bearer: str = Field("", description="Authorization Bearer token") + bearer: str = Field("", description="Authorization Bearer token", sensitive=True) username: str = Field("", description="Username (basic auth)") - password: str = Field("", description="Password (basic auth)") - headers: dict = Field({}, description="Additional headers to send with the request") + password: str = Field("", description="Password (basic auth)", sensitive=True) + headers: dict = Field({}, description="Additional headers to send with the request", sensitive=True) timeout: int = Field(10, description="HTTP timeout") async def setup(self): diff --git a/bbot/modules/output/json.py b/bbot/modules/output/json.py index dbd2bb7b7d..7ac7c33c56 100644 --- a/bbot/modules/output/json.py +++ b/bbot/modules/output/json.py @@ -2,8 +2,7 @@ from contextlib import suppress from bbot.modules.output.base import BaseOutputModule -from pydantic import Field -from bbot.core.config.models import BaseModuleConfig +from bbot.core.config.models import BaseModuleConfig, Field class JSON(BaseOutputModule): diff --git a/bbot/modules/output/kafka.py b/bbot/modules/output/kafka.py index d0db260cbf..101a38f59f 100644 --- a/bbot/modules/output/kafka.py +++ b/bbot/modules/output/kafka.py @@ -2,8 +2,7 @@ from aiokafka import AIOKafkaProducer from bbot.modules.output.base import BaseOutputModule -from pydantic import Field -from bbot.core.config.models import BaseModuleConfig +from bbot.core.config.models import BaseModuleConfig, Field class Kafka(BaseOutputModule): diff --git a/bbot/modules/output/mongo.py b/bbot/modules/output/mongo.py index 158f315d76..13f3e09fed 100644 --- a/bbot/modules/output/mongo.py +++ b/bbot/modules/output/mongo.py @@ -4,8 +4,7 @@ from bbot.models.pydantic import Event, Scan, Target from bbot.modules.output.base import BaseOutputModule -from pydantic import Field -from bbot.core.config.models import BaseModuleConfig +from bbot.core.config.models import BaseModuleConfig, Field class Mongo(BaseOutputModule): @@ -21,10 +20,10 @@ class Mongo(BaseOutputModule): } class Config(BaseModuleConfig): - uri: str = Field("mongodb://localhost:27017", description="The URI of the MongoDB server") + uri: str = Field("mongodb://localhost:27017", description="The URI of the MongoDB server", sensitive=True) database: str = Field("bbot", description="The name of the database to use") - username: str = Field("", description="The username to use to connect to the database") - password: str = Field("", description="The password to use to connect to the database") + username: str = Field("", description="The username to use to connect to the database", sensitive=True) + password: str = Field("", description="The password to use to connect to the database", sensitive=True) collection_prefix: str = Field("", description="Prefix the name of each collection with this string") deps_pip = ["pymongo~=4.15"] diff --git a/bbot/modules/output/mysql.py b/bbot/modules/output/mysql.py index 8fd1510acb..5d0438aede 100644 --- a/bbot/modules/output/mysql.py +++ b/bbot/modules/output/mysql.py @@ -1,6 +1,5 @@ from bbot.modules.templates.sql import SQLTemplate -from pydantic import Field -from bbot.core.config.models import BaseModuleConfig +from bbot.core.config.models import BaseModuleConfig, Field class MySQL(SQLTemplate): @@ -12,8 +11,8 @@ class MySQL(SQLTemplate): } class Config(BaseModuleConfig): - username: str = Field("root", description="The username to connect to MySQL") - password: str = Field("bbotislife", description="The password to connect to MySQL") + username: str = Field("root", description="The username to connect to MySQL", sensitive=True) + password: str = Field("bbotislife", description="The password to connect to MySQL", sensitive=True) host: str = Field("localhost", description="The server running MySQL") port: int = Field(3306, description="The port to connect to MySQL") database: str = Field("bbot", description="The database name to connect to") diff --git a/bbot/modules/output/nats.py b/bbot/modules/output/nats.py index 036a9a5750..b699f20d80 100644 --- a/bbot/modules/output/nats.py +++ b/bbot/modules/output/nats.py @@ -1,8 +1,7 @@ import json import nats from bbot.modules.output.base import BaseOutputModule -from pydantic import Field -from bbot.core.config.models import BaseModuleConfig +from bbot.core.config.models import BaseModuleConfig, Field class NATS(BaseOutputModule): diff --git a/bbot/modules/output/neo4j.py b/bbot/modules/output/neo4j.py index 8553bed357..1429323793 100644 --- a/bbot/modules/output/neo4j.py +++ b/bbot/modules/output/neo4j.py @@ -4,8 +4,7 @@ from neo4j import AsyncGraphDatabase from bbot.modules.output.base import BaseOutputModule -from pydantic import Field -from bbot.core.config.models import BaseModuleConfig +from bbot.core.config.models import BaseModuleConfig, Field # silence annoying neo4j logger @@ -38,8 +37,8 @@ class neo4j(BaseOutputModule): class Config(BaseModuleConfig): uri: str = Field("bolt://localhost:7687", description="Neo4j server + port") - username: str = Field("neo4j", description="Neo4j username") - password: str = Field("bbotislife", description="Neo4j password") + username: str = Field("neo4j", description="Neo4j username", sensitive=True) + password: str = Field("bbotislife", description="Neo4j password", sensitive=True) deps_pip = ["neo4j"] _batch_size = 500 diff --git a/bbot/modules/output/postgres.py b/bbot/modules/output/postgres.py index ffbbe72afe..93ace12111 100644 --- a/bbot/modules/output/postgres.py +++ b/bbot/modules/output/postgres.py @@ -1,6 +1,5 @@ from bbot.modules.templates.sql import SQLTemplate -from pydantic import Field -from bbot.core.config.models import BaseModuleConfig +from bbot.core.config.models import BaseModuleConfig, Field class Postgres(SQLTemplate): @@ -12,8 +11,8 @@ class Postgres(SQLTemplate): } class Config(BaseModuleConfig): - username: str = Field("postgres", description="The username to connect to Postgres") - password: str = Field("bbotislife", description="The password to connect to Postgres") + username: str = Field("postgres", description="The username to connect to Postgres", sensitive=True) + password: str = Field("bbotislife", description="The password to connect to Postgres", sensitive=True) host: str = Field("localhost", description="The server running Postgres") port: int = Field(5432, description="The port to connect to Postgres") database: str = Field("bbot", description="The database name to connect to") diff --git a/bbot/modules/output/rabbitmq.py b/bbot/modules/output/rabbitmq.py index c5bff3a78c..fdb2aef4d0 100644 --- a/bbot/modules/output/rabbitmq.py +++ b/bbot/modules/output/rabbitmq.py @@ -2,8 +2,7 @@ import aio_pika from bbot.modules.output.base import BaseOutputModule -from pydantic import Field -from bbot.core.config.models import BaseModuleConfig +from bbot.core.config.models import BaseModuleConfig, Field class RabbitMQ(BaseOutputModule): @@ -15,7 +14,7 @@ class RabbitMQ(BaseOutputModule): } class Config(BaseModuleConfig): - url: str = Field("amqp://guest:guest@localhost/", description="The RabbitMQ connection URL") + url: str = Field("amqp://guest:guest@localhost/", description="The RabbitMQ connection URL", sensitive=True) queue: str = Field("bbot_events", description="The RabbitMQ queue to publish events to") deps_pip = ["aio_pika~=9.5.0"] diff --git a/bbot/modules/output/slack.py b/bbot/modules/output/slack.py index b1109ac99d..3b80daf9e1 100644 --- a/bbot/modules/output/slack.py +++ b/bbot/modules/output/slack.py @@ -1,8 +1,7 @@ import yaml from bbot.modules.templates.webhook import WebhookOutputModule -from pydantic import Field -from bbot.core.config.models import BaseModuleConfig +from bbot.core.config.models import BaseModuleConfig, Field class Slack(WebhookOutputModule): @@ -14,7 +13,7 @@ class Slack(WebhookOutputModule): } class Config(BaseModuleConfig): - webhook_url: str = Field("", description="Discord webhook URL") + webhook_url: str = Field("", description="Slack webhook URL", sensitive=True) event_types: list[str] = Field(["FINDING"], description="Types of events to send") min_severity: str = Field("LOW", description="Only allow FINDING events of this severity or higher") retries: int = Field(10, description="Number of times to retry sending the message before skipping the event") diff --git a/bbot/modules/output/splunk.py b/bbot/modules/output/splunk.py index c0d45b89e6..1413db55e2 100644 --- a/bbot/modules/output/splunk.py +++ b/bbot/modules/output/splunk.py @@ -1,7 +1,6 @@ from bbot.errors import WebError from bbot.modules.output.base import BaseOutputModule -from pydantic import Field -from bbot.core.config.models import BaseModuleConfig +from bbot.core.config.models import BaseModuleConfig, Field class Splunk(BaseOutputModule): @@ -14,7 +13,7 @@ class Splunk(BaseOutputModule): class Config(BaseModuleConfig): url: str = Field("", description="Web URL") - hectoken: str = Field("", description="HEC Token") + hectoken: str = Field("", description="HEC Token", sensitive=True) index: str = Field("", description="Index to send data to") source: str = Field("", description="Source path to be added to the metadata") timeout: int = Field(10, description="HTTP timeout") diff --git a/bbot/modules/output/sqlite.py b/bbot/modules/output/sqlite.py index e306664354..d1f43d063c 100644 --- a/bbot/modules/output/sqlite.py +++ b/bbot/modules/output/sqlite.py @@ -1,8 +1,7 @@ from pathlib import Path from bbot.modules.templates.sql import SQLTemplate -from pydantic import Field -from bbot.core.config.models import BaseModuleConfig +from bbot.core.config.models import BaseModuleConfig, Field class SQLite(SQLTemplate): diff --git a/bbot/modules/output/subdomains.py b/bbot/modules/output/subdomains.py index eebda26720..64db0f37f1 100644 --- a/bbot/modules/output/subdomains.py +++ b/bbot/modules/output/subdomains.py @@ -1,7 +1,6 @@ from bbot.modules.output.txt import TXT from bbot.modules.base import BaseModule -from pydantic import Field -from bbot.core.config.models import BaseModuleConfig +from bbot.core.config.models import BaseModuleConfig, Field class Subdomains(TXT): diff --git a/bbot/modules/output/teams.py b/bbot/modules/output/teams.py index 31db1ceef9..40d72df668 100644 --- a/bbot/modules/output/teams.py +++ b/bbot/modules/output/teams.py @@ -1,6 +1,5 @@ from bbot.modules.templates.webhook import WebhookOutputModule -from pydantic import Field -from bbot.core.config.models import BaseModuleConfig +from bbot.core.config.models import BaseModuleConfig, Field class Teams(WebhookOutputModule): @@ -12,7 +11,7 @@ class Teams(WebhookOutputModule): } class Config(BaseModuleConfig): - webhook_url: str = Field("", description="Teams webhook URL") + webhook_url: str = Field("", description="Teams webhook URL", sensitive=True) event_types: list[str] = Field(["FINDING"], description="Types of events to send") min_severity: str = Field("LOW", description="Only allow FINDING events of this severity or higher") retries: int = Field(10, description="Number of times to retry sending the message before skipping the event") diff --git a/bbot/modules/output/txt.py b/bbot/modules/output/txt.py index b4c123168f..02adefc895 100644 --- a/bbot/modules/output/txt.py +++ b/bbot/modules/output/txt.py @@ -1,8 +1,7 @@ from contextlib import suppress from bbot.modules.output.base import BaseOutputModule -from pydantic import Field -from bbot.core.config.models import BaseModuleConfig +from bbot.core.config.models import BaseModuleConfig, Field class TXT(BaseOutputModule): diff --git a/bbot/modules/output/web_parameters.py b/bbot/modules/output/web_parameters.py index 0e4a6b625c..041f0ae29d 100644 --- a/bbot/modules/output/web_parameters.py +++ b/bbot/modules/output/web_parameters.py @@ -2,8 +2,7 @@ from collections import defaultdict from bbot.modules.output.base import BaseOutputModule -from pydantic import Field -from bbot.core.config.models import BaseModuleConfig +from bbot.core.config.models import BaseModuleConfig, Field class Web_parameters(BaseOutputModule): diff --git a/bbot/modules/output/web_report.py b/bbot/modules/output/web_report.py index d6dcfbbd95..fba2c98c0a 100644 --- a/bbot/modules/output/web_report.py +++ b/bbot/modules/output/web_report.py @@ -1,8 +1,7 @@ from bbot.modules.output.base import BaseOutputModule import markdown import html -from pydantic import Field -from bbot.core.config.models import BaseModuleConfig +from bbot.core.config.models import BaseModuleConfig, Field class web_report(BaseOutputModule): diff --git a/bbot/modules/output/websocket.py b/bbot/modules/output/websocket.py index 1ca60a36ca..6bd2908080 100644 --- a/bbot/modules/output/websocket.py +++ b/bbot/modules/output/websocket.py @@ -4,8 +4,7 @@ import websockets from bbot.modules.output.base import BaseOutputModule -from pydantic import Field -from bbot.core.config.models import BaseModuleConfig +from bbot.core.config.models import BaseModuleConfig, Field class Websocket(BaseOutputModule): @@ -14,7 +13,7 @@ class Websocket(BaseOutputModule): class Config(BaseModuleConfig): url: str = Field("", description="Web URL") - token: str = Field("", description="Authorization Bearer token") + token: str = Field("", description="Authorization Bearer token", sensitive=True) preserve_graph: bool = Field( True, description="Preserve full chains of events in the graph (prevents orphans)" ) diff --git a/bbot/modules/output/zeromq.py b/bbot/modules/output/zeromq.py index b481fa1e7c..17c7bc4da6 100644 --- a/bbot/modules/output/zeromq.py +++ b/bbot/modules/output/zeromq.py @@ -2,8 +2,7 @@ import json from bbot.modules.output.base import BaseOutputModule -from pydantic import Field -from bbot.core.config.models import BaseModuleConfig +from bbot.core.config.models import BaseModuleConfig, Field class ZeroMQ(BaseOutputModule): diff --git a/bbot/modules/paramminer_cookies.py b/bbot/modules/paramminer_cookies.py index acc3d55085..03fa3df287 100644 --- a/bbot/modules/paramminer_cookies.py +++ b/bbot/modules/paramminer_cookies.py @@ -1,6 +1,5 @@ from .paramminer_headers import paramminer_headers -from pydantic import Field -from bbot.core.config.models import BaseModuleConfig +from bbot.core.config.models import BaseModuleConfig, Field class paramminer_cookies(paramminer_headers): diff --git a/bbot/modules/paramminer_getparams.py b/bbot/modules/paramminer_getparams.py index 22261683c9..cdee9493a1 100644 --- a/bbot/modules/paramminer_getparams.py +++ b/bbot/modules/paramminer_getparams.py @@ -2,10 +2,9 @@ import string from urllib.parse import urlparse -from pydantic import Field from .paramminer_headers import paramminer_headers, _mutate_case -from bbot.core.config.models import BaseModuleConfig +from bbot.core.config.models import BaseModuleConfig, Field class paramminer_getparams(paramminer_headers): diff --git a/bbot/modules/paramminer_headers.py b/bbot/modules/paramminer_headers.py index 9da2e49392..95cbfc6fa5 100644 --- a/bbot/modules/paramminer_headers.py +++ b/bbot/modules/paramminer_headers.py @@ -2,8 +2,7 @@ from bbot.errors import HttpCompareError from bbot.modules.base import BaseModule -from pydantic import Field -from bbot.core.config.models import BaseModuleConfig +from bbot.core.config.models import BaseModuleConfig, Field _case_split = re.compile(r"[-_]+") diff --git a/bbot/modules/passivetotal.py b/bbot/modules/passivetotal.py index 8758693adb..278add25d2 100644 --- a/bbot/modules/passivetotal.py +++ b/bbot/modules/passivetotal.py @@ -1,6 +1,5 @@ from bbot.modules.templates.subdomain_enum import subdomain_enum_apikey -from pydantic import Field -from bbot.core.config.models import BaseModuleConfig +from bbot.core.config.models import BaseModuleConfig, Field class passivetotal(subdomain_enum_apikey): @@ -11,11 +10,12 @@ class passivetotal(subdomain_enum_apikey): "description": "Query the PassiveTotal API for subdomains", "created_date": "2022-08-08", "author": "@TheTechromancer", - "auth_required": True, } class Config(BaseModuleConfig): - api_key: str = Field("", description="PassiveTotal API Key in the format of 'username:api_key'") + api_key: str = Field( + "", description="PassiveTotal API Key in the format of 'username:api_key'", sensitive=True, mandatory=True + ) base_url = "https://api.passivetotal.org/v2" diff --git a/bbot/modules/pgp.py b/bbot/modules/pgp.py index 61f596c478..5dd3772c0b 100644 --- a/bbot/modules/pgp.py +++ b/bbot/modules/pgp.py @@ -1,6 +1,5 @@ from bbot.modules.templates.subdomain_enum import subdomain_enum -from pydantic import Field -from bbot.core.config.models import BaseModuleConfig +from bbot.core.config.models import BaseModuleConfig, Field class pgp(subdomain_enum): diff --git a/bbot/modules/portfilter.py b/bbot/modules/portfilter.py index 25d943c595..700f87a9cf 100644 --- a/bbot/modules/portfilter.py +++ b/bbot/modules/portfilter.py @@ -1,6 +1,5 @@ from bbot.modules.base import BaseInterceptModule -from pydantic import Field -from bbot.core.config.models import BaseModuleConfig +from bbot.core.config.models import BaseModuleConfig, Field class portfilter(BaseInterceptModule): diff --git a/bbot/modules/portscan.py b/bbot/modules/portscan.py index d9749e854a..be4b3d8db4 100644 --- a/bbot/modules/portscan.py +++ b/bbot/modules/portscan.py @@ -4,8 +4,7 @@ from radixtarget import RadixTarget, host_size_key from bbot.modules.base import BaseModule -from pydantic import Field -from bbot.core.config.models import BaseModuleConfig +from bbot.core.config.models import BaseModuleConfig, Field # TODO: this module is getting big. It should probably be two modules: one for ping and one for SYN. diff --git a/bbot/modules/postman.py b/bbot/modules/postman.py index a182894514..a7a2bd3d09 100644 --- a/bbot/modules/postman.py +++ b/bbot/modules/postman.py @@ -1,6 +1,5 @@ from bbot.modules.templates.postman import postman -from pydantic import Field -from bbot.core.config.models import BaseModuleConfig +from bbot.core.config.models import BaseModuleConfig, Field class postman(postman): @@ -14,7 +13,7 @@ class postman(postman): } class Config(BaseModuleConfig): - api_key: str = Field("", description="Postman API Key") + api_key: str = Field("", description="Postman API Key", sensitive=True, mandatory=True) reject_wildcards = False diff --git a/bbot/modules/postman_download.py b/bbot/modules/postman_download.py index 7f063303ec..91c1e0f88c 100644 --- a/bbot/modules/postman_download.py +++ b/bbot/modules/postman_download.py @@ -2,8 +2,7 @@ import json from pathlib import Path from bbot.modules.templates.postman import postman -from pydantic import Field -from bbot.core.config.models import BaseModuleConfig +from bbot.core.config.models import BaseModuleConfig, Field class postman_download(postman): @@ -21,7 +20,7 @@ class Config(BaseModuleConfig): "", description="Folder to download postman workspaces to. If not specified, downloaded workspaces will be deleted when the scan completes, to minimize disk usage.", ) - api_key: str = Field("", description="Postman API Key") + api_key: str = Field("", description="Postman API Key", sensitive=True, mandatory=True) scope_distance_modifier = 2 diff --git a/bbot/modules/robots.py b/bbot/modules/robots.py index df83d6cbb8..62199e8042 100644 --- a/bbot/modules/robots.py +++ b/bbot/modules/robots.py @@ -1,6 +1,5 @@ from bbot.modules.base import BaseModule -from pydantic import Field -from bbot.core.config.models import BaseModuleConfig +from bbot.core.config.models import BaseModuleConfig, Field class robots(BaseModule): diff --git a/bbot/modules/securitytrails.py b/bbot/modules/securitytrails.py index 092e478972..3b5f4e97ab 100644 --- a/bbot/modules/securitytrails.py +++ b/bbot/modules/securitytrails.py @@ -1,6 +1,5 @@ from bbot.modules.templates.subdomain_enum import subdomain_enum_apikey -from pydantic import Field -from bbot.core.config.models import BaseModuleConfig +from bbot.core.config.models import BaseModuleConfig, Field class securitytrails(subdomain_enum_apikey): @@ -11,11 +10,10 @@ class securitytrails(subdomain_enum_apikey): "description": "Query the SecurityTrails API for subdomains", "created_date": "2022-07-03", "author": "@TheTechromancer", - "auth_required": True, } class Config(BaseModuleConfig): - api_key: str = Field("", description="SecurityTrails API key") + api_key: str = Field("", description="SecurityTrails API key", sensitive=True, mandatory=True) base_url = "https://api.securitytrails.com/v1" ping_url = f"{base_url}/ping?apikey={{api_key}}" diff --git a/bbot/modules/securitytxt.py b/bbot/modules/securitytxt.py index e32c3f92b2..d6b0383007 100644 --- a/bbot/modules/securitytxt.py +++ b/bbot/modules/securitytxt.py @@ -50,8 +50,7 @@ import re from bbot.core.helpers.regexes import email_regex, url_regexes -from pydantic import Field -from bbot.core.config.models import BaseModuleConfig +from bbot.core.config.models import BaseModuleConfig, Field _securitytxt_regex = r"^(?P\w+): *(?P.*)$" securitytxt_regex = re.compile(_securitytxt_regex, re.I | re.M) diff --git a/bbot/modules/shodan_dns.py b/bbot/modules/shodan_dns.py index 7a023b47ef..a81d184655 100644 --- a/bbot/modules/shodan_dns.py +++ b/bbot/modules/shodan_dns.py @@ -1,21 +1,15 @@ from bbot.modules.templates.shodan import shodan -from pydantic import Field -from bbot.core.config.models import BaseModuleConfig +from bbot.core.config.models import BaseModuleConfig, Field class shodan_dns(shodan): watched_events = ["DNS_NAME"] produced_events = ["DNS_NAME"] flags = ["safe", "subdomain-enum", "passive"] - meta = { - "description": "Query Shodan for subdomains", - "created_date": "2022-07-03", - "author": "@TheTechromancer", - "auth_required": True, - } + meta = {"description": "Query Shodan for subdomains", "created_date": "2022-07-03", "author": "@TheTechromancer"} class Config(BaseModuleConfig): - api_key: str = Field("", description="Shodan API key") + api_key: str = Field("", description="Shodan API key", sensitive=True, mandatory=True) base_url = "https://api.shodan.io" diff --git a/bbot/modules/shodan_enterprise.py b/bbot/modules/shodan_enterprise.py index 46f9fb9511..2177c410e1 100644 --- a/bbot/modules/shodan_enterprise.py +++ b/bbot/modules/shodan_enterprise.py @@ -1,6 +1,5 @@ from bbot.modules.base import BaseModule -from pydantic import Field -from bbot.core.config.models import BaseModuleConfig +from bbot.core.config.models import BaseModuleConfig, Field class shodan_enterprise(BaseModule): @@ -11,11 +10,10 @@ class shodan_enterprise(BaseModule): "created_date": "2026-01-27", "author": "@Control-Punk-Delete", "description": "Shodan Enterprise API integration module.", - "auth_required": True, } class Config(BaseModuleConfig): - api_key: str = Field("", description="Shodan API Key") + api_key: str = Field("", description="Shodan API Key", sensitive=True, mandatory=True) in_scope_only: bool = Field( True, description="Only query in-scope IPs. If False, will query up to distance 1." ) diff --git a/bbot/modules/shodan_idb.py b/bbot/modules/shodan_idb.py index b2c1208b4f..853e476261 100644 --- a/bbot/modules/shodan_idb.py +++ b/bbot/modules/shodan_idb.py @@ -1,7 +1,6 @@ from bbot.modules.base import BaseModule import time -from pydantic import Field -from bbot.core.config.models import BaseModuleConfig +from bbot.core.config.models import BaseModuleConfig, Field from typing import Optional diff --git a/bbot/modules/sslcert.py b/bbot/modules/sslcert.py index 62911d1b3f..bb2d79794a 100644 --- a/bbot/modules/sslcert.py +++ b/bbot/modules/sslcert.py @@ -6,8 +6,7 @@ from bbot.modules.base import BaseModule from bbot.core.helpers.async_helpers import NamedLock from bbot.core.helpers.web.ssl_context import ssl_context_noverify -from pydantic import Field -from bbot.core.config.models import BaseModuleConfig +from bbot.core.config.models import BaseModuleConfig, Field class sslcert(BaseModule): diff --git a/bbot/modules/subdomainradar.py b/bbot/modules/subdomainradar.py index 8f7b542661..db81a95d9b 100644 --- a/bbot/modules/subdomainradar.py +++ b/bbot/modules/subdomainradar.py @@ -15,11 +15,10 @@ class SubdomainRadar(subdomain_enum_apikey): "description": "Query the Subdomain API for subdomains", "created_date": "2022-07-08", "author": "@TheTechromancer", - "auth_required": True, } class Config(BaseModuleConfig): - api_key: str = Field("", description="SubDomainRadar.io API key") + api_key: str = Field("", description="SubDomainRadar.io API key", sensitive=True, mandatory=True) group: Literal["fast", "medium", "deep"] = Field( "fast", description="The enumeration group to use. Choose from fast, medium, deep" ) diff --git a/bbot/modules/telerik.py b/bbot/modules/telerik.py index 62ae34404b..8b62be7735 100644 --- a/bbot/modules/telerik.py +++ b/bbot/modules/telerik.py @@ -1,8 +1,7 @@ from sys import executable from bbot.modules.base import BaseModule -from pydantic import Field -from bbot.core.config.models import BaseModuleConfig +from bbot.core.config.models import BaseModuleConfig, Field class telerik(BaseModule): diff --git a/bbot/modules/templates/postman.py b/bbot/modules/templates/postman.py index 490c6e62eb..f5c2fdc811 100644 --- a/bbot/modules/templates/postman.py +++ b/bbot/modules/templates/postman.py @@ -19,7 +19,6 @@ class postman(BaseModule): "Origin": "https://www.postman.com", "Referer": "https://www.postman.com/search?q=&scope=public&type=all", } - auth_required = True async def setup(self): await super().setup() diff --git a/bbot/modules/templates/subdomain_enum.py b/bbot/modules/templates/subdomain_enum.py index 3bdcdff07b..5383000e18 100644 --- a/bbot/modules/templates/subdomain_enum.py +++ b/bbot/modules/templates/subdomain_enum.py @@ -199,7 +199,7 @@ class subdomain_enum_apikey(subdomain_enum): watched_events = ["DNS_NAME"] produced_events = ["DNS_NAME"] flags = ["subdomain-enum", "passive"] - meta = {"description": "Query API for subdomains", "auth_required": True} + meta = {"description": "Query API for subdomains"} options = {"api_key": ""} options_desc = {"api_key": "API key"} diff --git a/bbot/modules/trajan.py b/bbot/modules/trajan.py index c7cf634f65..9191e9e536 100644 --- a/bbot/modules/trajan.py +++ b/bbot/modules/trajan.py @@ -2,8 +2,7 @@ from urllib.parse import urlparse from bbot.modules.base import BaseModule -from pydantic import Field -from bbot.core.config.models import BaseModuleConfig +from bbot.core.config.models import BaseModuleConfig, Field class trajan(BaseModule): @@ -19,13 +18,15 @@ class trajan(BaseModule): # Configuration options class Config(BaseModuleConfig): version: str = Field("1.0.0", description="Trajan version to download and use") - github_token: str = Field("", description="GitHub API token for rate-limiting and private repo access") - gitlab_token: str = Field("", description="GitLab API token for private repo access") - ado_token: str = Field("", description="Azure DevOps Personal Access Token (PAT)") - jfrog_token: str = Field("", description="JFrog API token") + github_token: str = Field( + "", description="GitHub API token for rate-limiting and private repo access", sensitive=True + ) + gitlab_token: str = Field("", description="GitLab API token for private repo access", sensitive=True) + ado_token: str = Field("", description="Azure DevOps Personal Access Token (PAT)", sensitive=True) + jfrog_token: str = Field("", description="JFrog API token", sensitive=True) jenkins_username: str = Field("", description="Jenkins username for basic auth") - jenkins_password: str = Field("", description="Jenkins password for basic auth") - jenkins_token: str = Field("", description="Jenkins API token") + jenkins_password: str = Field("", description="Jenkins password for basic auth", sensitive=True) + jenkins_token: str = Field("", description="Jenkins API token", sensitive=True) deps_ansible = [ { diff --git a/bbot/modules/trickest.py b/bbot/modules/trickest.py index 12f835ab24..1746b86e37 100644 --- a/bbot/modules/trickest.py +++ b/bbot/modules/trickest.py @@ -1,21 +1,15 @@ from bbot.modules.templates.subdomain_enum import subdomain_enum_apikey -from pydantic import Field -from bbot.core.config.models import BaseModuleConfig +from bbot.core.config.models import BaseModuleConfig, Field class Trickest(subdomain_enum_apikey): watched_events = ["DNS_NAME"] produced_events = ["DNS_NAME"] flags = ["safe", "affiliates", "subdomain-enum", "passive"] - meta = { - "description": "Query Trickest's API for subdomains", - "author": "@amiremami", - "created_date": "2024-07-27", - "auth_required": True, - } + meta = {"description": "Query Trickest's API for subdomains", "author": "@amiremami", "created_date": "2024-07-27"} class Config(BaseModuleConfig): - api_key: str = Field("", description="Trickest API key") + api_key: str = Field("", description="Trickest API key", sensitive=True, mandatory=True) base_url = "https://api.trickest.io/solutions/v1/public/solution/a7cba1f1-df07-4a5c-876a-953f178996be" ping_url = f"{base_url}/dataset" diff --git a/bbot/modules/trufflehog.py b/bbot/modules/trufflehog.py index 99a05149f8..2002cd7adf 100644 --- a/bbot/modules/trufflehog.py +++ b/bbot/modules/trufflehog.py @@ -1,8 +1,7 @@ import json from functools import partial from bbot.modules.base import BaseModule -from pydantic import Field -from bbot.core.config.models import BaseModuleConfig +from bbot.core.config.models import BaseModuleConfig, Field class trufflehog(BaseModule): diff --git a/bbot/modules/url_manipulation.py b/bbot/modules/url_manipulation.py index 6f56c4e5e7..58b599e0f9 100644 --- a/bbot/modules/url_manipulation.py +++ b/bbot/modules/url_manipulation.py @@ -1,7 +1,6 @@ from bbot.errors import HttpCompareError from bbot.modules.base import BaseModule -from pydantic import Field -from bbot.core.config.models import BaseModuleConfig +from bbot.core.config.models import BaseModuleConfig, Field class url_manipulation(BaseModule): diff --git a/bbot/modules/urlscan.py b/bbot/modules/urlscan.py index adaf72af73..b49870e7f6 100644 --- a/bbot/modules/urlscan.py +++ b/bbot/modules/urlscan.py @@ -1,6 +1,5 @@ from bbot.modules.templates.subdomain_enum import subdomain_enum -from pydantic import Field -from bbot.core.config.models import BaseModuleConfig +from bbot.core.config.models import BaseModuleConfig, Field class urlscan(subdomain_enum): diff --git a/bbot/modules/virustotal.py b/bbot/modules/virustotal.py index ac69dea3d1..f8837e2f20 100644 --- a/bbot/modules/virustotal.py +++ b/bbot/modules/virustotal.py @@ -1,6 +1,5 @@ from bbot.modules.templates.subdomain_enum import subdomain_enum_apikey -from pydantic import Field -from bbot.core.config.models import BaseModuleConfig +from bbot.core.config.models import BaseModuleConfig, Field class virustotal(subdomain_enum_apikey): @@ -11,11 +10,10 @@ class virustotal(subdomain_enum_apikey): "description": "Query VirusTotal's API for subdomains", "created_date": "2022-08-25", "author": "@TheTechromancer", - "auth_required": True, } class Config(BaseModuleConfig): - api_key: str = Field("", description="VirusTotal API Key") + api_key: str = Field("", description="VirusTotal API Key", sensitive=True, mandatory=True) base_url = "https://www.virustotal.com/api/v3" api_page_iter_kwargs = {"json": False, "next_key": lambda r: r.json().get("links", {}).get("next", "")} diff --git a/bbot/modules/wafw00f.py b/bbot/modules/wafw00f.py index 499fae3c0c..051e6ff3e3 100644 --- a/bbot/modules/wafw00f.py +++ b/bbot/modules/wafw00f.py @@ -3,8 +3,7 @@ # disable wafw00f logging import logging -from pydantic import Field -from bbot.core.config.models import BaseModuleConfig +from bbot.core.config.models import BaseModuleConfig, Field wafw00f_logger = logging.getLogger("wafw00f") wafw00f_logger.setLevel(logging.CRITICAL + 100) diff --git a/bbot/modules/wayback.py b/bbot/modules/wayback.py index 203ba7edc2..09281c000d 100644 --- a/bbot/modules/wayback.py +++ b/bbot/modules/wayback.py @@ -1,8 +1,7 @@ from datetime import datetime from bbot.modules.templates.subdomain_enum import subdomain_enum -from pydantic import Field -from bbot.core.config.models import BaseModuleConfig +from bbot.core.config.models import BaseModuleConfig, Field class wayback(subdomain_enum): diff --git a/bbot/modules/wpscan.py b/bbot/modules/wpscan.py index e6953bd08a..5ae1fdc621 100644 --- a/bbot/modules/wpscan.py +++ b/bbot/modules/wpscan.py @@ -1,7 +1,6 @@ import json from bbot.modules.base import BaseModule -from pydantic import Field -from bbot.core.config.models import BaseModuleConfig +from bbot.core.config.models import BaseModuleConfig, Field class wpscan(BaseModule): @@ -15,7 +14,7 @@ class wpscan(BaseModule): } class Config(BaseModuleConfig): - api_key: str = Field("", description="WPScan API Key") + api_key: str = Field("", description="WPScan API Key", sensitive=True) enumerate: str = Field( "vp,vt,cb,dbe", description="Enumeration Process see wpscan help documentation (default: vp,vt,cb,dbe)" ) diff --git a/bbot/test/test_step_1/test_presets.py b/bbot/test/test_step_1/test_presets.py index 571f869115..39873af4f9 100644 --- a/bbot/test/test_step_1/test_presets.py +++ b/bbot/test/test_step_1/test_presets.py @@ -256,7 +256,7 @@ async def test_preset_scope(clean_default_config): name="with_target_scope", seeds=["evilcorp.org"], blacklist=["evilcorp.co.uk:443", "bob@evilcorp.co.uk"], - config={"modules": {"secretsdb": {"api_key": "deadbeef", "otherthing": "asdf"}}}, + config={"modules": {"github_workflows": {"api_key": "deadbeef", "output_folder": "asdf"}}}, ) preset_domain_with_seed_baked = preset_domain_with_seed.bake() @@ -271,16 +271,16 @@ async def test_preset_scope(clean_default_config): scope_dict = preset_with_target_scope_baked.to_dict(include_target=True) assert set(scope_dict["target"]) == {"1.2.3.0/24", "http://evilcorp.net/"} assert set(scope_dict["blacklist"]) == {"bob@evilcorp.co.uk", "evilcorp.co.uk:443"} - # secretsdb config should be preserved (other module config may also be present) - assert scope_dict["config"]["modules"]["secretsdb"] == { + # github_workflows config should be preserved (other module config may also be present) + assert scope_dict["config"]["modules"]["github_workflows"] == { "api_key": "deadbeef", - "otherthing": "asdf", + "output_folder": "asdf", } redacted_dict = preset_with_target_scope_baked.to_dict(include_target=True, redact_secrets=True) assert set(redacted_dict["target"]) == {"1.2.3.0/24", "http://evilcorp.net/"} assert set(redacted_dict["blacklist"]) == {"bob@evilcorp.co.uk", "evilcorp.co.uk:443"} - assert redacted_dict["config"]["modules"]["secretsdb"] == {"otherthing": "asdf"} + assert redacted_dict["config"]["modules"]["github_workflows"] == {"output_folder": "asdf"} assert preset_domain_with_seed_baked.in_scope("www.evilcorp.com") assert not preset_domain_with_seed_baked.in_scope("www.evilcorp.de") diff --git a/docs/data/chord_graph/entities.json b/docs/data/chord_graph/entities.json index 7a0426345c..6d0ca3236f 100644 --- a/docs/data/chord_graph/entities.json +++ b/docs/data/chord_graph/entities.json @@ -592,6 +592,7 @@ 71, 74, 75, + 78, 83, 90, 97, @@ -1259,7 +1260,8 @@ 16 ], "produces": [ - 41 + 41, + 20 ] }, { diff --git a/docs/data/chord_graph/rels.json b/docs/data/chord_graph/rels.json index f20fef2f11..71c04b3bfc 100644 --- a/docs/data/chord_graph/rels.json +++ b/docs/data/chord_graph/rels.json @@ -754,6 +754,11 @@ "target": 78, "type": "produces" }, + { + "source": 20, + "target": 78, + "type": "produces" + }, { "source": 79, "target": 7, diff --git a/docs/dev/preset_validation.md b/docs/dev/preset_validation.md new file mode 100644 index 0000000000..48b993dd07 --- /dev/null +++ b/docs/dev/preset_validation.md @@ -0,0 +1,146 @@ +# Validating, Sanitizing, and Inspecting Presets + +BBOT's preset and module configs are described by [pydantic](https://docs.pydantic.dev/) models — the same models that power `bbot -l` and `bbot --help`. The helpers below let you reuse those models from your own code to validate a preset, redact sensitive values, or dump module metadata. + +All of these helpers work on plain dicts (i.e. what you'd get from `yaml.safe_load`), so you can call them before instantiating a `Preset` or `Scanner`. + +## Validate a preset + +[`validate_preset`](https://github.com/blacklanternsecurity/bbot/blob/dev/bbot/scanner/preset/validate.py) catches typos and type errors in one pass — top-level preset keys, global config, unknown module names, and per-module options: + +```python +from bbot.scanner import validate_preset + +errors = validate_preset({ + "modules": ["nuclei"], + "config": { + "scope": {"strict": True}, + "modules": {"nuclei": {"ratelimit": 200}}, + }, +}) + +for err in errors: + print(err) +``` + +Empty list → valid. Each [`PresetValidationError`](https://github.com/blacklanternsecurity/bbot/blob/dev/bbot/scanner/preset/validate.py) carries a `where` (e.g. `"preset"`, `"config"`, `"module:nuclei"`), a dotted `path`, and a human-readable `message` with closest-match suggestions when applicable. To validate a YAML file directly, use `validate_preset_file(path)`. + +## Redact secrets from a config + +Each module's `class Config(BaseModuleConfig)` declares which fields are sensitive (`sensitive=True`) and/or required (`mandatory=True`). `BBOTCore.no_secrets_config()` walks the config dict alongside those schemas and removes only fields explicitly marked sensitive — no fuzzy name matching: + +```python +from bbot.core import CORE + +config = { + "interactsh_token": "leaky", + "web": {"http_cookies": {"session": "abc"}}, + "modules": { + "shodan_dns": {"api_key": "sh-secret"}, + "robots": {"include_sitemap": True}, + }, +} + +print(CORE.no_secrets_config(config)) +# {'web': {}, 'modules': {'shodan_dns': {}, 'robots': {'include_sitemap': True}}} + +print(CORE.secrets_only_config(config)) +# {'interactsh_token': 'leaky', 'web': {'http_cookies': {'session': 'abc'}}, ...} +``` + +The `Preset.to_dict()` method exposes the same behavior via a `redact_secrets=True` flag: + +```python +preset.to_dict(include_target=True, redact_secrets=True) +``` + +## Mark your own fields sensitive or mandatory + +When you write a module, import `Field` from `bbot.core.config.models` (not from `pydantic` directly) so the `sensitive` and `mandatory` keyword args are accepted: + +```python +from bbot.core.config.models import BaseModuleConfig, Field + + +class my_module(BaseModule): + class Config(BaseModuleConfig): + api_key: str = Field("", description="My API key", sensitive=True, mandatory=True) + verbose: bool = Field(False, description="Print extra detail") +``` + +- `sensitive=True` — value is redacted by `no_secrets_config()` / `Preset.to_dict(redact_secrets=True)` and lives in `~/.config/bbot/secrets.yml` rather than `bbot.yml`. +- `mandatory=True` — module needs this option to function. Drives the **Needs API Key** column in `bbot -l` and the runtime `BaseModule.auth_required` property. + +A field can carry both, just one, or neither. + +## Dump all module metadata + +`MODULE_LOADER.preloaded()` returns the full preload dict for every module — descriptions, flags, watched/produced events, deps, and (since the field-metadata refactor) the per-module `options_sensitive` and `options_mandatory` field-name sets: + +```python +from bbot.core.modules import MODULE_LOADER + +MODULE_LOADER.preload() +shodan = MODULE_LOADER.preloaded()["shodan_dns"] +print(shodan["options_mandatory"]) # ['api_key'] +print(shodan["options_sensitive"]) # ['api_key'] +``` + +For pretty-printed tables, use `MODULE_LOADER.modules_table()` (module summary) or `MODULE_LOADER.modules_options_table()` (per-option name/type/description/default) — these are what `bbot -l` and `bbot --help -m ` render. + +## Filter options by `sensitive` / `mandatory` + +To collect dotted-path keys (e.g. `modules.shodan_dns.api_key`) for every option carrying a particular flag, you have two options depending on scope. + +**Per-module only** — fast, no schema walk. The flag sets are pre-extracted on each preloaded module, so this is just a dict comprehension: + +```python +from bbot.core.modules import MODULE_LOADER + +MODULE_LOADER.preload() + +sensitive_paths = [ + f"modules.{name}.{opt}" + for name, p in MODULE_LOADER.preloaded().items() + for opt in p.get("options_sensitive") or [] +] + +mandatory_paths = [ + f"modules.{name}.{opt}" + for name, p in MODULE_LOADER.preloaded().items() + for opt in p.get("options_mandatory") or [] +] + +# Both flags at once (e.g. an api-key audit — fields a module needs *and* hides): +both = [ + f"modules.{name}.{opt}" + for name, p in MODULE_LOADER.preloaded().items() + for opt in set(p.get("options_sensitive") or []) & set(p.get("options_mandatory") or []) +] +``` + +**Per-module *and* global** — walk `MODULE_LOADER.config_schema`, which is the composite pydantic model with global keys (e.g. `interactsh_token`, `web.http_cookies`) and every module's `Config` grafted in under `modules.`: + +```python +import typing +from pydantic import BaseModel +from bbot.core.config.models import is_sensitive, is_mandatory + +def schema_paths(model, *, predicate, prefix=""): + """Yield dotted paths for every field where `predicate(FieldInfo)` is True.""" + for name, field in model.model_fields.items(): + path = f"{prefix}{name}" + ann = field.annotation + # unwrap Optional[X] + if typing.get_origin(ann) is typing.Union: + ann = next((a for a in typing.get_args(ann) if a is not type(None)), ann) + if isinstance(ann, type) and issubclass(ann, BaseModel): + yield from schema_paths(ann, predicate=predicate, prefix=f"{path}.") + elif predicate(field): + yield path + +all_sensitive = list(schema_paths(MODULE_LOADER.config_schema, predicate=is_sensitive)) +# ['web.http_cookies', 'interactsh_token', 'modules.shodan_dns.api_key', ...] +``` + +The same pattern works for any field-metadata key you stash via `Field(..., json_schema_extra={...})` — swap the `predicate` for your own check. diff --git a/mkdocs.yml b/mkdocs.yml index 498b0b4a6d..0e9eb0d057 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -43,6 +43,7 @@ nav: - Setting Up a Dev Environment: dev/dev_environment.md - BBOT Internal Architecture: dev/architecture.md - How to Write a BBOT Module: dev/module_howto.md + - Validating & Inspecting Presets: dev/preset_validation.md - Unit Tests: dev/tests.md - Discord Bot Example: dev/discord_bot.md - Code Reference: From 20cd441696a2ca27097f58a1ac667d465b4ed406 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Tue, 5 May 2026 15:19:16 -0400 Subject: [PATCH 15/15] add tests --- bbot/test/test_step_1/test_field_metadata.py | 145 ++++++++++++++++ bbot/test/test_step_1/test_merge.py | 164 +++++++++++++++++++ 2 files changed, 309 insertions(+) create mode 100644 bbot/test/test_step_1/test_field_metadata.py create mode 100644 bbot/test/test_step_1/test_merge.py diff --git a/bbot/test/test_step_1/test_field_metadata.py b/bbot/test/test_step_1/test_field_metadata.py new file mode 100644 index 0000000000..9999fda1ef --- /dev/null +++ b/bbot/test/test_step_1/test_field_metadata.py @@ -0,0 +1,145 @@ +""" +Tests for the unified `sensitive` / `mandatory` field-level metadata that +drives: + +- redaction (`BBOTCore.no_secrets_config` / `secrets_only_config`) +- the "Needs API Key" doc column (`MODULE_LOADER.modules_table`) +- the runtime `BaseModule.auth_required` property +""" + +from ..bbot_fixtures import * # noqa: F401, F403 + + +def test_field_records_sensitive_and_mandatory_in_json_schema_extra(): + from bbot.core.config.models import Field, is_mandatory, is_sensitive + + f_secret = Field("", description="x", sensitive=True) + f_required = Field("", description="x", mandatory=True) + f_both = Field("", description="x", sensitive=True, mandatory=True) + f_plain = Field("", description="x") + + assert is_sensitive(f_secret) is True + assert is_mandatory(f_secret) is False + assert is_sensitive(f_required) is False + assert is_mandatory(f_required) is True + assert is_sensitive(f_both) is True + assert is_mandatory(f_both) is True + assert is_sensitive(f_plain) is False + assert is_mandatory(f_plain) is False + + +def test_global_config_marks_known_secrets_sensitive(): + from bbot.core.config.models import BBOTConfig, WebConfig, is_sensitive + + assert is_sensitive(BBOTConfig.model_fields["interactsh_token"]) is True + assert is_sensitive(WebConfig.model_fields["http_cookies"]) is True + # nearby non-secret control + assert is_sensitive(WebConfig.model_fields["http_timeout"]) is False + + +def test_module_preload_extracts_sensitive_and_mandatory_sets(): + """`_extract_pydantic_config` should return the field-level flags via AST.""" + from bbot.core.modules import MODULE_LOADER + + MODULE_LOADER.preload() + preloaded = MODULE_LOADER.preloaded() + + assert preloaded["shodan_dns"]["options_sensitive"] == ["api_key"] + assert preloaded["shodan_dns"]["options_mandatory"] == ["api_key"] + + # robots has a Config but no auth-related fields + assert preloaded["robots"]["options_sensitive"] == [] + assert preloaded["robots"]["options_mandatory"] == [] + + # credshed: 3 mandatory, 2 sensitive (url is mandatory but not sensitive) + assert sorted(preloaded["credshed"]["options_sensitive"]) == ["password", "username"] + assert sorted(preloaded["credshed"]["options_mandatory"]) == ["credshed_url", "password", "username"] + + +def test_modules_table_needs_api_key_derived_from_mandatory(): + from bbot.core.modules import MODULE_LOADER + + MODULE_LOADER.preload() + out = MODULE_LOADER.modules_table(["shodan_dns", "robots"]) + # Row format: | Module | Type | Needs API Key | ... + rows = [line for line in out.splitlines() if "shodan_dns" in line or "robots" in line] + shodan_row = next(r for r in rows if "shodan_dns" in r) + robots_row = next(r for r in rows if "robots" in r) + assert "Yes" in shodan_row.split("|")[3] + assert "No" in robots_row.split("|")[3] + + +def test_no_secrets_config_redacts_module_and_global_secrets(): + from bbot.core import CORE + from bbot.core.modules import MODULE_LOADER + + MODULE_LOADER.preload() + cfg = { + "home": "/tmp/bbot", + "interactsh_token": "leaky", + "web": {"http_cookies": {"session": "abc"}, "http_timeout": 30}, + "modules": { + "shodan_dns": {"api_key": "shodan-secret"}, + "credshed": {"username": "u", "password": "p", "credshed_url": "http://x"}, + "robots": {"include_sitemap": True}, + "unknown_mod": {"foo": "bar"}, # passes through + }, + } + redacted = CORE.no_secrets_config(cfg) + + # global non-secret kept + assert redacted["home"] == "/tmp/bbot" + assert redacted["web"]["http_timeout"] == 30 + # global secrets removed + assert "interactsh_token" not in redacted + assert "http_cookies" not in redacted["web"] + # module secrets removed, non-secret module fields kept (empty dicts kept + # so the module name still appears under `modules`, matching prior behavior) + assert redacted["modules"]["shodan_dns"] == {} + assert redacted["modules"]["credshed"] == {"credshed_url": "http://x"} + assert redacted["modules"]["robots"] == {"include_sitemap": True} + # unknown module passes through unchanged + assert redacted["modules"]["unknown_mod"] == {"foo": "bar"} + + +def test_secrets_only_config_extracts_sensitive_fields(): + from bbot.core import CORE + from bbot.core.modules import MODULE_LOADER + + MODULE_LOADER.preload() + cfg = { + "home": "/tmp/bbot", + "interactsh_token": "leaky", + "web": {"http_cookies": {"session": "abc"}, "http_timeout": 30}, + "modules": { + "shodan_dns": {"api_key": "shodan-secret"}, + "credshed": {"username": "u", "password": "p", "credshed_url": "http://x"}, + }, + } + secrets = CORE.secrets_only_config(cfg) + + assert secrets == { + "interactsh_token": "leaky", + "web": {"http_cookies": {"session": "abc"}}, + "modules": { + "shodan_dns": {"api_key": "shodan-secret"}, + "credshed": {"username": "u", "password": "p"}, + }, + } + + +def test_auth_required_property_derives_from_config(): + from bbot.core.modules import MODULE_LOADER + + MODULE_LOADER.preload() + shodan_dns = MODULE_LOADER.load_module("shodan_dns") + robots = MODULE_LOADER.load_module("robots") + + # `auth_required` is a property — the descriptor is on the class. + # We can resolve it without instantiating by walking model_fields ourselves. + from bbot.core.config.models import is_mandatory + + shodan_mandatories = [n for n, f in shodan_dns.Config.model_fields.items() if is_mandatory(f)] + robots_mandatories = [n for n, f in robots.Config.model_fields.items() if is_mandatory(f)] + assert shodan_mandatories == ["api_key"] + assert robots_mandatories == [] diff --git a/bbot/test/test_step_1/test_merge.py b/bbot/test/test_step_1/test_merge.py new file mode 100644 index 0000000000..edb5757476 --- /dev/null +++ b/bbot/test/test_step_1/test_merge.py @@ -0,0 +1,164 @@ +from bbot.core.config.merge import deep_merge, dotted_get, dotted_set, iter_dotted_paths + + +# ----- deep_merge ----- + + +def test_deep_merge_basic(): + assert deep_merge({"a": 1}, {"b": 2}) == {"a": 1, "b": 2} + + +def test_deep_merge_last_wins(): + assert deep_merge({"a": 1}, {"a": 2}, {"a": 3}) == {"a": 3} + + +def test_deep_merge_recurses_into_dicts(): + a = {"web": {"timeout": 5, "user_agent": "x"}} + b = {"web": {"timeout": 10}} + assert deep_merge(a, b) == {"web": {"timeout": 10, "user_agent": "x"}} + + +def test_deep_merge_lists_are_replaced_not_concatenated(): + # Matches OmegaConf's REPLACE list-merge mode (BBOT's previous default). + a = {"event_types": ["DNS_NAME", "URL"]} + b = {"event_types": ["EMAIL_ADDRESS"]} + assert deep_merge(a, b) == {"event_types": ["EMAIL_ADDRESS"]} + + +def test_deep_merge_dict_replaces_scalar(): + # Pydantic catches the resulting type mismatch downstream; merge itself + # does the structural replacement without complaint. + assert deep_merge({"a": 1}, {"a": {"b": 2}}) == {"a": {"b": 2}} + + +def test_deep_merge_scalar_replaces_dict(): + assert deep_merge({"a": {"b": 1}}, {"a": 5}) == {"a": 5} + + +def test_deep_merge_none_inputs(): + # All three forms used by callers in core.py / files.py + assert deep_merge(None, {"a": 1}) == {"a": 1} + assert deep_merge({"a": 1}, None) == {"a": 1} + assert deep_merge(None, None) == {} + + +def test_deep_merge_no_aliasing_with_base(): + """Mutating the result must not affect the base dict.""" + a = {"web": {"headers": {"X-Foo": "1"}}} + result = deep_merge(a, {}) + result["web"]["headers"]["X-Foo"] = "MUTATED" + result["web"]["new_key"] = "new" + assert a == {"web": {"headers": {"X-Foo": "1"}}} + + +def test_deep_merge_no_aliasing_with_update(): + """Mutating the result must not affect the update dict either.""" + b = {"web": {"headers": {"X-Foo": "1"}}} + result = deep_merge({}, b) + result["web"]["headers"]["X-Foo"] = "MUTATED" + assert b == {"web": {"headers": {"X-Foo": "1"}}} + + +def test_deep_merge_no_aliasing_for_lists(): + """Lists in update should be deep-copied, not aliased.""" + b = {"event_types": ["DNS_NAME"]} + result = deep_merge({}, b) + result["event_types"].append("URL") + assert b == {"event_types": ["DNS_NAME"]} + + +def test_deep_merge_empty_update_skipped(): + a = {"a": 1} + assert deep_merge(a, {}) == {"a": 1} + assert deep_merge(a, {}, {}, {"b": 2}) == {"a": 1, "b": 2} + + +# ----- dotted_get ----- + + +def test_dotted_get_hit(): + assert dotted_get({"a": {"b": {"c": 1}}}, "a.b.c") == 1 + + +def test_dotted_get_default_when_missing(): + assert dotted_get({"a": 1}, "a.b.c", default="x") == "x" + + +def test_dotted_get_default_when_blocked_by_scalar(): + # `a` exists but is a scalar, so `a.b` can't be resolved. + assert dotted_get({"a": 1}, "a.b") is None + assert dotted_get({"a": 1}, "a.b", default="fallback") == "fallback" + + +def test_dotted_get_top_level(): + assert dotted_get({"a": 1}, "a") == 1 + + +# ----- dotted_set ----- + + +def test_dotted_set_creates_intermediates(): + d = {} + dotted_set(d, "a.b.c", 1) + assert d == {"a": {"b": {"c": 1}}} + + +def test_dotted_set_preserves_siblings(): + d = {"a": {"b": 1}} + dotted_set(d, "a.c", 2) + assert d == {"a": {"b": 1, "c": 2}} + + +def test_dotted_set_overwrites_existing_leaf(): + d = {"a": {"b": 1}} + dotted_set(d, "a.b", 99) + assert d == {"a": {"b": 99}} + + +def test_dotted_set_silently_replaces_non_dict_intermediate(): + """ + Documented behavior: a scalar in the path is silently replaced by a dict. + Pydantic validation downstream surfaces the resulting type mismatch. + """ + d = {"a": 1} + dotted_set(d, "a.b", 2) + assert d == {"a": {"b": 2}} + + +def test_dotted_set_top_level(): + d = {} + dotted_set(d, "a", 1) + assert d == {"a": 1} + + +# ----- iter_dotted_paths ----- + + +def test_iter_dotted_paths_basic(): + assert iter_dotted_paths({"a": 1, "b": {"c": 2}}) == ["a", "b.c"] + + +def test_iter_dotted_paths_deep(): + paths = iter_dotted_paths({"a": {"b": {"c": 1, "d": 2}}, "e": 3}) + assert sorted(paths) == ["a.b.c", "a.b.d", "e"] + + +def test_iter_dotted_paths_empty_dict_treated_as_leaf(): + # An empty dict is a leaf path so dotted_get/dotted_set round-trip through it. + assert iter_dotted_paths({"a": {}, "b": 1}) == ["a", "b"] + + +def test_iter_dotted_paths_empty_input(): + assert iter_dotted_paths({}) == [] + + +# ----- round-trip ----- + + +def test_dotted_set_get_roundtrip(): + d = {} + dotted_set(d, "modules.shodan.api_key", "abc123") + dotted_set(d, "web.spider_distance", 2) + assert dotted_get(d, "modules.shodan.api_key") == "abc123" + assert dotted_get(d, "web.spider_distance") == 2 + assert sorted(iter_dotted_paths(d)) == ["modules.shodan.api_key", "web.spider_distance"]