Dev -> Stable 2.4.2

bbot/__init__.py

@@ -2,3 +2,5 @@
 __version__ = "v0.0.0"
 
 from .scanner import Scanner, Preset
+
+__all__ = ["Scanner", "Preset"]

bbot/core/event/__init__.py

@@ -1 +1,3 @@
 from .base import make_event, is_event, event_from_json
+
+__all__ = ["make_event", "is_event", "event_from_json"]

bbot/core/helpers/__init__.py

@@ -1,4 +1,4 @@
 from .url import *
 from .misc import *
-from . import regexes
-from . import validators
+from . import regexes as regexes
+from . import validators as validators

bbot/core/helpers/depsinstaller/__init__.py

@@ -1 +1,3 @@
 from .installer import DepsInstaller
+
+__all__ = ["DepsInstaller"]

bbot/core/helpers/dns/__init__.py

@@ -1 +1 @@
-from .dns import DNSHelper
+from .dns import DNSHelper  # noqa

bbot/core/helpers/misc.py

@@ -50,7 +50,7 @@ def is_domain(d):
     if is_ip(d):
         return False
     extracted = tldextract(d)
-    if extracted.registered_domain:
+    if extracted.top_domain_under_public_suffix:
         if not extracted.subdomain:
             return True
     else:
@@ -85,7 +85,7 @@ def is_subdomain(d):
     if is_ip(d):
         return False
     extracted = tldextract(d)
-    if extracted.registered_domain:
+    if extracted.top_domain_under_public_suffix:
         if extracted.subdomain:
             return True
     else:
@@ -486,7 +486,7 @@ def split_domain(hostname):
         return ("", hostname)
     parsed = tldextract(hostname)
     subdomain = parsed.subdomain
-    domain = parsed.registered_domain
+    domain = parsed.top_domain_under_public_suffix
     if not domain:
         split = hostname.split(".")
         subdomain = ".".join(split[:-2])

bbot/core/helpers/web/__init__.py

@@ -1 +1 @@
-from .web import WebHelper
+from .web import WebHelper  # noqa

bbot/core/helpers/web/client.py

@@ -70,7 +70,7 @@ def __init__(self, *args, **kwargs):
         kwargs["headers"] = headers
         # proxy
         proxies = self._web_config.get("http_proxy", None)
-        kwargs["proxies"] = proxies
+        kwargs["proxy"] = proxies
 
         log.verbose(f"Creating httpx.AsyncClient({args}, {kwargs})")
         super().__init__(*args, **kwargs)

bbot/core/helpers/web/web.py

@@ -95,7 +95,7 @@ async def request(self, *args, **kwargs):
             files (dict, optional): Dictionary of 'name': file-like-objects for multipart encoding upload.
             auth (tuple, optional): Auth tuple to enable Basic/Digest/Custom HTTP auth.
             timeout (float, optional): The maximum time to wait for the request to complete.
-            proxies (dict, optional): Dictionary mapping protocol schemes to proxy URLs.
+            proxy (str, optional): HTTP proxy URL.
             allow_redirects (bool, optional): Enables or disables redirection. Defaults to None.
             stream (bool, optional): Enables or disables response streaming.
             raise_error (bool, optional): Whether to raise exceptions for HTTP connect, timeout errors. Defaults to False.

bbot/defaults.yml

@@ -92,10 +92,18 @@ web:
   # These are attached to all in-scope HTTP requests
   # Note that some modules (e.g. github) may end up sending these to out-of-scope resources
   http_headers: {}
-  # HTTP retries (for Python requests; API calls, etc.)
+  # How many times to retry API requests
+  # Note that this is a separate mechanism on top of HTTP retries
+  # which will retry API requests that don't return a successful status code
+  api_retries: 2
+  # HTTP retries - try again if the raw connection fails
   http_retries: 1
   # HTTP retries (for httpx)
   httpx_retries: 1
+  # Default sleep interval when rate limited by 429 (and retry-after isn't provided)
+  429_sleep_interval: 30
+  # Maximum sleep interval when rate limited by 429 (and an excessive retry-after is provided)
+  429_max_sleep_interval: 60
   # Enable/disable debug messages for web requests/responses
   debug: false
   # Maximum number of HTTP redirects to follow

bbot/modules/baddns.py

@@ -22,7 +22,7 @@ class baddns(BaseModule):
         "enabled_submodules": "A list of submodules to enable. Empty list (default) enables CNAME, TXT and MX Only",
     }
     module_threads = 8
-    deps_pip = ["baddns~=1.4.13"]
+    deps_pip = ["baddns~=1.9.130"]
 
     def select_modules(self):
         selected_submodules = []

bbot/modules/baddns_direct.py

@@ -19,7 +19,7 @@ class baddns_direct(BaseModule):
         "custom_nameservers": "Force BadDNS to use a list of custom nameservers",
     }
     module_threads = 8
-    deps_pip = ["baddns~=1.4.13"]
+    deps_pip = ["baddns~=1.9.130"]
 
     scope_distance_modifier = 1
 

bbot/modules/baddns_zone.py

@@ -16,7 +16,7 @@ class baddns_zone(baddns_module):
         "only_high_confidence": "Do not emit low-confidence or generic detections",
     }
     module_threads = 8
-    deps_pip = ["baddns~=1.4.13"]
+    deps_pip = ["baddns~=1.9.130"]
 
     def set_modules(self):
         self.enabled_submodules = ["NSEC", "zonetransfer"]

bbot/modules/base.py

@@ -104,12 +104,8 @@ class BaseModule:
     _batch_size = 1
     batch_wait = 10
 
-    # API retries, etc.
-    _api_retries = 2
     # disable the module after this many failed attempts in a row
     _api_failure_abort_threshold = 3
-    # sleep for this many seconds after being rate limited
-    _429_sleep_interval = 30
 
     default_discovery_context = "{module} discovered {event.type}: {event.data}"
 
@@ -159,12 +155,18 @@ def __init__(self, scan):
         # track number of failures (for .api_request())
         self._api_request_failures = 0
 
+        self._default_api_retries = self.scan.config.get("web", {}).get("api_retries", 2)
+
         self._tasks = []
         self._event_received = None
 
         # used for optional "per host" tracking
         self._per_host_tracker = set()
 
+        # 429 rate limit handling
+        self._429_sleep_interval = self.scan.web_config.get("429_sleep_interval", 30)
+        self._429_max_sleep_interval = self.scan.web_config.get("429_max_sleep_interval", 60)
+
     async def setup(self):
         """
         Performs one-time setup tasks for the module.
@@ -338,7 +340,7 @@ def cycle_api_key(self):
 
     @property
     def api_retries(self):
-        return max(self._api_retries + 1, len(self._api_keys))
+        return max(self._default_api_retries + 1, len(self._api_keys))
 
     @property
     def api_failure_abort_threshold(self):
@@ -1172,6 +1174,11 @@ async def api_request(self, *args, **kwargs):
                     retry_after = self._get_retry_after(r)
                     if retry_after or status_code == 429:
                         sleep_interval = int(retry_after) if retry_after is not None else self._429_sleep_interval
+                        if retry_after and retry_after > self._429_max_sleep_interval:
+                            self.verbose(
+                                f"Got an excessive retry-after header of {retry_after} from {new_url}, using {self._429_max_sleep_interval} instead"
+                            )
+                            sleep_interval = self._429_max_sleep_interval
                         self.verbose(
                             f"Sleeping for {sleep_interval:,} seconds due to rate limit (HTTP status: {status_code})"
                         )
@@ -1205,7 +1212,8 @@ def _prepare_api_iter_req(self, url, page, page_size, offset, **requests_kwargs)
         return url, requests_kwargs
 
     def _api_response_is_success(self, r):
-        return r.is_success
+        # 404s typically indicate no data rather than an actual error with the API, so we don't want to retry them
+        return getattr(r, "is_success", False) or getattr(r, "status_code", 0) == 404
 
     async def api_page_iter(self, url, page_size=100, _json=True, next_key=None, iter_key=None, **requests_kwargs):
         """

bbot/modules/certspotter.py

@@ -15,7 +15,7 @@ class certspotter(subdomain_enum):
 
     def request_url(self, query):
         url = f"{self.base_url}/issuances?domain={self.helpers.quote(query)}&include_subdomains=true&expand=dns_names"
-        return self.api_request(url, timeout=self.http_timeout + 30)
+        return self.api_request(url)
 
     async def parse_results(self, r, query):
         results = set()

bbot/modules/internal/speculate.py

@@ -176,8 +176,8 @@ async def handle_event(self, event):
         org_stubs = set()
         if event.type == "DNS_NAME" and event.scope_distance == 0:
             tldextracted = self.helpers.tldextract(event.data)
-            registered_domain = getattr(tldextracted, "registered_domain", "")
-            if registered_domain:
+            top_domain_under_public_suffix = getattr(tldextracted, "top_domain_under_public_suffix", "")
+            if top_domain_under_public_suffix:
                 tld_stub = getattr(tldextracted, "domain", "")
                 if tld_stub:
                     decoded_tld_stub = self.helpers.smart_decode_punycode(tld_stub)

bbot/modules/shodan_idb.py

@@ -1,4 +1,5 @@
 from bbot.modules.base import BaseModule
+import time
 
 
 class shodan_idb(BaseModule):
@@ -46,23 +47,48 @@ class shodan_idb(BaseModule):
         "created_date": "2023-12-22",
         "author": "@TheTechromancer",
     }
+    options = {"retries": None}
+    options_desc = {
+        "retries": "How many times to retry API requests (e.g. after a 429 error). Overrides the global web.api_retries setting."
+    }
 
-    # we get lots of 404s, that's normal
+    # we typically don't want to abort this module
     _api_failure_abort_threshold = 9999999999
 
-    # there aren't any rate limits to speak of, so our outgoing queue can be pretty big
-    _qsize = 500
+    # since there are rate limits, we set a lower qsize
+    # this way when our queue is full, we can give the API a break
+    _qsize = 100
 
     base_url = "https://internetdb.shodan.io"
 
+    async def setup(self):
+        await super().setup()
+        self.last_request_time = 0
+        return True
+
     def _incoming_dedup_hash(self, event):
         return hash(self.get_ip(event))
 
+    @property
+    def api_retries(self):
+        # allow the module to override global retry setting
+        return self.config.get("retries", None) or super().api_retries
+
     async def handle_event(self, event):
         ip = self.get_ip(event)
         if ip is None:
             return
         url = f"{self.base_url}/{ip}"
+
+        # Rate limiting: ensure at least 1 second between requests
+        current_time = time.time()
+        time_since_last = current_time - self.last_request_time
+        if time_since_last < 1:
+            await self.helpers.sleep(1 - time_since_last)
+
+        # Update the last request time
+        self.last_request_time = time.time()
+
         r = await self.api_request(url)
         if r is None:
             self.debug(f"No response for {event.data}")

bbot/modules/telerik.py

@@ -1,5 +1,4 @@
 from sys import executable
-from urllib.parse import urlparse
 
 from bbot.modules.base import BaseModule
 

bbot/modules/templates/webhook.py

@@ -16,10 +16,9 @@ class WebhookOutputModule(BaseOutputModule):
     # abort module after 10 failed requests (not including retries)
     _api_failure_abort_threshold = 10
     # retry each request up to 10 times, respecting the Retry-After header
-    _api_retries = 10
+    _default_api_retries = 10
 
     async def setup(self):
-        self._api_retries = self.config.get("retries", 10)
         self.webhook_url = self.config.get("webhook_url", "")
         self.min_severity = self.config.get("min_severity", "LOW").strip().upper()
         assert self.min_severity in self.vuln_severities, (
@@ -31,6 +30,10 @@ async def setup(self):
             return False
         return await super().setup()
 
+    @property
+    def api_retries(self):
+        return self.config.get("retries", self._default_api_retries)
+
     async def handle_event(self, event):
         message = self.format_message(event)
         data = {self.content_key: message}

bbot/scanner/__init__.py

@@ -1,2 +1,4 @@
 from .preset import Preset
 from .scanner import Scanner
+
+__all__ = ["Preset", "Scanner"]

bbot/scanner/preset/__init__.py

@@ -1 +1,3 @@
 from .preset import Preset
+
+__all__ = ["Preset"]

bbot/scanner/target.py

@@ -28,6 +28,8 @@ class BaseTarget(RadixTarget):
     accept_target_types = ["TARGET"]
 
     def __init__(self, *targets, **kwargs):
+        # ignore blank targets (sometimes happens as a symptom of .splitlines())
+        targets = [stripped for t in targets if (stripped := (t.strip() if isinstance(t, str) else t))]
         self.event_seeds = set()
         super().__init__(*targets, **kwargs)
 

bbot/test/bbot_fixtures.py

@@ -1,11 +1,9 @@
 import os  # noqa
-import sys
-import zlib
+import sys  # noqa
 import pytest
 import shutil  # noqa
 import asyncio  # noqa
 import logging
-import subprocess
 import tldextract
 import pytest_httpserver
 from pathlib import Path
@@ -16,8 +14,8 @@
 from bbot.errors import *  # noqa: F401
 from bbot.core import CORE
 from bbot.scanner import Preset
+from bbot.core.helpers.misc import mkdir, rand_string
 from bbot.core.helpers.async_helpers import get_event_loop
-from bbot.core.helpers.misc import mkdir, rand_string, get_python_constraints
 
 
 log = logging.getLogger("bbot.test.fixtures")

bbot/test/conftest.py

@@ -1,5 +1,6 @@
 import os
 import ssl
+import time
 import shutil
 import pytest
 import asyncio
@@ -8,8 +9,6 @@
 from contextlib import suppress
 from omegaconf import OmegaConf
 from pytest_httpserver import HTTPServer
-import time
-import queue
 
 from bbot.core import CORE
 from bbot.core.helpers.misc import execute_sync_or_async

bbot/test/test_step_1/test_presets.py

@@ -1079,8 +1079,6 @@ async def test_preset_output_dir():
 
 # regression test for https://github.com/blacklanternsecurity/bbot/issues/2337
 def test_preset_serialization():
-    from ipaddress import ip_address, ip_network
-
     preset = Preset("192.168.1.1")
     preset = preset.bake()
 

bbot/test/test_step_2/module_tests/test_module_censys.py

@@ -7,7 +7,7 @@ class TestCensys(ModuleTestBase):
     async def setup_before_prep(self, module_test):
         module_test.httpx_mock.add_response(
             url="https://search.censys.io/api/v1/account",
-            # match_headers={"Authorization": "Basic YXBpX2lkOmFwaV9zZWNyZXQ="},
+            match_headers={"Authorization": "Basic YXBpX2lkOmFwaV9zZWNyZXQ="},
             json={
                 "email": "info@blacklanternsecurity.com",
                 "login": "nope",
@@ -18,8 +18,9 @@ async def setup_before_prep(self, module_test):
         )
         module_test.httpx_mock.add_response(
             url="https://search.censys.io/api/v2/certificates/search",
-            # match_headers={"Authorization": "Basic YXBpX2lkOmFwaV9zZWNyZXQ="},
-            match_content=b'{"q": "names: blacklanternsecurity.com", "per_page": 100}',
+            match_headers={"Authorization": "Basic YXBpX2lkOmFwaV9zZWNyZXQ="},
+            method="POST",
+            match_json={"q": "names: blacklanternsecurity.com", "per_page": 100},
             json={
                 "code": 200,
                 "status": "OK",
@@ -47,8 +48,9 @@ async def setup_before_prep(self, module_test):
         )
         module_test.httpx_mock.add_response(
             url="https://search.censys.io/api/v2/certificates/search",
-            # match_headers={"Authorization": "Basic YXBpX2lkOmFwaV9zZWNyZXQ="},
-            match_content=b'{"q": "names: blacklanternsecurity.com", "per_page": 100, "cursor": "NextToken"}',
+            match_headers={"Authorization": "Basic YXBpX2lkOmFwaV9zZWNyZXQ="},
+            method="POST",
+            match_json={"q": "names: blacklanternsecurity.com", "per_page": 100, "cursor": "NextToken"},
             json={
                 "code": 200,
                 "status": "OK",

bbot/test/test_step_2/module_tests/test_module_dotnetnuke.py

@@ -1,4 +1,3 @@
-import asyncio
 import re
 from .base import ModuleTestBase
 from werkzeug.wrappers import Response

bbot/test/test_step_2/module_tests/test_module_host_header.py

@@ -1,4 +1,3 @@
-import asyncio
 import re
 from werkzeug.wrappers import Response