Making boolean inference some more robust against jitter

Author: stamparm

lib/controller/checks.py

@@ -818,6 +818,12 @@ def genCmpPayload():
                         injection.data[stype].trueCode = trueCode
                         injection.data[stype].falseCode = falseCode
 
+                        # reference bodies for inference.py's "resembles neither TRUE nor FALSE model"
+                        # anomaly guard (runtime-only; lets a transient same-HTTP-code junk response
+                        # trigger a validateChar re-check during boolean extraction)
+                        if method == PAYLOAD.METHOD.COMPARISON:
+                            kb.trueTemplate, kb.falseTemplate = truePage, falsePage
+
                         injection.conf.textOnly = conf.textOnly
                         injection.conf.titles = conf.titles
                         injection.conf.code = conf.code

lib/core/option.py

@@ -2305,6 +2305,10 @@ def _setKnowledgeBaseAttributes(flushAll=True):
     kb.pageTemplate = None
     kb.pageTemplates = dict()
     kb.pageEncoding = DEFAULT_PAGE_ENCODING
+
+    # calibrated TRUE/FALSE reference bodies for the boolean same-HTTP-code anomaly guard (inference.py)
+    kb.trueTemplate = None
+    kb.falseTemplate = None
     kb.pageStable = None
     kb.pageStructurallyStable = None
     kb.partRun = None

lib/core/settings.py

@@ -20,7 +20,7 @@
 from thirdparty import six
 
 # sqlmap version (<major>.<minor>.<month>.<monthly commit>)
-VERSION = "1.10.7.241"
+VERSION = "1.10.7.242"
 TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable"
 TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34}
 VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE)
@@ -103,6 +103,11 @@
 LOWER_RATIO_BOUND = 0.02
 UPPER_RATIO_BOUND = 0.98
 
+# Minimum similarity at which a boolean extraction response is judged to "resemble" the calibrated
+# TRUE or FALSE model. A response resembling NEITHER (a transient same-HTTP-code junk page: WAF/CDN
+# interstitial, captcha, maintenance, empty/truncated body) triggers an extra validateChar re-check.
+BOOLEAN_MODEL_MATCH_RATIO = 0.9
+
 # Number of candidate names probed per request while mining for hidden parameters ('--mine-params')
 PARAMETER_MINING_BUCKET_SIZE = 25
 

lib/techniques/blind/inference.py

@@ -7,6 +7,7 @@
 
 from __future__ import division
 
+import difflib
 import heapq
 import re
 import time
@@ -26,6 +27,7 @@
 from lib.core.common import getTechniqueData
 from lib.core.common import getText
 from lib.core.common import predictValue
+from lib.core.common import removeDynamicContent
 from lib.core.common import hashDBRetrieve
 from lib.core.common import hashDBWrite
 from lib.core.common import incrementCounter
@@ -45,6 +47,7 @@
 from lib.core.exception import SqlmapThreadException
 from lib.core.exception import SqlmapUnsupportedFeatureException
 from lib.core.wordlist import Wordlist
+from lib.core.settings import BOOLEAN_MODEL_MATCH_RATIO
 from lib.core.settings import CHAR_INFERENCE_MARK
 from lib.core.settings import HUFFMAN_PROBE_LIMIT
 from lib.core.settings import HUFFMAN_PRIOR_WEIGHTS
@@ -256,6 +259,31 @@ def oracleReliabilityLitmus(expressionUnescaped, value, timeBasedCompare):
 
     return bool(mustBeTrue) and not bool(mustBeFalse)
 
+def _resemblesNeitherModel(page):
+    """
+    Returns True when a boolean extraction response resembles NEITHER the calibrated TRUE nor FALSE
+    model (kb.trueTemplate / kb.falseTemplate). A transient response that keeps the expected HTTP code
+    but swaps the body for junk (WAF/CDN interstitial, captcha, maintenance banner, empty/truncated
+    page) is invisible to the HTTP-code check, so it is flagged here for an extra validateChar re-check.
+
+    This only ever ADDS a re-validation - it never changes a decided bit - so it is a safe no-op on a
+    clean target (every response resembles its own model) and when no models were recorded (a resumed
+    session, or a non-boolean technique).
+    """
+
+    refs = [_ for _ in (kb.trueTemplate, kb.falseTemplate) if _]
+    if not refs:
+        return False
+    if page is None:
+        return True
+
+    cleaned = removeDynamicContent(page)
+    for ref in refs:
+        if difflib.SequenceMatcher(None, removeDynamicContent(ref), cleaned).quick_ratio() >= BOOLEAN_MODEL_MATCH_RATIO:
+            return False
+
+    return True
+
 def bisection(payload, expression, length=None, charsetType=None, firstChar=None, lastChar=None, dump=False):
     """
     Bisection algorithm that can be used to perform blind SQL injection
@@ -714,6 +742,7 @@ def getChar(idx, charTbl=None, continuousOrder=True, expand=charsetType is None,
             firstCheck = False
             lastCheck = False
             unexpectedCode = False
+            unexpectedResponse = False
 
             if continuousOrder:
                 while len(charTbl) > 1:
@@ -787,6 +816,13 @@ def getChar(idx, charTbl=None, continuousOrder=True, expand=charsetType is None,
 
                             singleTimeWarnMessage(warnMsg)
 
+                        # same-HTTP-code body anomaly (WAF/CDN interstitial, captcha, maintenance, empty
+                        # or truncated body) - invisible to the code check above, so re-validate when the
+                        # response resembles neither calibrated model
+                        elif not unexpectedResponse and not kb.nullConnection and _resemblesNeitherModel(threadData.lastPage):
+                            unexpectedResponse = True
+                            singleTimeWarnMessage("unexpected response content detected. Will use (extra) validation step in similar cases")
+
                     if result:
                         minValue = posValue
 
@@ -826,7 +862,7 @@ def getChar(idx, charTbl=None, continuousOrder=True, expand=charsetType is None,
                             retVal = minValue + 1
 
                             if retVal in originalTbl or (retVal == ord('\n') and CHAR_INFERENCE_MARK in payload):
-                                if (timeBasedCompare or unexpectedCode) and kb.get("timeless") is None and not validateChar(idx, retVal):
+                                if (timeBasedCompare or unexpectedCode or unexpectedResponse) and kb.get("timeless") is None and not validateChar(idx, retVal):
                                     if restricted:
                                         # the character fell outside this column's observed range - re-extract
                                         # over the full charset (not timing noise, so no delay increase / retry count)

tests/test_boolean_jitter.py

@@ -51,8 +51,14 @@
 _PARSE = re.compile(r"IDX=(\d+) CMP(!=|=|>)(\d+)")
 _SECRET = "Str0ng!"
 _STRING = "luther"
-_TRUE_BODY = "<html><body>welcome %s, here is your dashboard with 12 private items</body></html>" % _STRING
-_FALSE_BODY = "<html><body>invalid credentials, no such record, please retry</body></html>"
+# realistic-size bodies (shared nav/footer boilerplate) so the "resembles neither model" anomaly guard
+# behaves as on a real page: benign dynamic noise is proportionally tiny (stays a match), while a junk
+# interstitial/maintenance/empty body clearly matches neither
+_BOILER = "<html><head><title>Acme Portal</title></head><body><nav>home about contact help terms privacy account</nav><div class=main>" * 8
+_FOOT = "</div><footer>(c) Acme Corp - all rights reserved - support@acme.example - v4.2</footer></body></html>" * 8
+_TRUE_BODY = _BOILER + "welcome %s, dashboard: orders profile settings billing (12 items)" % _STRING + _FOOT
+_FALSE_BODY = _BOILER + "invalid credentials, no such record found, please retry" + _FOOT
+_INTERSTITIAL = "<html><body>Just a moment... checking your browser before access (DDoS protection)</body></html>"
 _STRESS = os.environ.get("SQLMAP_JITTER_STRESS")
 
 
@@ -97,7 +103,7 @@ class _BooleanJitterBase(unittest.TestCase):
     _KB = ("negativeLogic", "nullConnection", "errorIsNone", "pageTemplate", "matchRatio", "heavilyDynamic",
            "pageStructurallyStable", "skipSeqMatcher", "pageEncoding", "partRun", "safeCharEncode",
            "bruteMode", "fileReadMode", "disableShiftTable", "prependFlag", "timeless", "counters",
-           "originalCode", "originalPage")
+           "originalCode", "originalPage", "trueTemplate", "falseTemplate", "dynamicMarkings")
 
     def setUp(self):
         self._saved_conf = {k: conf.get(k) for k in self._CONF}
@@ -129,6 +135,8 @@ def _configure(self):
         kb.partRun = None; kb.safeCharEncode = False; kb.bruteMode = False; kb.fileReadMode = False
         kb.disableShiftTable = False; kb.prependFlag = False; kb.timeless = None; kb.counters = {}
         kb.originalCode = None; kb.originalPage = None
+        # calibrated reference bodies for the same-code anomaly guard (Fix B); no learned dynamic markings
+        kb.trueTemplate = _TRUE_BODY; kb.falseTemplate = _FALSE_BODY; kb.dynamicMarkings = []
         kb.injection.data = {PAYLOAD.TECHNIQUE.BOOLEAN: _vector()}
         setTechnique(PAYLOAD.TECHNIQUE.BOOLEAN)
         kb.data.processChar = None
@@ -224,6 +232,36 @@ def respond(payload, cond):
         self._configure()
         self.assertEqual(self._extract(respond), _SECRET)
 
+    def test_anomaly_classifier_flags_only_junk(self):
+        # Fix B core: a response resembling NEITHER calibrated model is flagged; the models themselves
+        # and a benign dynamic variant are not. Deterministic, no network.
+        self._configure()
+        self.assertFalse(inf._resemblesNeitherModel(_TRUE_BODY))
+        self.assertFalse(inf._resemblesNeitherModel(_FALSE_BODY))
+        self.assertFalse(inf._resemblesNeitherModel(_TRUE_BODY.replace("dashboard", "dashboard <b>7 new</b> tok=abc123")))
+        for junk in (_INTERSTITIAL, "", "<html><h1>502 Bad Gateway</h1></html>", _TRUE_BODY[:60]):
+            self.assertTrue(inf._resemblesNeitherModel(junk), msg="must flag junk %r" % junk[:40])
+
+    def test_same_code_body_jitter_is_ridden_out(self):
+        # Fix B guard: a transient same-HTTP-code junk page that resembles NEITHER model makes a
+        # character mis-resolve to a wrong (valid) value; the anomaly guard triggers validateChar to
+        # re-extract it. The junk here carries the --string token (so it reads True and pushes the char
+        # HIGH -> a wrong valid char, the case validateChar covers), and is unlike both models -> flagged.
+        junk = "<html><body>notice: %s service temporarily degraded, retry</body></html>" % _STRING
+        poisoned = {"n": 0}
+
+        def respond(payload, cond):
+            m = _PARSE.search(payload)
+            idx = int(m.group(1)) if m else 0
+            if idx == 4 and "!=" not in payload and poisoned["n"] < 3:
+                poisoned["n"] += 1
+                return junk, 200, "text/html"
+            return (_TRUE_BODY if cond else _FALSE_BODY), 200, "text/html"
+
+        self._configure()
+        self.assertTrue(inf._resemblesNeitherModel(junk))   # precondition: the junk IS anomalous
+        self.assertEqual(self._extract(respond), _SECRET)
+
 
 @unittest.skipUnless(_STRESS, "creative boolean-jitter sweep is opt-in (set SQLMAP_JITTER_STRESS=1)")
 class TestBooleanJitterSweep(_BooleanJitterBase):