Pick up more master branch commits to 10.2.x for 10.2.0

NOTICE

@@ -95,6 +95,7 @@ https://github.com/jbeder/yaml-cpp
 ~~
 
 fastlz: an ANSI C/C90 implementation of Lempel-Ziv 77 algorithm (LZ77) of lossless data compression.
+Copyright (C) 2005-2020 Ariya Hidayat (MIT License)
 https://github.com/ariya/FastLZ
 
 ~~

cmake/ExperimentalPlugins.cmake

@@ -83,7 +83,6 @@ auto_option(
 )
 auto_option(RATE_LIMIT FEATURE_VAR BUILD_RATE_LIMIT DEFAULT ${_DEFAULT})
 auto_option(REALIP FEATURE_VAR BUILD_REALIP DEFAULT ${_DEFAULT})
-auto_option(REDO_CACHE_LOOKUP FEATURE_VAR BUILD_REDO_CACHE_LOOKUP DEFAULT ${_DEFAULT})
 auto_option(SSLHEADERS FEATURE_VAR BUILD_SSLHEADERS DEFAULT ${_DEFAULT})
 auto_option(STALE_RESPONSE FEATURE_VAR BUILD_STALE_RESPONSE DEFAULT ${_DEFAULT})
 auto_option(

doc/admin-guide/files/records.yaml.en.rst

@@ -982,7 +982,7 @@ allow-plain
 
 .. note::
 
-   The ``Via`` transaction codes can be decoded with the `Via Decoder Ring <https://trafficserver.apache.org/tools/via>`_.
+   The ``Via`` transaction codes can be decoded with the `Via Decoder Ring <https://trafficserver.apache.org/via.html>`_.
 
 .. ts:cv:: CONFIG proxy.config.http.request_via_str STRING ApacheTrafficServer/${PACKAGE_VERSION}
    :reloadable:
@@ -1008,7 +1008,7 @@ allow-plain
 
 .. note::
 
-   The ``Via`` transaction code can be decoded with the `Via Decoder Ring <https://trafficserver.apache.org/tools/via>`_.
+   The ``Via`` transaction code can be decoded with the `Via Decoder Ring <https://trafficserver.apache.org/via.html>`_.
 
 .. ts:cv:: CONFIG proxy.config.http.response_via_str STRING ApacheTrafficServer/${PACKAGE_VERSION}
    :reloadable:

doc/admin-guide/plugins/maxmind_acl.en.rst

@@ -113,4 +113,49 @@ The plugin also supports optional fields from GeoGuard databases which includes:
 ``vpn_datacenter``
 ``relay_proxy``
 ``proxy_over_vpn``
-``smart_dns_proxy``
+``smart_dns_proxy``
+
+Bypass
+======
+
+An optional ``bypass`` field allows a request to skip all geo checks entirely and pass through
+unmodified. Both a header name and an expected value must be configured; when the named header
+is present in the request **and** its value matches exactly, the plugin returns immediately
+without performing any country, IP, regex, or anonymous evaluation.
+
+``header``
+   Required sub-key. The name of the HTTP request header to look for, e.g. ``@GeoBypass``.
+
+``value``
+   Required sub-key. The header field value must match this string exactly for the bypass to
+   trigger. Both ``header`` and ``value`` must be present and non-empty; omitting either
+   disables the bypass entirely and a warning is emitted to the ATS error log.
+
+The comparison uses the complete, raw field value of the first occurrence of the named header.
+Duplicate headers with the same name (repeated lines) are ignored — only the first is evaluated.
+Within that first field, the entire value must match exactly, so a comma-separated multi-value
+(e.g. ``@GeoBypass: 1, extra``) in a single header line will not match a simple configured value.
+
+An example configuration ::
+
+   maxmind:
+    database: GeoIP2-City.mmdb
+    bypass:
+     header: "@GeoBypass"
+     value: "1"
+    allow:
+     country:
+      - US
+
+This is useful for internal or trusted upstream services that should not be subject to geo
+restrictions. If ``bypass`` is absent from the configuration, or if either ``header`` or
+``value`` is missing, bypass is disabled and all requests are evaluated normally.
+
+.. warning::
+
+   Because the bypass skips **all** ACL checks, the configured header must be
+   unforgeable by external clients. Use an internal ``@``-prefixed header (e.g.
+   ``@GeoBypass``) that is set by ATS itself or a trusted upstream, or
+   ensure the edge strips/overwrites the header before it reaches this plugin.
+   Configuring a normal client-supplied header allows end users to opt out of
+   geo restrictions by simply sending the header in their request.

doc/appendices/faq.en.rst

@@ -106,7 +106,7 @@ Please refer to the :ref:`forward-proxy` documentation.
 How do I interpret the Via: header code?
 ----------------------------------------
 
-The ``Via`` header string can be decoded with the `Via Decoder Ring <https://trafficserver.apache.org/tools/via>`_.
+The ``Via`` header string can be decoded with the `Via Decoder Ring <https://trafficserver.apache.org/via.html>`_.
 
 The Via header is an optional HTTP header added by Traffic Server and other HTTP proxies. If a request goes through multiple proxies, each one appends its Via header content to the end of the existing Via header. Via header content is for general information and diagnostic use only and should not be used as a programmatic interface to Traffic Server. The header is cached by each intermediary with the object as received from its downstream node. Thus, the last node in the list to report a cache hit is the end of the transaction for that specific request. Nodes reported earlier were from a previous transaction.
 

doc/developer-guide/api/functions/TSHttpHdrHostGet.en.rst

@@ -43,3 +43,19 @@ header field.
 
    This is much faster than calling :func:`TSHttpTxnEffectiveUrlStringGet` and
    extracting the host from the result.
+
+.. note::
+
+   :func:`TSHttpHdrHostGet` checks both the URL and the ``Host`` header field,
+   making it reliable at any hook stage. In contrast, :func:`TSUrlHostGet`
+   operates only on the URL object obtained from :func:`TSHttpHdrUrlGet`. In
+   early hooks such as ``TS_HTTP_READ_REQUEST_HDR_HOOK``, the URL object may
+   not yet be fully parsed, and :func:`TSUrlHostGet` may return ``NULL`` even
+   when a ``Host`` header is present.
+
+See Also
+========
+
+:func:`TSUrlHostGet`,
+:func:`TSHttpHdrUrlGet`,
+:func:`TSHttpTxnEffectiveUrlStringGet`

doc/developer-guide/api/functions/TSHttpHdrUrlGet.en.rst

@@ -42,11 +42,23 @@ The value placed in :arg:`locp` is stable only for a single callback, as other c
 change the URL object itself (see :func:`TSHttpHdrUrlSet`), not just the data in it. That value is
 also valid only if this function return ``TS_SUCCESS``.
 
+.. note::
+
+   Not all URL components may be available at every hook stage. In early hooks
+   such as ``TS_HTTP_READ_REQUEST_HDR_HOOK``, the URL object may not yet be
+   fully parsed. In particular, the host component retrieved via
+   :func:`TSUrlHostGet` may be ``NULL`` even when a ``Host`` header is present.
+   For reliable host retrieval across all hook stages, use
+   :func:`TSHttpHdrHostGet` instead, which checks both the URL and the ``Host``
+   header field.
+
 See Also
 ========
 
 :manpage:`TSAPI(3ts)`,
 :manpage:`TSHttpTxnClientReqGet(3ts)`,
 :manpage:`TSHttpTxnServerReqGet(3ts)`,
 :manpage:`TSHttpTxnServerRespGet(3ts)`,
-:manpage:`TSHttpTxnClientRespGet(3ts)`
+:manpage:`TSHttpTxnClientRespGet(3ts)`,
+:manpage:`TSHttpHdrHostGet(3ts)`,
+:manpage:`TSUrlHostGet(3ts)`

doc/developer-guide/api/functions/TSHttpTxnClientPacketMarkSet.en.rst

@@ -32,11 +32,24 @@ Synopsis
 Description
 ===========
 
-Change packet firewall :arg:`mark` for the client side connection.
+Change the packet firewall :arg:`mark` for the client side connection. The
+entire firewall mark is replaced with :arg:`mark`, which is interpreted as a
+32-bit unsigned bit pattern.
+
+Returns :const:`TS_SUCCESS` when the client connection was modified, and
+:const:`TS_ERROR` when there is no client connection to modify.
+
+.. note::
+
+   The firewall mark is only honored on platforms whose OS supports it,
+   specifically Linux via ``SO_MARK``. On platforms without ``SO_MARK`` support
+   the call still returns :const:`TS_SUCCESS` when a client connection is
+   present, but setting the mark has no effect at the OS layer (it is a safe
+   no-op).
 
 .. note::
 
-   Changes take effect immediately.
+   The change takes effect immediately on the live client connection.
 
 See Also
 ========

doc/developer-guide/api/functions/TSHttpTxnServerPacketMarkSet.en.rst

@@ -20,8 +20,6 @@
 TSHttpTxnServerPacketMarkSet
 ****************************
 
-Change packet firewall mark for the server side connection.
-
 Synopsis
 ========
 
@@ -34,13 +32,25 @@ Synopsis
 Description
 ===========
 
+Change the packet firewall :arg:`mark` for the server side (origin) connection.
+The entire firewall mark is replaced with :arg:`mark`, which is interpreted as a
+32-bit unsigned bit pattern.
+
+Always returns :const:`TS_SUCCESS`, including when no server connection has been
+established yet.
+
 .. note::
 
-   The change takes effect immediately. If no OS connection has been
-   made, then this sets the mark that will be used. If an OS connection
-   is established
+   The firewall mark is only honored on platforms whose OS supports it,
+   specifically Linux via ``SO_MARK``. On platforms without ``SO_MARK`` support
+   the call still returns :const:`TS_SUCCESS`, but setting the mark has no effect
+   at the OS layer (it is a safe no-op).
+
+.. note::
 
-.. XXX Third sentence above needs to be completed.
+   If a live server connection exists, the mark is applied to it immediately; the
+   mark is also recorded on the transaction so that any subsequent server
+   connection for this transaction uses it.
 
 See Also
 ========

doc/developer-guide/api/functions/TSUrlHostGet.en.rst

@@ -71,6 +71,16 @@ scheme.
 :arg:`offset` within the marshal buffer :arg:`bufp`. If there is no explicit
 port number in the URL, zero is returned.
 
+.. note::
+
+   :func:`TSUrlHostGet` operates on a URL object obtained from
+   :func:`TSHttpHdrUrlGet`. In early hooks such as
+   ``TS_HTTP_READ_REQUEST_HDR_HOOK``, the URL object may not yet be fully
+   parsed, and :func:`TSUrlHostGet` may return ``NULL`` even when a ``Host``
+   header is present. For reliable host retrieval at any hook stage, use
+   :func:`TSHttpHdrHostGet` instead, which checks both the URL and the
+   ``Host`` header field.
+
 Return Values
 =============
 
@@ -90,6 +100,7 @@ See Also
 :manpage:`TSAPI(3ts)`,
 :manpage:`TSUrlCreate(3ts)`,
 :manpage:`TSHttpHdrUrlGet(3ts)`,
+:manpage:`TSHttpHdrHostGet(3ts)`,
 :manpage:`TSUrlHostSet(3ts)`,
 :manpage:`TSUrlStringGet(3ts)`,
 :manpage:`TSUrlPercentEncode(3ts)`

example/plugins/c-api/CMakeLists.txt

@@ -24,6 +24,7 @@ add_atsplugin(secure_link ./secure_link/secure_link.cc)
 target_link_libraries(secure_link PRIVATE OpenSSL::SSL)
 add_atsplugin(remap ./remap/remap.cc)
 add_atsplugin(redirect_1 ./redirect_1/redirect_1.cc)
+add_atsplugin(redo_cache_lookup ./redo_cache_lookup/redo_cache_lookup.cc)
 add_atsplugin(query_remap ./query_remap/query_remap.cc)
 add_atsplugin(thread_pool ./thread_pool/psi.cc ./thread_pool/thread.cc)
 add_atsplugin(bnull_transform ./bnull_transform/bnull_transform.cc)
@@ -66,3 +67,10 @@ add_atsplugin(protocol_stack ./protocol_stack/protocol_stack.cc)
 add_atsplugin(client_context_dump ./client_context_dump/client_context_dump.cc)
 target_link_libraries(client_context_dump PRIVATE OpenSSL::SSL libswoc::libswoc)
 add_atsplugin(custom_logfield ./custom_logfield/custom_logfield.cc)
+
+if(BUILD_TESTING)
+  add_executable(test_redo_cache_lookup ./redo_cache_lookup/unit_tests/test_redo_cache_lookup_config.cc)
+  target_include_directories(test_redo_cache_lookup PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/redo_cache_lookup)
+  target_link_libraries(test_redo_cache_lookup PRIVATE Catch2::Catch2WithMain)
+  add_catch2_test(NAME test_redo_cache_lookup COMMAND test_redo_cache_lookup)
+endif()

example/plugins/c-api/redirect_1/redirect_1.cc

@@ -97,7 +97,7 @@ static void
 handle_client_lookup(TSHttpTxn txnp, TSCont contp)
 {
   TSMBuffer bufp;
-  TSMLoc    hdr_loc, url_loc;
+  TSMLoc    hdr_loc;
   int       host_length;
 
   in_addr_t clientip = 0;
@@ -130,16 +130,9 @@ handle_client_lookup(TSHttpTxn txnp, TSCont contp)
     goto done;
   }
 
-  if (TSHttpHdrUrlGet(bufp, hdr_loc, &url_loc) != TS_SUCCESS) {
-    TSError("[%s] Couldn't retrieve request url", PLUGIN_NAME);
-    TSHandleMLocRelease(bufp, TS_NULL_MLOC, hdr_loc);
-    goto done;
-  }
-
-  host = TSUrlHostGet(bufp, url_loc, &host_length);
+  host = TSHttpHdrHostGet(bufp, hdr_loc, &host_length);
   if (!host) {
     TSError("[%s] Couldn't retrieve request hostname", PLUGIN_NAME);
-    TSHandleMLocRelease(bufp, hdr_loc, url_loc);
     TSHandleMLocRelease(bufp, TS_NULL_MLOC, hdr_loc);
     goto done;
   }
@@ -148,7 +141,6 @@ handle_client_lookup(TSHttpTxn txnp, TSCont contp)
    *   Check to see if the client is already headed to the redirect site.
    */
   if (strncmp(host, url_redirect, host_length) == 0) {
-    TSHandleMLocRelease(bufp, hdr_loc, url_loc);
     TSHandleMLocRelease(bufp, TS_NULL_MLOC, hdr_loc);
     goto done;
   }
@@ -159,7 +151,6 @@ handle_client_lookup(TSHttpTxn txnp, TSCont contp)
 
     update_redirected_method_stats(bufp, hdr_loc);
 
-    TSHandleMLocRelease(bufp, hdr_loc, url_loc);
     TSHandleMLocRelease(bufp, TS_NULL_MLOC, hdr_loc);
 
     /*

example/plugins/c-api/redo_cache_lookup/readme.txt

@@ -0,0 +1,13 @@
+# Redo Cache Lookup Example Plugin
+
+This plugin shows how to use the `TSHttpTxnRedoCacheLookup` C API. It
+checks cache lookup results and asks ATS to retry the lookup with a fallback
+URL when the original lookup misses or is skipped.
+
+## Configuration
+
+Add this plugin to `plugin.config` with the `--fallback` option:
+
+```
+redo_cache_lookup.so --fallback http://example.com/fallback_url
+```

example/plugins/c-api/redo_cache_lookup/redo_cache_lookup.cc

@@ -0,0 +1,91 @@
+/** @file
+
+  An example plugin to redo cache lookups with a fallback URL.
+
+  @section license License
+
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+ */
+
+#include <string>
+#include <string_view>
+
+#include "ts/ts.h"
+#include "redo_cache_lookup_config.h"
+
+#define PLUGIN_NAME "redo_cache_lookup"
+
+namespace
+{
+DbgCtl dbg_ctl{PLUGIN_NAME};
+
+struct RedoCacheLookupConfig {
+  RedoCacheLookupConfig(std::string_view fallback) : fallback(fallback) {}
+
+  std::string fallback;
+};
+
+int
+handle_cache_lookup_complete(TSCont contp, TSEvent event, void *edata)
+{
+  if (event != TS_EVENT_HTTP_CACHE_LOOKUP_COMPLETE) {
+    return 0;
+  }
+
+  TSHttpTxn txnp   = static_cast<TSHttpTxn>(edata);
+  auto     *config = static_cast<RedoCacheLookupConfig *>(TSContDataGet(contp));
+  int       status = TS_CACHE_LOOKUP_MISS;
+
+  if (TSHttpTxnCacheLookupStatusGet(txnp, &status) != TS_SUCCESS || status == TS_CACHE_LOOKUP_MISS ||
+      status == TS_CACHE_LOOKUP_SKIPPED) {
+    Dbg(dbg_ctl, "rewinding to check for fallback url: %s", config->fallback.c_str());
+    TSHttpTxnRedoCacheLookup(txnp, config->fallback.c_str(), static_cast<int>(config->fallback.size()));
+  }
+
+  TSHttpTxnReenable(txnp, TS_EVENT_HTTP_CONTINUE);
+  return 0;
+}
+} // namespace
+
+void
+TSPluginInit(int argc, const char *argv[])
+{
+  TSPluginRegistrationInfo info;
+
+  Dbg(dbg_ctl, "Init");
+  info.plugin_name   = PLUGIN_NAME;
+  info.vendor_name   = "Apache Software Foundation";
+  info.support_email = "dev@trafficserver.apache.org";
+
+  if (TSPluginRegister(&info) != TS_SUCCESS) {
+    TSError("[%s] Plugin registration failed", PLUGIN_NAME);
+    return;
+  }
+
+  auto fallback = redo_cache_lookup::parse_fallback_url(argc, argv);
+
+  if (!fallback) {
+    Dbg(dbg_ctl, "Missing fallback option.");
+    TSError("[%s] Missing fallback option", PLUGIN_NAME);
+    return;
+  }
+  Dbg(dbg_ctl, "Initialized with fallback: %s", fallback->c_str());
+
+  TSCont contp = TSContCreate(handle_cache_lookup_complete, nullptr);
+  TSContDataSet(contp, new RedoCacheLookupConfig(*fallback));
+  TSHttpHookAdd(TS_HTTP_CACHE_LOOKUP_COMPLETE_HOOK, contp);
+}