From f41dcb75a38629dd1c8cc8a863b21592d6187628 Mon Sep 17 00:00:00 2001 From: Darin Kotter Date: Mon, 6 Apr 2026 13:48:07 -0600 Subject: [PATCH 01/10] Add a new image_meta field that can contain information about the image, link whether it's a link or has a caption. Support returning decorative alt text --- .../Abilities/Image/Alt_Text_Generation.php | 41 ++++++++++++++++--- 1 file changed, 35 insertions(+), 6 deletions(-) diff --git a/includes/Abilities/Image/Alt_Text_Generation.php b/includes/Abilities/Image/Alt_Text_Generation.php index 586395a17..6ffb637cb 100644 --- a/includes/Abilities/Image/Alt_Text_Generation.php +++ b/includes/Abilities/Image/Alt_Text_Generation.php @@ -57,6 +57,11 @@ protected function input_schema(): array { 'sanitize_callback' => 'sanitize_textarea_field', 'description' => esc_html__( 'Optional context about the image or surrounding content to improve alt text relevance.', 'ai' ), ), + 'image_meta' => array( + 'type' => 'string', + 'sanitize_callback' => 'sanitize_textarea_field', + 'description' => esc_html__( 'Structured metadata about how the image block is used, such as whether it is linked.', 'ai' ), + ), ), ); } @@ -70,10 +75,14 @@ protected function output_schema(): array { return array( 'type' => 'object', 'properties' => array( - 'alt_text' => array( + 'alt_text' => array( 'type' => 'string', 'description' => esc_html__( 'Generated alt text for the image.', 'ai' ), ), + 'is_decorative' => array( + 'type' => 'boolean', + 'description' => esc_html__( 'Whether the image was determined to be decorative.', 'ai' ), + ), ), ); } @@ -91,6 +100,7 @@ protected function execute_callback( $input ) { 'attachment_id' => null, 'image_url' => null, 'context' => '', + 'image_meta' => '', ), ); @@ -102,7 +112,11 @@ protected function execute_callback( $input ) { } // Generate the alt text. - $result = $this->generate_alt_text( $image_reference, normalize_content( $args['context'] ) ); + $result = $this->generate_alt_text( + $image_reference, + normalize_content( $args['context'] ), + sanitize_textarea_field( $args['image_meta'] ) + ); if ( is_wp_error( $result ) ) { return $result; @@ -115,6 +129,14 @@ protected function execute_callback( $input ) { ); } + // Detect the decorative token from the AI response. + if ( '[[DECORATIVE_ALT]]' === trim( $result ) ) { + return array( + 'alt_text' => '', + 'is_decorative' => true, + ); + } + // Return the alt text in the format the Ability expects. return array( 'alt_text' => sanitize_text_field( $result ), @@ -185,10 +207,11 @@ protected function get_image_reference( array $args ) { * * @param array{reference: string} $image_reference Prepared image reference containing a data URI. * @param string $context Optional context to improve alt text relevance. + * @param string $image_meta Optional metadata about how the image block is used. * @return string|\WP_Error The generated alt text or WP_Error on failure. */ - protected function generate_alt_text( array $image_reference, string $context = '' ) { - $result = wp_ai_client_prompt( $this->build_prompt( $context ) ) + protected function generate_alt_text( array $image_reference, string $context = '', string $image_meta = '' ) { + $result = wp_ai_client_prompt( $this->build_prompt( $context, $image_meta ) ) ->with_file( $image_reference['reference'] ) ->using_system_instruction( $this->get_system_instruction( 'alt-text-system-instruction.php' ) ) ->using_temperature( 0.3 ) @@ -368,12 +391,18 @@ protected function normalize_upload_url( string $url ): string { * * @since 0.3.0 * - * @param string $context Optional context about the image. + * @param string $context Optional context about the image. + * @param string $image_meta Optional metadata about how the image block is used. * @return string The prompt for the AI. */ - protected function build_prompt( string $context = '' ): string { + protected function build_prompt( string $context = '', string $image_meta = '' ): string { $prompt = __( 'Generate alt text for this image.', 'ai' ); + // If we have image block usage metadata, add it to the prompt. + if ( ! empty( $image_meta ) ) { + $prompt .= "\n\n" . $image_meta . ''; + } + // If we have additional context, add it to the prompt. if ( ! empty( $context ) ) { $prompt .= "\n\n" . $context . ''; From 4eada20ac31eb0eb8ec153fc88761e62c4a6f87e Mon Sep 17 00:00:00 2001 From: Darin Kotter Date: Mon, 6 Apr 2026 13:48:30 -0600 Subject: [PATCH 02/10] Update our system instructions to try and use the W3C Alt decision tree --- .../Image/alt-text-system-instruction.php | 38 +++++++++++++------ 1 file changed, 26 insertions(+), 12 deletions(-) diff --git a/includes/Abilities/Image/alt-text-system-instruction.php b/includes/Abilities/Image/alt-text-system-instruction.php index 502ab3e14..53a8c2f4d 100644 --- a/includes/Abilities/Image/alt-text-system-instruction.php +++ b/includes/Abilities/Image/alt-text-system-instruction.php @@ -12,21 +12,35 @@ // phpcs:ignore Squiz.PHP.Heredoc.NotAllowed return <<<'INSTRUCTION' -You are an accessibility expert that generates alt text for images on websites. +You are an accessibility expert that proposes alternative (alt) text for HTML images. Your output must follow the same decisions authors make with the W3C "An alt Decision Tree" (decorative vs functional vs informative vs complex images). -Goal: Analyze the provided image and generate concise, descriptive alt text that accurately describes the image content for users who cannot see it. The alt text should be optimized for screen readers and accessibility compliance. If additional context is provided, use it to generate a more relevant alt text. +Core rule: Alt text is not always a description of what the picture looks like. It must convey the information or purpose that the image serves in this specific context. If the image disappeared, what would be lost for someone who cannot see it—that is what belongs in alt text (or in empty alt when nothing should be announced). -Requirements for the alt text: +Follow this order: -- Be concise: Keep it under 125 characters when possible -- Be descriptive: Describe what is visually present in the image -- Be objective: Describe what you see, not interpretations or assumptions -- Avoid redundancy: Do not start with "Image of", "Picture of", or "Photo of" -- Include relevant details: People, objects, actions, colors, and context when meaningful -- Consider context: If context is provided, ensure the alt text is relevant to the surrounding content -- Plain text only: No markdown, quotes, or special formatting +1) Decorative or redundant? +- Purely decorative (flourish, spacer, visual-only styling) OR the same information is already in adjacent text (including visible link text in the same link as the image). +- Output: respond with exactly this token and nothing else: [[DECORATIVE_ALT]] +- Do not describe the image for decorative/redundant cases. -For images containing text, include the text in your description if it's essential to understanding the image. +2) Functional (image is a control or the main content of a link or button)? +- Examples: linked image with no other text in the link; icon-only button; logo linking home. +- Output: short text that describes the action or destination (where the link goes, what happens when activated)—not the photo or illustration subject. +- If `` gives a URL or destination name, base the alt on that purpose. Do not substitute a visual description of the image. -Respond with only the alt text, nothing else. +3) Informative (image adds meaning that is not covered by nearby text)? +- Output: concise objective description of the information the image communicates (people, objects, setting, actions) relevant to context. +- Do not start with "Image of", "Picture of", or "Photo of". +- If the image contains essential text, include that text in the alt (or summarize if it is very long, and note that a longer text alternative may be needed elsewhere). +- If `` is provided, use it to understand the purpose, subject, and relevance of the image within the article. Be sure to describe only information not already conveyed in nearby text + +4) Complex (chart, diagram, infographic, detailed map)? +- Output: a short summary of the main point; do not paste entire data sets into alt text. If context implies a longer description exists or should exist on the page, you may mention that the full explanation is in surrounding content. + +General requirements: +- Prefer under 125 characters when possible (except when essential text in the image requires more). +- Plain text only: no markdown, quotes, or labels—except the exact token [[DECORATIVE_ALT]] when appropriate. +- Use any `` section and `` to decide role and wording; they override guessing from pixels alone. + +Respond with only the alt text string, or exactly [[DECORATIVE_ALT]] for empty alternative text—nothing else. INSTRUCTION; From 29363575b394059171f35d37388fc877e4dc2333 Mon Sep 17 00:00:00 2001 From: Darin Kotter Date: Mon, 6 Apr 2026 13:49:27 -0600 Subject: [PATCH 03/10] Get additional information about the image to send, like if it's a link or has a caption. Provide support for decorative images and blank alt text --- .../components/AltTextControls.tsx | 69 ++++++++++++++-- src/experiments/alt-text-generation/types.ts | 1 + .../functions/upload-image.ts | 5 +- src/utils/generate-alt-text.ts | 81 ++++++++++++++++--- 4 files changed, 138 insertions(+), 18 deletions(-) diff --git a/src/experiments/alt-text-generation/components/AltTextControls.tsx b/src/experiments/alt-text-generation/components/AltTextControls.tsx index 7aa9f9173..b7e5f99d7 100644 --- a/src/experiments/alt-text-generation/components/AltTextControls.tsx +++ b/src/experiments/alt-text-generation/components/AltTextControls.tsx @@ -5,7 +5,12 @@ /** * WordPress dependencies */ -import { Button, TextareaControl, Spinner } from '@wordpress/components'; +import { + Button, + TextareaControl, + Spinner, + Notice, +} from '@wordpress/components'; import { InspectorControls } from '@wordpress/block-editor'; import { useState } from '@wordpress/element'; import { __ } from '@wordpress/i18n'; @@ -57,6 +62,7 @@ export function AltTextControls( { const [ isGenerating, setIsGenerating ] = useState< boolean >( false ); const [ generatedAlt, setGeneratedAlt ] = useState< string | null >( null ); + const [ isDecorative, setIsDecorative ] = useState< boolean >( false ); // Don't show controls if there's no image. if ( ! attachmentId && ! imageUrl ) { @@ -72,6 +78,7 @@ export function AltTextControls( { const handleGenerate = async () => { setIsGenerating( true ); setGeneratedAlt( null ); + setIsDecorative( false ); // Clear any previous notices. ( dispatch( noticesStore ) as any ).removeNotice( @@ -84,9 +91,24 @@ export function AltTextControls( { attachmentId, imageUrl, content, - clientId + clientId, + { + linkDestination: attributes?.linkDestination, + href: attributes?.href, + linkTarget: attributes?.linkTarget, + caption: + typeof attributes?.caption === 'string' + ? attributes.caption + : ( attributes?.caption as any )?.text, + } ); - setGeneratedAlt( result ); + + if ( result.is_decorative ) { + setIsDecorative( true ); + setGeneratedAlt( '' ); + } else { + setGeneratedAlt( result.alt_text ); + } } catch ( err: any ) { const errorMessage = err?.message || @@ -107,10 +129,13 @@ export function AltTextControls( { * Applies the generated alt text to the image block. */ const handleApply = () => { - if ( generatedAlt ) { + if ( isDecorative ) { + setAttributes( { alt: '' } ); + } else if ( generatedAlt ) { setAttributes( { alt: generatedAlt } ); - setGeneratedAlt( null ); } + setGeneratedAlt( null ); + setIsDecorative( false ); }; /** @@ -118,6 +143,7 @@ export function AltTextControls( { */ const handleDismiss = () => { setGeneratedAlt( null ); + setIsDecorative( false ); }; return ( @@ -127,7 +153,7 @@ export function AltTextControls( { style={ { padding: '0 16px' } } > { /* Generated alt text preview */ } - { hasGeneratedAlt && ( + { hasGeneratedAlt && ! isDecorative && (
) } + { /* Decorative image notice */ } + { isDecorative && ( +
+ + { __( + 'This image appears to be decorative. Applying will set an empty alt attribute, which tells screen readers to skip it.', + 'ai' + ) } + +
+ + +
+
+ ) } + { /* Generate button */ } - { ! hasGeneratedAlt && ( + { ! hasGeneratedAlt && ! isDecorative && (