Fixes #26305: Implement progressive infinite scroll for Explore dropdowns

openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/ExploreQuickFilters.spec.ts

@@ -426,4 +426,70 @@ test.describe('Metric search result highlight', () => {
       expect(fullText?.trim()).toBe(metric.entity.name);
     });
   });
+
+  test.afterAll('Cleanup metric entity', async ({ browser }) => {
+    const { apiContext, afterAction } = await createNewPage(browser);
+    await metric.delete(apiContext);
+    await afterAction();
+  });
 });
+
+test.describe(
+  'Quick filter dropdown infinite scroll',
+  { tag: ['@Features', '@Discovery'] },
+  () => {
+    test('should progressively load more options on scroll to bottom', async ({
+      page,
+    }) => {
+      test.slow();
+
+      await test.step('Open the Tag quick filter and capture initial aggregate request with size=50', async () => {
+        const initialAggregateResponse = page.waitForResponse(
+          (response) =>
+            response.url().includes('/api/v1/search/aggregate') &&
+            response.url().includes('field=tags.tagFQN') &&
+            response.url().includes('size=50') &&
+            response.status() === 200
+        );
+
+        await page.getByTestId('search-dropdown-Tag').click();
+        await initialAggregateResponse;
+        await waitForAllLoadersToDisappear(page);
+      });
+
+      await test.step('Scroll to the bottom of the dropdown and verify a second request with size=100', async () => {
+        const scrollContainer = page.getByTestId(
+          'search-dropdown-scroll-container'
+        );
+        await expect(scrollContainer).toBeVisible();
+
+        const nextAggregateResponse = page.waitForResponse(
+          (response) =>
+            response.url().includes('/api/v1/search/aggregate') &&
+            response.url().includes('field=tags.tagFQN') &&
+            response.url().includes('size=100') &&
+            response.status() === 200
+        );
+
+        await scrollContainer.evaluate((el) => {
+          el.scrollTop = el.scrollHeight;
+        });
+
+        await nextAggregateResponse;
+        await waitForAllLoadersToDisappear(page);
+      });
+
+      await clickOutside(page);
+    });
+
+    test.afterAll('Cleanup', async ({ browser }) => {
+      const { apiContext, afterAction } = await createNewPage(browser);
+      await table.delete(apiContext);
+      await domain.delete(apiContext);
+      await tier.delete(apiContext);
+      await tierWithoutAsset.delete(apiContext);
+      await afterAction();
+    });
+  }
+);
+

openmetadata-ui/src/main/resources/ui/src/components/Explore/ExploreQuickFilters.tsx

@@ -15,7 +15,8 @@ import { Space } from 'antd';
 import { AxiosError } from 'axios';
 import { isEqual, uniqWith } from 'lodash';
 import Qs from 'qs';
-import { FC, useCallback, useMemo, useState } from 'react';
+import { FC, useCallback, useMemo, useRef, useState } from 'react';
+import { EXPLORE_QUICK_FILTER_PAGE_SIZE } from '../../constants/explore.constants';
 import { EntityFields } from '../../enums/AdvancedSearch.enum';
 import { SearchIndex } from '../../enums/search.enum';
 import useCustomLocation from '../../hooks/useCustomLocation/useCustomLocation';
@@ -50,8 +51,20 @@ const ExploreQuickFilters: FC<ExploreQuickFiltersProps> = ({
   const location = useCustomLocation();
   const [options, setOptions] = useState<SearchDropdownOption[]>();
   const [isOptionsLoading, setIsOptionsLoading] = useState<boolean>(false);
+  const [isLoadingMore, setIsLoadingMore] = useState<boolean>(false);
+  const [hasMore, setHasMore] = useState<boolean>(false);
   const { queryFilter } = useAdvanceSearch();
   const { isNLPEnabled } = useSearchStore();
+
+  const currentSizeRef = useRef<number>(EXPLORE_QUICK_FILTER_PAGE_SIZE);
+  const isLoadingMoreRef = useRef<boolean>(false);
+  const activeFieldRef = useRef<{
+    key: string;
+    searchIndex?: SearchIndex;
+    searchKey?: string;
+  } | null>(null);
+  const searchTextRef = useRef<string>('');
+
   const getStaticOptions = useCallback(
     (key: string) => fields.find((item) => item.key === key)?.options,
     [fields]
@@ -88,43 +101,45 @@ const ExploreQuickFilters: FC<ExploreQuickFiltersProps> = ({
     defaultQueryFilter as unknown as QueryFilterInterface
   );
 
-  const fetchDefaultOptions = async (
-    index: SearchIndex | SearchIndex[],
-    key: string,
-    fieldSearchIndex?: SearchIndex,
-    fieldSearchKey?: string
-  ) => {
-    const staticOptions = getStaticOptions(key);
-    if (staticOptions) {
-      setOptions(staticOptions);
-
-      return;
-    }
+  const pageSize = optionPageSize ?? EXPLORE_QUICK_FILTER_PAGE_SIZE;
 
-    // Use field-specific searchIndex if provided, otherwise use the default index
-    const searchIndexToUse = fieldSearchIndex ?? index;
-    // Use field-specific searchKey if provided, otherwise use the key
-    const searchKeyToUse = fieldSearchKey ?? key;
+  const fetchAggregationBuckets = useCallback(
+    async (
+      key: string,
+      value: string,
+      size: number,
+      fieldSearchIndex?: SearchIndex,
+      fieldSearchKey?: string
+    ) => {
+      const searchIndexToUse = fieldSearchIndex ?? index;
+      const searchKeyToUse = fieldSearchKey ?? key;
 
-    let buckets = aggregations?.[key]?.buckets;
-    if (!buckets) {
       const res = await getAggregationOptions(
         searchIndexToUse,
         searchKeyToUse,
-        '',
+        value,
         JSON.stringify(combinedQueryFilter),
         independent,
         showDeleted,
-        optionPageSize,
+        size,
         isNLPEnabled,
         searchText
       );
 
-      buckets = res.data.aggregations[`sterms#${searchKeyToUse}`].buckets;
-    }
+      const buckets =
+        res.data.aggregations[`sterms#${searchKeyToUse}`].buckets;
+      const newOptions = uniqWith(
+        getOptionsFromAggregationBucket(buckets),
+        isEqual
+      );
 
-    setOptions(uniqWith(getOptionsFromAggregationBucket(buckets), isEqual));
-  };
+      setOptions(newOptions);
+      setHasMore(buckets.length >= size);
+
+      return newOptions;
+    },
+    [index, combinedQueryFilter, independent, showDeleted, isNLPEnabled, searchText]
+  );
 
   const getInitialOptions = async (
     key: string,
@@ -134,14 +149,42 @@ const ExploreQuickFilters: FC<ExploreQuickFiltersProps> = ({
     const staticOptions = getStaticOptions(key);
     if (staticOptions) {
       setOptions(staticOptions);
+      setHasMore(false);
 
       return;
     }
 
+    currentSizeRef.current = pageSize;
+    searchTextRef.current = '';
+    isLoadingMoreRef.current = false;
+    activeFieldRef.current = {
+      key,
+      searchIndex: fieldSearchIndex,
+      searchKey: fieldSearchKey,
+    };
+
+    setIsLoadingMore(false);
     setIsOptionsLoading(true);
     setOptions([]);
+    setHasMore(false);
+
+    const buckets = aggregations?.[key]?.buckets;
+    if (buckets) {
+      setOptions(uniqWith(getOptionsFromAggregationBucket(buckets), isEqual));
+      setHasMore(buckets.length >= pageSize);
+      setIsOptionsLoading(false);
+
+      return;
+    }
+
     try {
-      await fetchDefaultOptions(index, key, fieldSearchIndex, fieldSearchKey);
+      await fetchAggregationBuckets(
+        key,
+        '',
+        pageSize,
+        fieldSearchIndex,
+        fieldSearchKey
+      );
     } catch (error) {
       showErrorToast(error as AxiosError);
     } finally {
@@ -163,43 +206,70 @@ const ExploreQuickFilters: FC<ExploreQuickFiltersProps> = ({
           )
         : staticOptions;
       setOptions(filteredOptions);
+      setHasMore(false);
 
       return;
     }
 
+    currentSizeRef.current = pageSize;
+    searchTextRef.current = value;
+    activeFieldRef.current = {
+      key,
+      searchIndex: fieldSearchIndex,
+      searchKey: fieldSearchKey,
+    };
+
     setIsOptionsLoading(true);
     setOptions([]);
+    setHasMore(false);
     try {
       if (!value) {
         getInitialOptions(key, fieldSearchIndex, fieldSearchKey);
 
         return;
       }
 
-      const searchIndexToUse = fieldSearchIndex ?? index;
-      const searchKeyToUse = fieldSearchKey ?? key;
-
-      const res = await getAggregationOptions(
-        searchIndexToUse,
-        searchKeyToUse,
+      await fetchAggregationBuckets(
+        key,
         value,
-        JSON.stringify(combinedQueryFilter),
-        independent,
-        showDeleted,
-        undefined,
-        isNLPEnabled,
-        searchText
+        pageSize,
+        fieldSearchIndex,
+        fieldSearchKey
       );
-
-      const buckets = res.data.aggregations[`sterms#${searchKeyToUse}`].buckets;
-      setOptions(uniqWith(getOptionsFromAggregationBucket(buckets), isEqual));
     } catch (error) {
       showErrorToast(error as AxiosError);
     } finally {
       setIsOptionsLoading(false);
     }
   };
 
+  const handleScrollEnd = useCallback(async () => {
+    if (isLoadingMoreRef.current || !hasMore || !activeFieldRef.current) {
+      return;
+    }
+
+    isLoadingMoreRef.current = true;
+    setIsLoadingMore(true);
+
+    const nextSize = currentSizeRef.current + pageSize;
+    currentSizeRef.current = nextSize;
+
+    try {
+      await fetchAggregationBuckets(
+        activeFieldRef.current.key,
+        searchTextRef.current,
+        nextSize,
+        activeFieldRef.current.searchIndex,
+        activeFieldRef.current.searchKey
+      );
+    } catch (error) {
+      showErrorToast(error as AxiosError);
+    } finally {
+      isLoadingMoreRef.current = false;
+      setIsLoadingMore(false);
+    }
+  }, [hasMore, pageSize, fetchAggregationBuckets]);
+
   return (
     <Space wrap className="explore-quick-filters-container" size={[8, 0]}>
       {fields.map((field) => {
@@ -211,12 +281,14 @@ const ExploreQuickFilters: FC<ExploreQuickFiltersProps> = ({
         return (
           <SearchDropdown
             highlight
+            isPaginated
             dropdownClassName={field.dropdownClassName}
             hasNullOption={hasNullOption}
             hideCounts={field.hideCounts ?? false}
             hideSearchBar={field.hideSearchBar ?? false}
             independent={independent}
             index={displayIndex as ExploreSearchIndex}
+            isLoadingMore={isLoadingMore}
             isSuggestionsLoading={isOptionsLoading}
             key={field.key}
             label={translateWithNestedKeys(field.label, field.labelKeyOptions)}
@@ -232,6 +304,7 @@ const ExploreQuickFilters: FC<ExploreQuickFiltersProps> = ({
             onGetInitialOptions={(key) =>
               getInitialOptions(key, field.searchIndex, field.searchKey)
             }
+            onScrollEnd={handleScrollEnd}
             onSearch={(value, key) =>
               getFilterOptions(value, key, field.searchIndex, field.searchKey)
             }
@@ -244,3 +317,4 @@ const ExploreQuickFilters: FC<ExploreQuickFiltersProps> = ({
 };
 
 export default ExploreQuickFilters;
+

openmetadata-ui/src/main/resources/ui/src/components/SearchDropdown/SearchDropdown.interface.ts

@@ -35,6 +35,9 @@ export interface SearchDropdownProps {
   hideSearchBar?: boolean; // Determines if the search bar should be hidden. Default is false
   singleSelect?: boolean; // Enable single-select mode with radio buttons instead of checkboxes
   getPopupContainer?: (triggerNode: HTMLElement) => HTMLElement;
+  isPaginated?: boolean; // Enable infinite scroll with progressive loading
+  onScrollEnd?: () => void; // Callback fired when the user scrolls to the bottom of the options list
+  isLoadingMore?: boolean; // When true, a spinner is rendered at the bottom of the options list
 }
 
 export interface SearchDropdownOption {