diff --git a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/ExploreQuickFilters.spec.ts b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/ExploreQuickFilters.spec.ts index 6d14772ae73c..a1a802d19409 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/ExploreQuickFilters.spec.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/ExploreQuickFilters.spec.ts @@ -426,4 +426,70 @@ test.describe('Metric search result highlight', () => { expect(fullText?.trim()).toBe(metric.entity.name); }); }); + + test.afterAll('Cleanup metric entity', async ({ browser }) => { + const { apiContext, afterAction } = await createNewPage(browser); + await metric.delete(apiContext); + await afterAction(); + }); }); + +test.describe( + 'Quick filter dropdown infinite scroll', + { tag: ['@Features', '@Discovery'] }, + () => { + test('should progressively load more options on scroll to bottom', async ({ + page, + }) => { + test.slow(); + + await test.step('Open the Tag quick filter and capture initial aggregate request with size=50', async () => { + const initialAggregateResponse = page.waitForResponse( + (response) => + response.url().includes('/api/v1/search/aggregate') && + response.url().includes('field=tags.tagFQN') && + response.url().includes('size=50') && + response.status() === 200 + ); + + await page.getByTestId('search-dropdown-Tag').click(); + await initialAggregateResponse; + await waitForAllLoadersToDisappear(page); + }); + + await test.step('Scroll to the bottom of the dropdown and verify a second request with size=100', async () => { + const scrollContainer = page.getByTestId( + 'search-dropdown-scroll-container' + ); + await expect(scrollContainer).toBeVisible(); + + const nextAggregateResponse = page.waitForResponse( + (response) => + response.url().includes('/api/v1/search/aggregate') && + response.url().includes('field=tags.tagFQN') && + response.url().includes('size=100') && + response.status() === 200 + ); + + await scrollContainer.evaluate((el) => { + el.scrollTop = el.scrollHeight; + }); + + await nextAggregateResponse; + await waitForAllLoadersToDisappear(page); + }); + + await clickOutside(page); + }); + + test.afterAll('Cleanup', async ({ browser }) => { + const { apiContext, afterAction } = await createNewPage(browser); + await table.delete(apiContext); + await domain.delete(apiContext); + await tier.delete(apiContext); + await tierWithoutAsset.delete(apiContext); + await afterAction(); + }); + } +); + diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Explore/ExploreQuickFilters.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Explore/ExploreQuickFilters.tsx index ac1dba819ee4..d563aafb933a 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Explore/ExploreQuickFilters.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Explore/ExploreQuickFilters.tsx @@ -15,7 +15,8 @@ import { Space } from 'antd'; import { AxiosError } from 'axios'; import { isEqual, uniqWith } from 'lodash'; import Qs from 'qs'; -import { FC, useCallback, useMemo, useState } from 'react'; +import { FC, useCallback, useMemo, useRef, useState } from 'react'; +import { EXPLORE_QUICK_FILTER_PAGE_SIZE } from '../../constants/explore.constants'; import { EntityFields } from '../../enums/AdvancedSearch.enum'; import { SearchIndex } from '../../enums/search.enum'; import useCustomLocation from '../../hooks/useCustomLocation/useCustomLocation'; @@ -50,8 +51,20 @@ const ExploreQuickFilters: FC = ({ const location = useCustomLocation(); const [options, setOptions] = useState(); const [isOptionsLoading, setIsOptionsLoading] = useState(false); + const [isLoadingMore, setIsLoadingMore] = useState(false); + const [hasMore, setHasMore] = useState(false); const { queryFilter } = useAdvanceSearch(); const { isNLPEnabled } = useSearchStore(); + + const currentSizeRef = useRef(EXPLORE_QUICK_FILTER_PAGE_SIZE); + const isLoadingMoreRef = useRef(false); + const activeFieldRef = useRef<{ + key: string; + searchIndex?: SearchIndex; + searchKey?: string; + } | null>(null); + const searchTextRef = useRef(''); + const getStaticOptions = useCallback( (key: string) => fields.find((item) => item.key === key)?.options, [fields] @@ -88,43 +101,45 @@ const ExploreQuickFilters: FC = ({ defaultQueryFilter as unknown as QueryFilterInterface ); - const fetchDefaultOptions = async ( - index: SearchIndex | SearchIndex[], - key: string, - fieldSearchIndex?: SearchIndex, - fieldSearchKey?: string - ) => { - const staticOptions = getStaticOptions(key); - if (staticOptions) { - setOptions(staticOptions); - - return; - } + const pageSize = optionPageSize ?? EXPLORE_QUICK_FILTER_PAGE_SIZE; - // Use field-specific searchIndex if provided, otherwise use the default index - const searchIndexToUse = fieldSearchIndex ?? index; - // Use field-specific searchKey if provided, otherwise use the key - const searchKeyToUse = fieldSearchKey ?? key; + const fetchAggregationBuckets = useCallback( + async ( + key: string, + value: string, + size: number, + fieldSearchIndex?: SearchIndex, + fieldSearchKey?: string + ) => { + const searchIndexToUse = fieldSearchIndex ?? index; + const searchKeyToUse = fieldSearchKey ?? key; - let buckets = aggregations?.[key]?.buckets; - if (!buckets) { const res = await getAggregationOptions( searchIndexToUse, searchKeyToUse, - '', + value, JSON.stringify(combinedQueryFilter), independent, showDeleted, - optionPageSize, + size, isNLPEnabled, searchText ); - buckets = res.data.aggregations[`sterms#${searchKeyToUse}`].buckets; - } + const buckets = + res.data.aggregations[`sterms#${searchKeyToUse}`].buckets; + const newOptions = uniqWith( + getOptionsFromAggregationBucket(buckets), + isEqual + ); - setOptions(uniqWith(getOptionsFromAggregationBucket(buckets), isEqual)); - }; + setOptions(newOptions); + setHasMore(buckets.length >= size); + + return newOptions; + }, + [index, combinedQueryFilter, independent, showDeleted, isNLPEnabled, searchText] + ); const getInitialOptions = async ( key: string, @@ -134,14 +149,42 @@ const ExploreQuickFilters: FC = ({ const staticOptions = getStaticOptions(key); if (staticOptions) { setOptions(staticOptions); + setHasMore(false); return; } + currentSizeRef.current = pageSize; + searchTextRef.current = ''; + isLoadingMoreRef.current = false; + activeFieldRef.current = { + key, + searchIndex: fieldSearchIndex, + searchKey: fieldSearchKey, + }; + + setIsLoadingMore(false); setIsOptionsLoading(true); setOptions([]); + setHasMore(false); + + const buckets = aggregations?.[key]?.buckets; + if (buckets) { + setOptions(uniqWith(getOptionsFromAggregationBucket(buckets), isEqual)); + setHasMore(buckets.length >= pageSize); + setIsOptionsLoading(false); + + return; + } + try { - await fetchDefaultOptions(index, key, fieldSearchIndex, fieldSearchKey); + await fetchAggregationBuckets( + key, + '', + pageSize, + fieldSearchIndex, + fieldSearchKey + ); } catch (error) { showErrorToast(error as AxiosError); } finally { @@ -163,12 +206,22 @@ const ExploreQuickFilters: FC = ({ ) : staticOptions; setOptions(filteredOptions); + setHasMore(false); return; } + currentSizeRef.current = pageSize; + searchTextRef.current = value; + activeFieldRef.current = { + key, + searchIndex: fieldSearchIndex, + searchKey: fieldSearchKey, + }; + setIsOptionsLoading(true); setOptions([]); + setHasMore(false); try { if (!value) { getInitialOptions(key, fieldSearchIndex, fieldSearchKey); @@ -176,23 +229,13 @@ const ExploreQuickFilters: FC = ({ return; } - const searchIndexToUse = fieldSearchIndex ?? index; - const searchKeyToUse = fieldSearchKey ?? key; - - const res = await getAggregationOptions( - searchIndexToUse, - searchKeyToUse, + await fetchAggregationBuckets( + key, value, - JSON.stringify(combinedQueryFilter), - independent, - showDeleted, - undefined, - isNLPEnabled, - searchText + pageSize, + fieldSearchIndex, + fieldSearchKey ); - - const buckets = res.data.aggregations[`sterms#${searchKeyToUse}`].buckets; - setOptions(uniqWith(getOptionsFromAggregationBucket(buckets), isEqual)); } catch (error) { showErrorToast(error as AxiosError); } finally { @@ -200,6 +243,33 @@ const ExploreQuickFilters: FC = ({ } }; + const handleScrollEnd = useCallback(async () => { + if (isLoadingMoreRef.current || !hasMore || !activeFieldRef.current) { + return; + } + + isLoadingMoreRef.current = true; + setIsLoadingMore(true); + + const nextSize = currentSizeRef.current + pageSize; + currentSizeRef.current = nextSize; + + try { + await fetchAggregationBuckets( + activeFieldRef.current.key, + searchTextRef.current, + nextSize, + activeFieldRef.current.searchIndex, + activeFieldRef.current.searchKey + ); + } catch (error) { + showErrorToast(error as AxiosError); + } finally { + isLoadingMoreRef.current = false; + setIsLoadingMore(false); + } + }, [hasMore, pageSize, fetchAggregationBuckets]); + return ( {fields.map((field) => { @@ -211,12 +281,14 @@ const ExploreQuickFilters: FC = ({ return ( = ({ onGetInitialOptions={(key) => getInitialOptions(key, field.searchIndex, field.searchKey) } + onScrollEnd={handleScrollEnd} onSearch={(value, key) => getFilterOptions(value, key, field.searchIndex, field.searchKey) } @@ -244,3 +317,4 @@ const ExploreQuickFilters: FC = ({ }; export default ExploreQuickFilters; + diff --git a/openmetadata-ui/src/main/resources/ui/src/components/SearchDropdown/SearchDropdown.interface.ts b/openmetadata-ui/src/main/resources/ui/src/components/SearchDropdown/SearchDropdown.interface.ts index fe47c4870c59..1146387fc71e 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/SearchDropdown/SearchDropdown.interface.ts +++ b/openmetadata-ui/src/main/resources/ui/src/components/SearchDropdown/SearchDropdown.interface.ts @@ -35,6 +35,9 @@ export interface SearchDropdownProps { hideSearchBar?: boolean; // Determines if the search bar should be hidden. Default is false singleSelect?: boolean; // Enable single-select mode with radio buttons instead of checkboxes getPopupContainer?: (triggerNode: HTMLElement) => HTMLElement; + isPaginated?: boolean; // Enable infinite scroll with progressive loading + onScrollEnd?: () => void; // Callback fired when the user scrolls to the bottom of the options list + isLoadingMore?: boolean; // When true, a spinner is rendered at the bottom of the options list } export interface SearchDropdownOption { diff --git a/openmetadata-ui/src/main/resources/ui/src/components/SearchDropdown/SearchDropdown.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/SearchDropdown/SearchDropdown.test.tsx index 9518260196e9..a4e760704918 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/SearchDropdown/SearchDropdown.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/SearchDropdown/SearchDropdown.test.tsx @@ -562,4 +562,147 @@ describe('Search DropDown Component', () => { ); }); }); + + describe('Paginated / Infinite Scroll', () => { + const mockOnScrollEnd = jest.fn(); + + const paginatedProps: SearchDropdownProps = { + ...mockProps, + isPaginated: true, + onScrollEnd: mockOnScrollEnd, + isLoadingMore: false, + }; + + beforeEach(() => { + mockOnScrollEnd.mockClear(); + }); + + it('should render scroll container when isPaginated is true', async () => { + render(); + + const container = await screen.findByTestId('search-dropdown-Owner'); + await act(async () => { + fireEvent.click(container); + }); + + expect( + await screen.findByTestId('search-dropdown-scroll-container') + ).toBeInTheDocument(); + }); + + it('should NOT render scroll container when isPaginated is false', async () => { + render(); + + const container = await screen.findByTestId('search-dropdown-Owner'); + await act(async () => { + fireEvent.click(container); + }); + + expect( + screen.queryByTestId('search-dropdown-scroll-container') + ).not.toBeInTheDocument(); + }); + + it('should call onScrollEnd when scrolled to bottom within threshold', async () => { + render(); + + const container = await screen.findByTestId('search-dropdown-Owner'); + await act(async () => { + fireEvent.click(container); + }); + + const scrollContainer = await screen.findByTestId( + 'search-dropdown-scroll-container' + ); + + Object.defineProperty(scrollContainer, 'scrollHeight', { + value: 500, + writable: true, + }); + Object.defineProperty(scrollContainer, 'clientHeight', { + value: 200, + writable: true, + }); + + await act(async () => { + fireEvent.scroll(scrollContainer, { + target: { scrollTop: 295 }, + }); + }); + + expect(mockOnScrollEnd).toHaveBeenCalledTimes(1); + }); + + it('should NOT call onScrollEnd when not scrolled to bottom', async () => { + render(); + + const container = await screen.findByTestId('search-dropdown-Owner'); + await act(async () => { + fireEvent.click(container); + }); + + const scrollContainer = await screen.findByTestId( + 'search-dropdown-scroll-container' + ); + + Object.defineProperty(scrollContainer, 'scrollHeight', { + value: 500, + writable: true, + }); + Object.defineProperty(scrollContainer, 'clientHeight', { + value: 200, + writable: true, + }); + + await act(async () => { + fireEvent.scroll(scrollContainer, { + target: { scrollTop: 100 }, + }); + }); + + expect(mockOnScrollEnd).not.toHaveBeenCalled(); + }); + + it('should NOT call onScrollEnd when isLoadingMore is true', async () => { + render(); + + const container = await screen.findByTestId('search-dropdown-Owner'); + await act(async () => { + fireEvent.click(container); + }); + + const scrollContainer = await screen.findByTestId( + 'search-dropdown-scroll-container' + ); + + Object.defineProperty(scrollContainer, 'scrollHeight', { + value: 500, + writable: true, + }); + Object.defineProperty(scrollContainer, 'clientHeight', { + value: 200, + writable: true, + }); + + await act(async () => { + fireEvent.scroll(scrollContainer, { + target: { scrollTop: 295 }, + }); + }); + + expect(mockOnScrollEnd).not.toHaveBeenCalled(); + }); + + it('should render Loader at bottom when isLoadingMore is true', async () => { + render(); + + const container = await screen.findByTestId('search-dropdown-Owner'); + await act(async () => { + fireEvent.click(container); + }); + + expect(await screen.findByTestId('loader')).toBeInTheDocument(); + }); + }); }); + diff --git a/openmetadata-ui/src/main/resources/ui/src/components/SearchDropdown/SearchDropdown.tsx b/openmetadata-ui/src/main/resources/ui/src/components/SearchDropdown/SearchDropdown.tsx index 4be199eaecf1..dfce4de56a51 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/SearchDropdown/SearchDropdown.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/SearchDropdown/SearchDropdown.tsx @@ -29,7 +29,7 @@ import { } from 'antd'; import classNames from 'classnames'; import { debounce, isEmpty, isUndefined } from 'lodash'; -import { +import React, { FC, ReactNode, useCallback, @@ -75,6 +75,9 @@ const SearchDropdown: FC = ({ hideSearchBar = false, singleSelect = false, getPopupContainer, + isPaginated = false, + onScrollEnd, + isLoadingMore = false, }) => { const tabsInfo = searchClassBase.getTabsInfo(); const { t } = useTranslation(); @@ -238,6 +241,19 @@ const SearchDropdown: FC = ({ ); }, [isDropDownOpen, selectedKeys]); + const handleMenuScroll = useCallback( + (e: React.UIEvent) => { + if (!isPaginated || !onScrollEnd || isLoadingMore) { + return; + } + const { scrollHeight, scrollTop, clientHeight } = e.currentTarget; + if (scrollHeight - (scrollTop + clientHeight) <= 10) { + onScrollEnd(); + } + }, + [isPaginated, onScrollEnd, isLoadingMore] + ); + const getDropdownBody = useCallback( (menuNode: ReactNode) => { const entityLabel = index && tabsInfo[index]?.label; @@ -252,9 +268,30 @@ const SearchDropdown: FC = ({ ); } - return options.length > 0 || selectedOptions.length > 0 ? ( - menuNode - ) : ( + if (options.length > 0 || selectedOptions.length > 0) { + return isPaginated ? ( +
+ {menuNode} + {isLoadingMore && ( + + + + + + )} +
+ ) : ( + menuNode + ); + } + + return ( @@ -268,7 +305,16 @@ const SearchDropdown: FC = ({ ); }, - [isSuggestionsLoading, options, selectedOptions, index, searchKey] + [ + isSuggestionsLoading, + options, + selectedOptions, + index, + searchKey, + isPaginated, + handleMenuScroll, + isLoadingMore, + ] ); const dropdownCardComponent = useCallback( diff --git a/openmetadata-ui/src/main/resources/ui/src/components/SearchDropdown/search-dropdown.less b/openmetadata-ui/src/main/resources/ui/src/components/SearchDropdown/search-dropdown.less index 930e35785223..598705fd6e23 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/SearchDropdown/search-dropdown.less +++ b/openmetadata-ui/src/main/resources/ui/src/components/SearchDropdown/search-dropdown.less @@ -32,6 +32,18 @@ } } + .search-dropdown-scroll-container { + max-height: 200px; + overflow-y: auto; + margin-bottom: 8px; + + .ant-dropdown-menu { + max-height: none; + overflow-y: visible; + margin-bottom: 0; + } + } + .dropdown-option-label { max-width: 65vw; } diff --git a/openmetadata-ui/src/main/resources/ui/src/constants/explore.constants.ts b/openmetadata-ui/src/main/resources/ui/src/constants/explore.constants.ts index 53b58098bcd9..d14867057b45 100644 --- a/openmetadata-ui/src/main/resources/ui/src/constants/explore.constants.ts +++ b/openmetadata-ui/src/main/resources/ui/src/constants/explore.constants.ts @@ -24,6 +24,8 @@ export const TAG_FQN_KEY = 'tags.tagFQN'; export const MAX_RESULT_HITS = 10000; +export const EXPLORE_QUICK_FILTER_PAGE_SIZE = 50; + export const SUPPORTED_EMPTY_FILTER_FIELDS = [ EntityFields.OWNERS, EntityFields.DOMAINS, diff --git a/openmetadata-ui/src/main/resources/ui/src/rest/miscAPI.ts b/openmetadata-ui/src/main/resources/ui/src/rest/miscAPI.ts index 97d29aef00f1..55aed38356c7 100644 --- a/openmetadata-ui/src/main/resources/ui/src/rest/miscAPI.ts +++ b/openmetadata-ui/src/main/resources/ui/src/rest/miscAPI.ts @@ -221,7 +221,8 @@ export const getAggregateFieldOptions = ( sourceFields?: string, deleted = false, isNLPEnabled = false, - queryText?: string + queryText?: string, + size?: number ) => { const withWildCardValue = value ? `.*${escapeESReservedCharacters(value)}.*` @@ -234,6 +235,7 @@ export const getAggregateFieldOptions = ( sourceFields, deleted, ...(queryText ? { queryText } : {}), + ...(size !== undefined ? { size } : {}), }; return APIClient.get>( diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/ExploreUtils.tsx b/openmetadata-ui/src/main/resources/ui/src/utils/ExploreUtils.tsx index eaab856bac38..2e62b6ba34c3 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/ExploreUtils.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/utils/ExploreUtils.tsx @@ -382,7 +382,8 @@ export const getAggregationOptions = async ( undefined, deleted, isNLPEnabled, - queryText + queryText, + size ); };