From 4f46f7cb684ac0e90c1b57072c0eb8790faa5e80 Mon Sep 17 00:00:00 2001 From: Jatin Kathuria Date: Tue, 7 Jan 2025 07:20:30 +0100 Subject: [PATCH] [Security Solution] Fix timeline dynamic batching (#204034) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Handles : ### Issue with Batches - https://github.com/elastic/kibana/issues/201405 - Timeline had a bug where if users fetched multiple batches and then if user adds a new column, the value of this new columns will only be fetched for the latest batch and not old batches. - This PR fixes that ✅ by cumulatively fetching the data for old batches till current batch `iff a new column has been added`. - For example, if user has already fetched the 3rd batch, data for 1st,2nd and 3rd will be fetched together when a column has been added, otherwise, data will be fetched incrementally. ### Issue with Elastic search limit - Elastic search has a limit of 10K hits at max but we throw error at 10K which should be allowed. - Error should be thrown at anything `>10K`. 10001 for example. - ✅ This PR fixes that just for timeline by allowing 10K hits. ### Removal of obsolete code Below files related to old Timeline code are removed as well: - x-pack/plugins/security_solution/public/timelines/components/timeline/footer/index.test.tsx - x-pack/plugins/security_solution/public/timelines/components/timeline/footer/index.tsx --------- Co-authored-by: Philippe Oberti (cherry picked from commit 088169f446788f9fa8800d77817881524514943e) # Conflicts: # packages/kbn-babel-preset/styled_components_files.js # x-pack/plugins/security_solution/public/common/mock/mock_timeline_search_service.ts # x-pack/plugins/security_solution/public/timelines/components/timeline/footer/index.test.tsx # x-pack/plugins/security_solution/public/timelines/components/timeline/footer/index.tsx # x-pack/plugins/security_solution/public/timelines/components/timeline/tabs/query/index.tsx # x-pack/solutions/security/plugins/security_solution/public/timelines/containers/index.test.tsx --- .../common/types/timeline/store.ts | 4 +- .../mock/mock_timeline_search_service.ts | 51 ++ .../components/timeline/footer/index.test.tsx | 236 ------ .../components/timeline/footer/index.tsx | 380 --------- .../components/timeline/tabs/eql/index.tsx | 4 +- .../components/timeline/tabs/pinned/index.tsx | 4 +- .../timeline/tabs/query/index.test.tsx | 299 ++++--- .../components/timeline/tabs/query/index.tsx | 36 +- .../data_table/index.test.tsx | 18 +- .../unified_components/data_table/index.tsx | 9 +- .../timelines/containers/index.test.tsx | 773 ++++++++++++------ .../public/timelines/containers/index.tsx | 160 ++-- .../search_strategy/timeline/eql/helpers.ts | 2 +- .../timeline/factory/events/all/index.ts | 2 +- .../translations/translations/fr-FR.json | 1 - .../translations/translations/ja-JP.json | 1 - .../translations/translations/zh-CN.json | 1 - 17 files changed, 858 insertions(+), 1123 deletions(-) create mode 100644 x-pack/plugins/security_solution/public/common/mock/mock_timeline_search_service.ts delete mode 100644 x-pack/plugins/security_solution/public/timelines/components/timeline/footer/index.test.tsx delete mode 100644 x-pack/plugins/security_solution/public/timelines/components/timeline/footer/index.tsx diff --git a/x-pack/plugins/security_solution/common/types/timeline/store.ts b/x-pack/plugins/security_solution/common/types/timeline/store.ts index 834949d2ed591..d2575b86344d0 100644 --- a/x-pack/plugins/security_solution/common/types/timeline/store.ts +++ b/x-pack/plugins/security_solution/common/types/timeline/store.ts @@ -72,8 +72,8 @@ export type OnColumnRemoved = (columnId: ColumnId) => void; export type OnColumnResized = ({ columnId, delta }: { columnId: ColumnId; delta: number }) => void; -/** Invoked when a user clicks to load more item */ -export type OnFetchMoreRecords = (nextPage: number) => void; +/** Invoked when a user clicks to load next batch */ +export type OnFetchMoreRecords = VoidFunction; /** Invoked when a user checks/un-checks a row */ export type OnRowSelected = ({ diff --git a/x-pack/plugins/security_solution/public/common/mock/mock_timeline_search_service.ts b/x-pack/plugins/security_solution/public/common/mock/mock_timeline_search_service.ts new file mode 100644 index 0000000000000..c38f49d3eb635 --- /dev/null +++ b/x-pack/plugins/security_solution/public/common/mock/mock_timeline_search_service.ts @@ -0,0 +1,51 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { mockTimelineData } from './mock_timeline_data'; + +const mockEvents = structuredClone(mockTimelineData); + +/* + * This helps to mock `data.search.search` method to mock the timeline data + * */ +export const getMockTimelineSearchSubscription = () => { + const mockSearchWithArgs = jest.fn(); + + const mockTimelineSearchSubscription = jest.fn().mockImplementation((args) => { + mockSearchWithArgs(args); + return { + subscribe: jest.fn().mockImplementation(({ next }) => { + const start = args.pagination.activePage * args.pagination.querySize; + const end = start + args.pagination.querySize; + const timelineOut = setTimeout(() => { + next({ + isRunning: false, + isPartial: false, + inspect: { + dsl: [], + response: [], + }, + edges: mockEvents.map((item) => ({ node: item })).slice(start, end), + pageInfo: { + activePage: args.pagination.activePage, + querySize: args.pagination.querySize, + }, + rawResponse: {}, + totalCount: mockEvents.length, + }); + }, 50); + return { + unsubscribe: jest.fn(() => { + clearTimeout(timelineOut); + }), + }; + }), + }; + }); + + return { mockTimelineSearchSubscription, mockSearchWithArgs }; +}; diff --git a/x-pack/plugins/security_solution/public/timelines/components/timeline/footer/index.test.tsx b/x-pack/plugins/security_solution/public/timelines/components/timeline/footer/index.test.tsx deleted file mode 100644 index 77eae288dbaa0..0000000000000 --- a/x-pack/plugins/security_solution/public/timelines/components/timeline/footer/index.test.tsx +++ /dev/null @@ -1,236 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { render, screen, fireEvent } from '@testing-library/react'; -import React from 'react'; - -import { TestProviders } from '../../../../common/mock/test_providers'; - -import { FooterComponent, PagingControlComponent } from '.'; -import { TimelineId } from '../../../../../common/types/timeline'; - -jest.mock('../../../../common/lib/kibana'); - -describe('Footer Timeline Component', () => { - const loadMore = jest.fn(); - const updatedAt = 1546878704036; - const serverSideEventCount = 15546; - const itemsCount = 2; - - describe('rendering', () => { - it('shoult render the default timeline footer', () => { - render( - - - - ); - - expect(screen.getByTestId('timeline-footer')).toBeInTheDocument(); - }); - - it('should render the loading panel at the beginning ', () => { - render( - - - - ); - - expect(screen.getByTestId('LoadingPanelTimeline')).toBeInTheDocument(); - }); - - it('should render the loadMore button if it needs to fetch more', () => { - render( - - - - ); - - expect(screen.getByTestId('timeline-pagination')).toBeInTheDocument(); - }); - - it('should render `Loading...` when fetching new data', () => { - render( - - ); - - expect(screen.queryByTestId('LoadingPanelTimeline')).not.toBeInTheDocument(); - expect(screen.getByText('Loading...')).toBeInTheDocument(); - }); - - it('should render the Pagination in the more load button when fetching new data', () => { - render( - - ); - - expect(screen.getByTestId('timeline-pagination')).toBeInTheDocument(); - }); - - it('should NOT render the loadMore button because there is nothing else to fetch', () => { - render( - - - - ); - - expect(screen.queryByTestId('timeline-pagination')).not.toBeInTheDocument(); - }); - - it('should render the popover to select new itemsPerPage in timeline', () => { - render( - - - - ); - - fireEvent.click(screen.getByTestId('local-events-count-button')); - expect(screen.getByTestId('timelinePickSizeRow')).toBeInTheDocument(); - }); - }); - - describe('Events', () => { - it('should call loadmore when clicking on the button load more', () => { - render( - - - - ); - - fireEvent.click(screen.getByTestId('pagination-button-next')); - expect(loadMore).toBeCalled(); - }); - - it('should render the auto-refresh message instead of load more button when stream live is on', () => { - render( - - - - ); - - expect(screen.queryByTestId('timeline-pagination')).not.toBeInTheDocument(); - expect(screen.getByTestId('is-live-on-message')).toBeInTheDocument(); - }); - - it('should render the load more button when stream live is off', () => { - render( - - - - ); - - expect(screen.getByTestId('timeline-pagination')).toBeInTheDocument(); - expect(screen.queryByTestId('is-live-on-message')).not.toBeInTheDocument(); - }); - }); -}); diff --git a/x-pack/plugins/security_solution/public/timelines/components/timeline/footer/index.tsx b/x-pack/plugins/security_solution/public/timelines/components/timeline/footer/index.tsx deleted file mode 100644 index 611aa8953a7b8..0000000000000 --- a/x-pack/plugins/security_solution/public/timelines/components/timeline/footer/index.tsx +++ /dev/null @@ -1,380 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { - EuiBadge, - EuiButtonEmpty, - EuiContextMenuItem, - EuiContextMenuPanel, - EuiFlexGroup, - EuiFlexItem, - EuiIconTip, - EuiPopover, - EuiText, - EuiToolTip, - EuiPagination, -} from '@elastic/eui'; -import { FormattedMessage } from '@kbn/i18n-react'; -import React, { useCallback, useEffect, useState, useMemo } from 'react'; -import styled from 'styled-components'; -import { useDispatch } from 'react-redux'; - -import type { OnChangePage } from '../events'; -import { EVENTS_COUNT_BUTTON_CLASS_NAME } from '../helpers'; - -import * as i18n from './translations'; -import { timelineActions, timelineSelectors } from '../../../store'; -import { useDeepEqualSelector } from '../../../../common/hooks/use_selector'; -import { useKibana } from '../../../../common/lib/kibana'; -import { LastUpdatedContainer } from './last_updated'; - -interface HeightProp { - height: number; -} - -const FooterContainer = styled(EuiFlexGroup).attrs(({ height }) => ({ - style: { - height: `${height}px`, - }, -}))` - flex: 0 0 auto; -`; - -FooterContainer.displayName = 'FooterContainer'; - -const FooterFlexGroup = styled(EuiFlexGroup)` - height: 35px; - width: 100%; -`; - -FooterFlexGroup.displayName = 'FooterFlexGroup'; - -const LoadingPanelContainer = styled.div` - padding-top: 3px; -`; - -LoadingPanelContainer.displayName = 'LoadingPanelContainer'; - -export const ServerSideEventCount = styled.div` - margin: 0 5px 0 5px; -`; - -ServerSideEventCount.displayName = 'ServerSideEventCount'; - -/** The height of the footer, exported for use in height calculations */ -export const footerHeight = 40; // px - -/** Displays the server-side count of events */ -export const EventsCountComponent = ({ - closePopover, - documentType, - footerText, - isOpen, - items, - itemsCount, - onClick, - serverSideEventCount, -}: { - closePopover: () => void; - documentType: string; - isOpen: boolean; - items: React.ReactElement[]; - itemsCount: number; - onClick: () => void; - serverSideEventCount: number; - footerText: string | React.ReactNode; -}) => { - const totalCount = useMemo( - () => (serverSideEventCount > 0 ? serverSideEventCount : 0), - [serverSideEventCount] - ); - return ( -
- - - {itemsCount} - - - {` ${i18n.OF} `} - - } - isOpen={isOpen} - closePopover={closePopover} - panelPaddingSize="none" - > - - - - {totalCount} {footerText} - - } - > - - - {totalCount} - {' '} - {documentType} - - -
- ); -}; - -EventsCountComponent.displayName = 'EventsCountComponent'; - -export const EventsCount = React.memo(EventsCountComponent); - -EventsCount.displayName = 'EventsCount'; - -interface PagingControlProps { - activePage: number; - isLoading: boolean; - onPageClick: OnChangePage; - totalCount: number; - totalPages: number; -} - -const TimelinePaginationContainer = styled.div<{ hideLastPage: boolean }>` - ul.euiPagination__list { - li.euiPagination__item:last-child { - ${({ hideLastPage }) => `${hideLastPage ? 'display:none' : ''}`}; - } - } -`; - -export const PagingControlComponent: React.FC = ({ - activePage, - isLoading, - onPageClick, - totalCount, - totalPages, -}) => { - if (isLoading) { - return <>{`${i18n.LOADING}...`}; - } - - if (!totalPages) { - return null; - } - - return ( - 9999}> - - - ); -}; - -PagingControlComponent.displayName = 'PagingControlComponent'; - -export const PagingControl = React.memo(PagingControlComponent); - -PagingControl.displayName = 'PagingControl'; -interface FooterProps { - updatedAt: number; - activePage: number; - height: number; - id: string; - isLive: boolean; - isLoading: boolean; - itemsCount: number; - itemsPerPage: number; - itemsPerPageOptions: number[]; - onChangePage: OnChangePage; - totalCount: number; -} - -/** Renders a loading indicator and paging controls */ -export const FooterComponent = ({ - activePage, - updatedAt, - height, - id, - isLive, - isLoading, - itemsCount, - itemsPerPage, - itemsPerPageOptions, - onChangePage, - totalCount, -}: FooterProps) => { - const dispatch = useDispatch(); - const { timelines } = useKibana().services; - const [isPopoverOpen, setIsPopoverOpen] = useState(false); - const [paginationLoading, setPaginationLoading] = useState(false); - - const getManageTimeline = useMemo(() => timelineSelectors.getTimelineByIdSelector(), []); - const { - documentType = i18n.TOTAL_COUNT_OF_EVENTS, - loadingText = i18n.LOADING_EVENTS, - footerText = i18n.TOTAL_COUNT_OF_EVENTS, - } = useDeepEqualSelector((state) => getManageTimeline(state, id)); - - const handleChangePageClick = useCallback( - (nextPage: number) => { - setPaginationLoading(true); - onChangePage(nextPage); - }, - [onChangePage] - ); - - const onButtonClick = useCallback( - () => setIsPopoverOpen(!isPopoverOpen), - [isPopoverOpen, setIsPopoverOpen] - ); - - const closePopover = useCallback(() => setIsPopoverOpen(false), [setIsPopoverOpen]); - - const onChangeItemsPerPage = useCallback( - (itemsChangedPerPage: number) => - dispatch(timelineActions.updateItemsPerPage({ id, itemsPerPage: itemsChangedPerPage })), - [dispatch, id] - ); - - const rowItems = useMemo( - () => - itemsPerPageOptions && - itemsPerPageOptions.map((item) => ( - { - closePopover(); - onChangeItemsPerPage(item); - }} - > - {`${item} ${i18n.ROWS}`} - - )), - [closePopover, itemsPerPage, itemsPerPageOptions, onChangeItemsPerPage] - ); - - const totalPages = useMemo( - () => Math.ceil(totalCount / itemsPerPage), - [itemsPerPage, totalCount] - ); - - useEffect(() => { - if (paginationLoading && !isLoading) { - setPaginationLoading(false); - } - }, [isLoading, paginationLoading]); - - if (isLoading && !paginationLoading) { - return ( - - {timelines.getLoadingPanel({ - dataTestSubj: 'LoadingPanelTimeline', - height: '35px', - showBorder: false, - text: loadingText, - width: '100%', - })} - - ); - } - - return ( - - - - - - - - - - - - - - {isLive ? ( - - - {i18n.AUTO_REFRESH_ACTIVE}{' '} - - } - type="iInCircle" - /> - - - ) : ( - - )} - - - - ); -}; - -FooterComponent.displayName = 'FooterComponent'; - -export const Footer = React.memo(FooterComponent); - -Footer.displayName = 'Footer'; diff --git a/x-pack/plugins/security_solution/public/timelines/components/timeline/tabs/eql/index.tsx b/x-pack/plugins/security_solution/public/timelines/components/timeline/tabs/eql/index.tsx index 874acc77ed1b7..209b4305d7b91 100644 --- a/x-pack/plugins/security_solution/public/timelines/components/timeline/tabs/eql/index.tsx +++ b/x-pack/plugins/security_solution/public/timelines/components/timeline/tabs/eql/index.tsx @@ -105,7 +105,7 @@ export const EqlTabContentComponent: React.FC = ({ [end, isBlankTimeline, loadingSourcerer, start] ); - const [dataLoadingState, { events, inspect, totalCount, loadPage, refreshedAt, refetch }] = + const [dataLoadingState, { events, inspect, totalCount, loadNextBatch, refreshedAt, refetch }] = useTimelineEvents({ dataViewId, endDate: end, @@ -289,7 +289,7 @@ export const EqlTabContentComponent: React.FC = ({ refetch={refetch} dataLoadingState={dataLoadingState} totalCount={isBlankTimeline ? 0 : totalCount} - onFetchMoreRecords={loadPage} + onFetchMoreRecords={loadNextBatch} activeTab={activeTab} updatedAt={refreshedAt} isTextBasedQuery={false} diff --git a/x-pack/plugins/security_solution/public/timelines/components/timeline/tabs/pinned/index.tsx b/x-pack/plugins/security_solution/public/timelines/components/timeline/tabs/pinned/index.tsx index 3e3b8f6c564c6..07f9772977624 100644 --- a/x-pack/plugins/security_solution/public/timelines/components/timeline/tabs/pinned/index.tsx +++ b/x-pack/plugins/security_solution/public/timelines/components/timeline/tabs/pinned/index.tsx @@ -139,7 +139,7 @@ export const PinnedTabContentComponent: React.FC = ({ ); const { augmentedColumnHeaders } = useTimelineColumns(columns); - const [queryLoadingState, { events, totalCount, loadPage, refreshedAt, refetch }] = + const [queryLoadingState, { events, totalCount, loadNextBatch, refreshedAt, refetch }] = useTimelineEvents({ endDate: '', id: `pinned-${timelineId}`, @@ -286,7 +286,7 @@ export const PinnedTabContentComponent: React.FC = ({ refetch={refetch} dataLoadingState={queryLoadingState} totalCount={totalCount} - onFetchMoreRecords={loadPage} + onFetchMoreRecords={loadNextBatch} activeTab={TimelineTabs.pinned} updatedAt={refreshedAt} isTextBasedQuery={false} diff --git a/x-pack/plugins/security_solution/public/timelines/components/timeline/tabs/query/index.test.tsx b/x-pack/plugins/security_solution/public/timelines/components/timeline/tabs/query/index.test.tsx index cfd8f86af9dac..b28b223ce46fd 100644 --- a/x-pack/plugins/security_solution/public/timelines/components/timeline/tabs/query/index.test.tsx +++ b/x-pack/plugins/security_solution/public/timelines/components/timeline/tabs/query/index.test.tsx @@ -10,7 +10,6 @@ import React, { useEffect } from 'react'; import QueryTabContent from '.'; import { defaultRowRenderers } from '../../body/renderers'; import { TimelineId } from '../../../../../../common/types/timeline'; -import { useTimelineEvents } from '../../../../containers'; import { useTimelineEventsDetails } from '../../../../containers/details'; import { useSourcererDataView } from '../../../../../sourcerer/containers'; import { mockSourcererScope } from '../../../../../sourcerer/containers/mocks'; @@ -42,12 +41,20 @@ import { createExpandableFlyoutApiMock } from '../../../../../common/mock/expand import { OPEN_FLYOUT_BUTTON_TEST_ID } from '../../../../../notes/components/test_ids'; import { userEvent } from '@testing-library/user-event'; import * as notesApi from '../../../../../notes/api/api'; +import { getMockTimelineSearchSubscription } from '../../../../../common/mock/mock_timeline_search_service'; +import * as useTimelineEventsModule from '../../../../containers'; -jest.mock('../../../../../common/components/user_privileges'); +jest.mock('../../../../../common/utils/route/use_route_spy', () => { + return { + useRouteSpy: jest.fn().mockReturnValue([ + { + pageName: 'timeline', + }, + ]), + }; +}); -jest.mock('../../../../containers', () => ({ - useTimelineEvents: jest.fn(), -})); +jest.mock('../../../../../common/components/user_privileges'); jest.mock('../../../../containers/details'); @@ -60,8 +67,6 @@ jest.mock('../../../../../sourcerer/containers/use_signal_helpers', () => ({ useSignalHelpers: () => ({ signalIndexNeedsInit: false }), })); -jest.mock('../../../../../common/lib/kuery'); - jest.mock('../../../../../common/hooks/use_experimental_features'); jest.mock('react-router-dom', () => ({ @@ -72,6 +77,8 @@ jest.mock('react-router-dom', () => ({ })), })); +const { mockTimelineSearchSubscription } = getMockTimelineSearchSubscription(); + // These tests can take more than standard timeout of 5s // that is why we are increasing it. const SPECIAL_TEST_TIMEOUT = 50000; @@ -128,8 +135,32 @@ const customColumnOrder = [ }, ]; -const mockState = { - ...structuredClone(mockGlobalState), +const mockBaseState = structuredClone(mockGlobalState); + +const mockState: typeof mockGlobalState = { + ...mockBaseState, + timeline: { + ...mockBaseState.timeline, + timelineById: { + [TimelineId.test]: { + ...mockBaseState.timeline.timelineById[TimelineId.test], + /* 1 record for each page */ + itemsPerPage: 1, + itemsPerPageOptions: [1, 2, 3, 4, 5], + /* Returns 1 records in one query */ + sampleSize: 1, + kqlQuery: { + filterQuery: { + kuery: { + kind: 'kuery', + expression: '*', + }, + serializedQuery: '*', + }, + }, + }, + }, + }, }; mockState.timeline.timelineById[TimelineId.test].columns = customColumnOrder; @@ -144,20 +175,18 @@ const renderTestComponents = (props?: Partial { - const fetchNotesMock = jest.spyOn(notesApi, 'fetchNotesByDocumentIds'); + const fetchNotesSpy = jest.spyOn(notesApi, 'fetchNotesByDocumentIds'); beforeAll(() => { - fetchNotesMock.mockImplementation(jest.fn()); + fetchNotesSpy.mockImplementation(jest.fn()); jest.mocked(useExpandableFlyoutApi).mockImplementation(() => ({ ...createExpandableFlyoutApiMock(), openFlyout: mockOpenFlyout, @@ -171,34 +200,30 @@ describe('query tab with unified timeline', () => { }, }); }); + + const baseKibanaServicesMock = createStartServicesMock(); + const kibanaServiceMock: StartServices = { - ...createStartServicesMock(), + ...baseKibanaServicesMock, storage: storageMock, + data: { + ...baseKibanaServicesMock.data, + search: { + ...baseKibanaServicesMock.data.search, + search: mockTimelineSearchSubscription, + }, + }, }; afterEach(() => { jest.clearAllMocks(); storageMock.clear(); - fetchNotesMock.mockClear(); + fetchNotesSpy.mockClear(); cleanup(); localStorage.clear(); }); beforeEach(() => { - useTimelineEventsMock = jest.fn(() => [ - false, - { - events: structuredClone(mockTimelineData.slice(0, 1)), - pageInfo: { - activePage: 0, - totalPages: 3, - }, - refreshedAt: Date.now(), - totalCount: 3, - loadPage: loadPageMock, - }, - ]); - HTMLElement.prototype.getBoundingClientRect = jest.fn(() => { return { width: 1000, @@ -214,8 +239,6 @@ describe('query tab with unified timeline', () => { }; }); - (useTimelineEvents as jest.Mock).mockImplementation(useTimelineEventsMock); - (useTimelineEventsDetails as jest.Mock).mockImplementation(() => [false, {}]); (useSourcererDataView as jest.Mock).mockImplementation(useSourcererDataViewMocked); @@ -297,33 +320,24 @@ describe('query tab with unified timeline', () => { }); describe('pagination', () => { - beforeEach(() => { - // pagination tests need more than 1 record so here - // we return 5 records instead of just 1. - useTimelineEventsMock = jest.fn(() => [ - false, - { - events: structuredClone(mockTimelineData.slice(0, 5)), - pageInfo: { - activePage: 0, - totalPages: 5, + const mockStateWithNoteInTimeline = { + ...mockState, + timeline: { + ...mockState.timeline, + timelineById: { + [TimelineId.test]: { + ...mockState.timeline.timelineById[TimelineId.test], + /* 1 record for each page */ + itemsPerPage: 1, + itemsPerPageOptions: [1, 2, 3, 4, 5], + savedObjectId: 'timeline-1', // match timelineId in mocked notes data + pinnedEventIds: { '1': true }, + /* Returns 3 records */ + sampleSize: 3, }, - refreshedAt: Date.now(), - /* - * `totalCount` could be any number w.r.t this test - * and actually means total hits on elastic search - * and not the fecthed number of records. - * - * This helps in testing `sampleSize` and `loadMore` - */ - totalCount: 50, - loadPage: loadPageMock, }, - ]); - - (useTimelineEvents as jest.Mock).mockImplementation(useTimelineEventsMock); - }); - + }, + }; afterEach(() => { jest.clearAllMocks(); }); @@ -331,23 +345,6 @@ describe('query tab with unified timeline', () => { it( 'should paginate correctly', async () => { - const mockStateWithNoteInTimeline = { - ...mockGlobalState, - timeline: { - ...mockGlobalState.timeline, - timelineById: { - [TimelineId.test]: { - ...mockGlobalState.timeline.timelineById[TimelineId.test], - /* 1 record for each page */ - itemsPerPage: 1, - itemsPerPageOptions: [1, 2, 3, 4, 5], - savedObjectId: 'timeline-1', // match timelineId in mocked notes data - pinnedEventIds: { '1': true }, - }, - }, - }, - }; - render( { ); expect(screen.getByTestId('pagination-button-0')).toHaveAttribute('aria-current', 'true'); - expect(screen.getByTestId('pagination-button-4')).toBeVisible(); - expect(screen.queryByTestId('pagination-button-5')).toBeNull(); + expect(screen.getByTestId('pagination-button-2')).toBeVisible(); + expect(screen.queryByTestId('pagination-button-3')).toBeNull(); - fireEvent.click(screen.getByTestId('pagination-button-4')); + fireEvent.click(screen.getByTestId('pagination-button-2')); await waitFor(() => { - expect(screen.getByTestId('pagination-button-4')).toHaveAttribute('aria-current', 'true'); + expect(screen.getByTestId('pagination-button-2')).toHaveAttribute('aria-current', 'true'); }); }, SPECIAL_TEST_TIMEOUT @@ -381,27 +378,6 @@ describe('query tab with unified timeline', () => { it( 'should load more records according to sample size correctly', async () => { - const mockStateWithNoteInTimeline = { - ...mockGlobalState, - timeline: { - ...mockGlobalState.timeline, - timelineById: { - [TimelineId.test]: { - ...mockGlobalState.timeline.timelineById[TimelineId.test], - itemsPerPage: 1, - /* - * `sampleSize` is the max number of records that are fetched from elasticsearch - * in one request. If hits > sampleSize, you can fetch more records ( <= sampleSize) - */ - sampleSize: 5, - itemsPerPageOptions: [1, 2, 3, 4, 5], - savedObjectId: 'timeline-1', // match timelineId in mocked notes data - pinnedEventIds: { '1': true }, - }, - }, - }, - }; - render( { await waitFor(() => { expect(screen.getByTestId('pagination-button-0')).toHaveAttribute('aria-current', 'true'); - expect(screen.getByTestId('pagination-button-4')).toBeVisible(); + expect(screen.getByTestId('pagination-button-2')).toBeVisible(); }); // Go to last page - fireEvent.click(screen.getByTestId('pagination-button-4')); + fireEvent.click(screen.getByTestId('pagination-button-2')); await waitFor(() => { expect(screen.getByTestId('dscGridSampleSizeFetchMoreLink')).toBeVisible(); }); fireEvent.click(screen.getByTestId('dscGridSampleSizeFetchMoreLink')); - expect(loadPageMock).toHaveBeenNthCalledWith(1, 1); + await waitFor(() => { + expect(screen.getByTestId('pagination-button-2')).toHaveAttribute('aria-current', 'true'); + expect(screen.getByTestId('pagination-button-5')).toBeVisible(); + }); }, SPECIAL_TEST_TIMEOUT ); @@ -432,24 +411,6 @@ describe('query tab with unified timeline', () => { it( 'should load notes for current page only', async () => { - const mockStateWithNoteInTimeline = { - ...mockGlobalState, - timeline: { - ...mockGlobalState.timeline, - timelineById: { - [TimelineId.test]: { - ...mockGlobalState.timeline.timelineById[TimelineId.test], - /* 1 record for each page */ - itemsPerPage: 1, - pageIndex: 0, - itemsPerPageOptions: [1, 2, 3, 4, 5], - savedObjectId: 'timeline-1', // match timelineId in mocked notes data - pinnedEventIds: { '1': true }, - }, - }, - }, - }; - render( { expect(screen.getByTestId('pagination-button-previous')).toBeVisible(); expect(screen.getByTestId('pagination-button-0')).toHaveAttribute('aria-current', 'true'); - expect(fetchNotesMock).toHaveBeenCalledWith(['1']); + expect(fetchNotesSpy).toHaveBeenCalledWith(['1']); // Page : 2 - fetchNotesMock.mockClear(); + fetchNotesSpy.mockClear(); expect(screen.getByTestId('pagination-button-1')).toBeVisible(); fireEvent.click(screen.getByTestId('pagination-button-1')); @@ -477,19 +438,19 @@ describe('query tab with unified timeline', () => { await waitFor(() => { expect(screen.getByTestId('pagination-button-1')).toHaveAttribute('aria-current', 'true'); - expect(fetchNotesMock).toHaveBeenNthCalledWith(1, [mockTimelineData[1]._id]); + expect(fetchNotesSpy).toHaveBeenNthCalledWith(1, [mockTimelineData[1]._id]); }); // Page : 3 - fetchNotesMock.mockClear(); + fetchNotesSpy.mockClear(); expect(screen.getByTestId('pagination-button-2')).toBeVisible(); fireEvent.click(screen.getByTestId('pagination-button-2')); await waitFor(() => { expect(screen.getByTestId('pagination-button-2')).toHaveAttribute('aria-current', 'true'); - expect(fetchNotesMock).toHaveBeenNthCalledWith(1, [mockTimelineData[2]._id]); + expect(fetchNotesSpy).toHaveBeenNthCalledWith(1, [mockTimelineData[2]._id]); }); }, SPECIAL_TEST_TIMEOUT @@ -498,24 +459,6 @@ describe('query tab with unified timeline', () => { it( 'should load notes for correct page size', async () => { - const mockStateWithNoteInTimeline = { - ...mockGlobalState, - timeline: { - ...mockGlobalState.timeline, - timelineById: { - [TimelineId.test]: { - ...mockGlobalState.timeline.timelineById[TimelineId.test], - /* 1 record for each page */ - itemsPerPage: 1, - pageIndex: 0, - itemsPerPageOptions: [1, 2, 3, 4, 5], - savedObjectId: 'timeline-1', // match timelineId in mocked notes data - pinnedEventIds: { '1': true }, - }, - }, - }, - }; - render( { expect(screen.getByTestId('tablePagination-2-rows')).toBeVisible(); }); - fetchNotesMock.mockClear(); + fetchNotesSpy.mockClear(); fireEvent.click(screen.getByTestId('tablePagination-2-rows')); await waitFor(() => { - expect(fetchNotesMock).toHaveBeenNthCalledWith(1, [ + expect(fetchNotesSpy).toHaveBeenNthCalledWith(1, [ mockTimelineData[0]._id, mockTimelineData[1]._id, ]); @@ -554,6 +497,53 @@ describe('query tab with unified timeline', () => { ); }); + const openDisplaySettings = async () => { + expect(screen.getByTestId('dataGridDisplaySelectorButton')).toBeVisible(); + + fireEvent.click(screen.getByTestId('dataGridDisplaySelectorButton')); + + await waitFor(() => { + expect( + screen + .getAllByTestId('unifiedDataTableSampleSizeInput') + .find((el) => el.getAttribute('type') === 'number') + ).toBeVisible(); + }); + }; + + const updateSampleSize = async (sampleSize: number) => { + const sampleSizeInput = screen + .getAllByTestId('unifiedDataTableSampleSizeInput') + .find((el) => el.getAttribute('type') === 'number'); + + expect(sampleSizeInput).toBeVisible(); + + fireEvent.change(sampleSizeInput as HTMLElement, { + target: { value: sampleSize }, + }); + }; + + describe('controls', () => { + it( + 'should reftech on sample size change', + async () => { + renderTestComponents(); + + await waitFor(() => { + expect(screen.getByTestId('discoverDocTable')).toBeVisible(); + }); + expect(screen.queryByTestId('pagination-button-1')).not.toBeInTheDocument(); + + await openDisplaySettings(); + await updateSampleSize(2); + await waitFor(() => { + expect(screen.getByTestId('pagination-button-1')).toBeVisible(); + }); + }, + SPECIAL_TEST_TIMEOUT + ); + }); + describe('columns', () => { it( 'should move column left/right correctly ', @@ -640,12 +630,11 @@ describe('query tab with unified timeline', () => { }); expect(screen.getByTitle('Unsort New-Old')).toBeVisible(); - useTimelineEventsMock.mockClear(); - + useTimelineEventsSpy.mockClear(); fireEvent.click(screen.getByTitle('Sort Old-New')); await waitFor(() => { - expect(useTimelineEventsMock).toHaveBeenNthCalledWith( + expect(useTimelineEventsSpy).toHaveBeenNthCalledWith( 1, expect.objectContaining({ sort: [ @@ -684,12 +673,12 @@ describe('query tab with unified timeline', () => { expect(screen.getByTitle('Sort A-Z')).toBeVisible(); expect(screen.getByTitle('Sort Z-A')).toBeVisible(); - useTimelineEventsMock.mockClear(); + useTimelineEventsSpy.mockClear(); fireEvent.click(screen.getByTitle('Sort A-Z')); await waitFor(() => { - expect(useTimelineEventsMock).toHaveBeenNthCalledWith( + expect(useTimelineEventsSpy).toHaveBeenNthCalledWith( 1, expect.objectContaining({ sort: [ @@ -739,12 +728,12 @@ describe('query tab with unified timeline', () => { expect(screen.getByTitle('Sort Low-High')).toBeVisible(); expect(screen.getByTitle('Sort High-Low')).toBeVisible(); - useTimelineEventsMock.mockClear(); + useTimelineEventsSpy.mockClear(); fireEvent.click(screen.getByTitle('Sort Low-High')); await waitFor(() => { - expect(useTimelineEventsMock).toHaveBeenNthCalledWith( + expect(useTimelineEventsSpy).toHaveBeenNthCalledWith( 1, expect.objectContaining({ sort: [ @@ -1212,12 +1201,12 @@ describe('query tab with unified timeline', () => { 'should disable pinning when event has notes attached in timeline', async () => { const mockStateWithNoteInTimeline = { - ...mockGlobalState, + ...mockState, timeline: { - ...mockGlobalState.timeline, + ...mockState.timeline, timelineById: { [TimelineId.test]: { - ...mockGlobalState.timeline.timelineById[TimelineId.test], + ...mockState.timeline.timelineById[TimelineId.test], savedObjectId: 'timeline-1', // match timelineId in mocked notes data pinnedEventIds: { '1': true }, }, diff --git a/x-pack/plugins/security_solution/public/timelines/components/timeline/tabs/query/index.tsx b/x-pack/plugins/security_solution/public/timelines/components/timeline/tabs/query/index.tsx index d15cf2b5539c3..09db2d1f502a2 100644 --- a/x-pack/plugins/security_solution/public/timelines/components/timeline/tabs/query/index.tsx +++ b/x-pack/plugins/security_solution/public/timelines/components/timeline/tabs/query/index.tsx @@ -174,24 +174,22 @@ export const QueryTabContentComponent: React.FC = ({ const { augmentedColumnHeaders, defaultColumns, timelineQueryFieldsFromColumns } = useTimelineColumns(columns); - const [ - dataLoadingState, - { events, inspect, totalCount, loadPage: loadNextEventBatch, refreshedAt, refetch }, - ] = useTimelineEvents({ - dataViewId, - endDate: end, - fields: timelineQueryFieldsFromColumns, - filterQuery: combinedQueries?.filterQuery, - id: timelineId, - indexNames: selectedPatterns, - language: kqlQuery.language, - limit: sampleSize, - runtimeMappings: sourcererDataView?.runtimeFieldMap as RunTimeMappings, - skip: !canQueryTimeline, - sort: timelineQuerySortField, - startDate: start, - timerangeKind, - }); + const [dataLoadingState, { events, inspect, totalCount, loadNextBatch, refreshedAt, refetch }] = + useTimelineEvents({ + dataViewId, + endDate: end, + fields: timelineQueryFieldsFromColumns, + filterQuery: combinedQueries?.filterQuery, + id: timelineId, + indexNames: selectedPatterns, + language: kqlQuery.language, + limit: sampleSize, + runtimeMappings: sourcererDataView.runtimeFieldMap as RunTimeMappings, + skip: !canQueryTimeline, + sort: timelineQuerySortField, + startDate: start, + timerangeKind, + }); const { onLoad: loadNotesOnEventsLoad } = useFetchNotes(); @@ -384,7 +382,7 @@ export const QueryTabContentComponent: React.FC = ({ dataLoadingState={dataLoadingState} totalCount={isBlankTimeline ? 0 : totalCount} leadingControlColumns={leadingControlColumns as EuiDataGridControlColumn[]} - onFetchMoreRecords={loadNextEventBatch} + onFetchMoreRecords={loadNextBatch} activeTab={activeTab} updatedAt={refreshedAt} isTextBasedQuery={false} diff --git a/x-pack/plugins/security_solution/public/timelines/components/timeline/unified_components/data_table/index.test.tsx b/x-pack/plugins/security_solution/public/timelines/components/timeline/unified_components/data_table/index.test.tsx index 3f24fc8df4aa9..99d65ef5101aa 100644 --- a/x-pack/plugins/security_solution/public/timelines/components/timeline/unified_components/data_table/index.test.tsx +++ b/x-pack/plugins/security_solution/public/timelines/components/timeline/unified_components/data_table/index.test.tsx @@ -15,7 +15,7 @@ import { useSourcererDataView } from '../../../../../sourcerer/containers'; import type { ComponentProps } from 'react'; import { getColumnHeaders } from '../../body/column_headers/helpers'; import { mockSourcererScope } from '../../../../../sourcerer/containers/mocks'; -import { timelineActions } from '../../../../store'; +import * as timelineActions from '../../../../store/actions'; import { useExpandableFlyoutApi } from '@kbn/expandable-flyout'; import { defaultUdtHeaders } from '../../body/column_headers/default_headers'; @@ -31,10 +31,12 @@ jest.mock('react-router-dom', () => ({ const onFieldEditedMock = jest.fn(); const refetchMock = jest.fn(); -const onChangePageMock = jest.fn(); +const onFetchMoreRecordsMock = jest.fn(); const openFlyoutMock = jest.fn(); +const updateSampleSizeSpy = jest.spyOn(timelineActions, 'updateSampleSize'); + jest.mock('@kbn/expandable-flyout'); const initialEnrichedColumns = getColumnHeaders( @@ -72,7 +74,7 @@ const TestComponent = (props: TestComponentProps) => { refetch={refetchMock} dataLoadingState={DataLoadingState.loaded} totalCount={mockTimelineData.length} - onFetchMoreRecords={onChangePageMock} + onFetchMoreRecords={onFetchMoreRecordsMock} updatedAt={Date.now()} onSetColumns={jest.fn()} onFilter={jest.fn()} @@ -97,6 +99,7 @@ describe('unified data table', () => { }); }); afterEach(() => { + updateSampleSizeSpy.mockClear(); jest.clearAllMocks(); }); @@ -199,7 +202,7 @@ describe('unified data table', () => { }); it( - 'should refetch on sample size change', + 'should update sample size correctly', async () => { render(); @@ -217,8 +220,11 @@ describe('unified data table', () => { target: { value: '10' }, }); + updateSampleSizeSpy.mockClear(); + await waitFor(() => { - expect(refetchMock).toHaveBeenCalledTimes(1); + expect(updateSampleSizeSpy).toHaveBeenCalledTimes(1); + expect(updateSampleSizeSpy).toHaveBeenCalledWith({ id: TimelineId.test, sampleSize: 10 }); }); }, SPECIAL_TEST_TIMEOUT @@ -315,7 +321,7 @@ describe('unified data table', () => { expect(screen.getByTestId('dscGridSampleSizeFetchMoreLink')).toBeVisible(); fireEvent.click(screen.getByTestId('dscGridSampleSizeFetchMoreLink')); await waitFor(() => { - expect(onChangePageMock).toHaveBeenNthCalledWith(1, 1); + expect(onFetchMoreRecordsMock).toHaveBeenCalledTimes(1); }); }, SPECIAL_TEST_TIMEOUT diff --git a/x-pack/plugins/security_solution/public/timelines/components/timeline/unified_components/data_table/index.tsx b/x-pack/plugins/security_solution/public/timelines/components/timeline/unified_components/data_table/index.tsx index 99e00547f1c33..55ad2fb97def3 100644 --- a/x-pack/plugins/security_solution/public/timelines/components/timeline/unified_components/data_table/index.tsx +++ b/x-pack/plugins/security_solution/public/timelines/components/timeline/unified_components/data_table/index.tsx @@ -136,7 +136,6 @@ export const TimelineDataTableComponent: React.FC = memo( } = useKibana(); const [expandedDoc, setExpandedDoc] = useState(); - const [fetchedPage, setFechedPage] = useState(0); const onCloseExpandableFlyout = useCallback((id: string) => { setExpandedDoc((prev) => (!prev ? prev : undefined)); @@ -237,9 +236,8 @@ export const TimelineDataTableComponent: React.FC = memo( ); const handleFetchMoreRecords = useCallback(() => { - onFetchMoreRecords(fetchedPage + 1); - setFechedPage(fetchedPage + 1); - }, [fetchedPage, onFetchMoreRecords]); + onFetchMoreRecords(); + }, [onFetchMoreRecords]); const additionalControls = useMemo( () => , @@ -252,10 +250,9 @@ export const TimelineDataTableComponent: React.FC = memo( (newSampleSize: number) => { if (newSampleSize !== sampleSize) { dispatch(timelineActions.updateSampleSize({ id: timelineId, sampleSize: newSampleSize })); - refetch(); } }, - [dispatch, sampleSize, timelineId, refetch] + [dispatch, sampleSize, timelineId] ); const onUpdateRowHeight = useCallback( diff --git a/x-pack/plugins/security_solution/public/timelines/containers/index.test.tsx b/x-pack/plugins/security_solution/public/timelines/containers/index.test.tsx index ba0151ff77b59..c41e62dc2d1cd 100644 --- a/x-pack/plugins/security_solution/public/timelines/containers/index.test.tsx +++ b/x-pack/plugins/security_solution/public/timelines/containers/index.test.tsx @@ -6,16 +6,18 @@ */ import { DataLoadingState } from '@kbn/unified-data-table'; -import { renderHook, act } from '@testing-library/react-hooks'; +import { act, waitFor, renderHook } from '@testing-library/react'; import type { TimelineArgs, UseTimelineEventsProps } from '.'; -import { initSortDefault, useTimelineEvents } from '.'; +import * as useTimelineEventsModule from '.'; import { SecurityPageName } from '../../../common/constants'; import { TimelineId } from '../../../common/types/timeline'; import { useIsExperimentalFeatureEnabled } from '../../common/hooks/use_experimental_features'; -import { mockTimelineData } from '../../common/mock'; import { useRouteSpy } from '../../common/utils/route/use_route_spy'; import { useFetchNotes } from '../../notes/hooks/use_fetch_notes'; -import { waitFor } from '@testing-library/dom'; +import { useKibana } from '../../common/lib/kibana'; +import { getMockTimelineSearchSubscription } from '../../common/mock/mock_timeline_search_service'; + +const { initSortDefault, useTimelineEvents } = useTimelineEventsModule; const mockDispatch = jest.fn(); jest.mock('react-redux', () => { @@ -31,10 +33,6 @@ jest.mock('../../notes/hooks/use_fetch_notes'); const onLoadMock = jest.fn(); const useFetchNotesMock = useFetchNotes as jest.Mock; -const mockEvents = mockTimelineData.slice(0, 10); - -const mockSearch = jest.fn(); - jest.mock('../../common/lib/apm/use_track_http_request'); jest.mock('../../common/hooks/use_experimental_features'); const useIsExperimentalFeatureEnabledMock = useIsExperimentalFeatureEnabled as jest.Mock; @@ -46,55 +44,7 @@ jest.mock('../../common/lib/kibana', () => ({ addWarning: jest.fn(), remove: jest.fn(), }), - useKibana: jest.fn().mockReturnValue({ - services: { - application: { - capabilities: { - siem: { - crud: true, - }, - }, - }, - data: { - search: { - search: jest.fn().mockImplementation((args) => { - mockSearch(); - return { - subscribe: jest.fn().mockImplementation(({ next }) => { - const timeoutHandler = setTimeout(() => { - next({ - isRunning: false, - isPartial: false, - inspect: { - dsl: [], - response: [], - }, - edges: mockEvents.map((item) => ({ node: item })), - pageInfo: { - activePage: args.pagination.activePage, - totalPages: 10, - }, - rawResponse: {}, - totalCount: mockTimelineData.length, - }); - }, 50); - return { - unsubscribe: jest.fn(() => { - clearTimeout(timeoutHandler); - }), - }; - }), - }; - }), - }, - }, - notifications: { - toasts: { - addWarning: jest.fn(), - }, - }, - }, - }), + useKibana: jest.fn(), })); const mockUseRouteSpy: jest.Mock = useRouteSpy as jest.Mock; @@ -112,7 +62,40 @@ mockUseRouteSpy.mockReturnValue([ }, ]); -describe('useTimelineEvents', () => { +const startDate: string = '2020-07-07T08:20:18.966Z'; +const endDate: string = '3000-01-01T00:00:00.000Z'; +const props: UseTimelineEventsProps = { + dataViewId: 'data-view-id', + endDate, + id: TimelineId.active, + indexNames: ['filebeat-*'], + fields: ['@timestamp', 'event.kind'], + filterQuery: '*', + startDate, + limit: 25, + runtimeMappings: {}, + sort: initSortDefault, + skip: false, +}; + +const { mockTimelineSearchSubscription: mockSearchSubscription, mockSearchWithArgs: mockSearch } = + getMockTimelineSearchSubscription(); + +const loadNextBatch = async (result: { current: [DataLoadingState, TimelineArgs] }) => { + act(() => { + result.current[1].loadNextBatch(); + }); + + await waitFor(() => { + expect(result.current[0]).toBe(DataLoadingState.loadingMore); + }); + + await waitFor(() => { + expect(result.current[0]).toBe(DataLoadingState.loaded); + }); +}; + +describe('useTimelineEventsHandler', () => { useIsExperimentalFeatureEnabledMock.mockReturnValue(false); beforeEach(() => { @@ -123,155 +106,136 @@ describe('useTimelineEvents', () => { useFetchNotesMock.mockReturnValue({ onLoad: onLoadMock, }); + + (useKibana as jest.Mock).mockReturnValue({ + services: { + application: { + capabilities: { + siem: { + crud: true, + }, + }, + }, + data: { + search: { + search: mockSearchSubscription, + }, + }, + notifications: { + toasts: { + addWarning: jest.fn(), + }, + }, + }, + }); }); - const startDate: string = '2020-07-07T08:20:18.966Z'; - const endDate: string = '3000-01-01T00:00:00.000Z'; - const props: UseTimelineEventsProps = { - dataViewId: 'data-view-id', - endDate, - id: TimelineId.active, - indexNames: ['filebeat-*'], - fields: ['@timestamp', 'event.kind'], - filterQuery: '', - startDate, - limit: 25, - runtimeMappings: {}, - sort: initSortDefault, - skip: false, - }; + test('should init empty response', async () => { + const { result } = renderHook((args) => useTimelineEvents(args), { + initialProps: props, + }); - test('init', async () => { - await act(async () => { - const { result, waitForNextUpdate } = renderHook< - UseTimelineEventsProps, - [DataLoadingState, TimelineArgs] - >((args) => useTimelineEvents(args), { - initialProps: { ...props }, - }); + expect(result.current).toEqual([ + DataLoadingState.loading, + { + events: [], + id: TimelineId.active, + inspect: expect.objectContaining({ dsl: [], response: [] }), + loadNextBatch: expect.any(Function), + pageInfo: expect.objectContaining({ + activePage: 0, + querySize: 0, + }), + refetch: expect.any(Function), + totalCount: -1, + refreshedAt: 0, + }, + ]); + }); - // useEffect on params request - await waitForNextUpdate(); + test('should make events search request correctly', async () => { + const { result } = renderHook<[DataLoadingState, TimelineArgs], UseTimelineEventsProps>( + (args) => useTimelineEvents(args), + { + initialProps: props, + } + ); + await waitFor(() => { + expect(mockSearch).toHaveBeenCalledTimes(1); + expect(mockSearch).toHaveBeenNthCalledWith( + 1, + expect.objectContaining({ pagination: { activePage: 0, querySize: 25 } }) + ); + expect(result.current[1].events).toHaveLength(25); expect(result.current).toEqual([ DataLoadingState.loaded, { - events: [], + events: expect.any(Array), id: TimelineId.active, inspect: result.current[1].inspect, - loadPage: result.current[1].loadPage, - pageInfo: result.current[1].pageInfo, + loadNextBatch: result.current[1].loadNextBatch, + pageInfo: { + activePage: 0, + querySize: 25, + }, refetch: result.current[1].refetch, - totalCount: -1, - refreshedAt: 0, + totalCount: 32, + refreshedAt: result.current[1].refreshedAt, }, ]); }); }); - test('happy path query', async () => { - await act(async () => { - const { result, waitForNextUpdate, rerender } = renderHook< - UseTimelineEventsProps, - [DataLoadingState, TimelineArgs] - >((args) => useTimelineEvents(args), { - initialProps: { ...props, startDate: '', endDate: '' }, - }); - - // useEffect on params request - await waitForNextUpdate(); - rerender({ ...props, startDate, endDate }); - // useEffect on params request - await waitForNextUpdate(); - - await waitFor(() => { - expect(mockSearch).toHaveBeenCalledTimes(2); - expect(result.current).toEqual([ - DataLoadingState.loaded, - { - events: mockEvents, - id: TimelineId.active, - inspect: result.current[1].inspect, - loadPage: result.current[1].loadPage, - pageInfo: result.current[1].pageInfo, - refetch: result.current[1].refetch, - totalCount: 32, - refreshedAt: result.current[1].refreshedAt, - }, - ]); - }); + test('should mock cache for active timeline when switching page', async () => { + const { result, rerender } = renderHook< + [DataLoadingState, TimelineArgs], + UseTimelineEventsProps + >((args) => useTimelineEvents(args), { + initialProps: props, }); - }); - test('Mock cache for active timeline when switching page', async () => { - await act(async () => { - const { result, waitForNextUpdate, rerender } = renderHook< - UseTimelineEventsProps, - [DataLoadingState, TimelineArgs] - >((args) => useTimelineEvents(args), { - initialProps: { ...props, startDate: '', endDate: '' }, - }); + mockUseRouteSpy.mockReturnValue([ + { + pageName: SecurityPageName.timelines, + detailName: undefined, + tabName: undefined, + search: '', + pathName: '/timelines', + }, + ]); - // useEffect on params request - await waitForNextUpdate(); - rerender({ ...props, startDate, endDate }); - // useEffect on params request - await waitForNextUpdate(); + rerender({ ...props, startDate, endDate }); - mockUseRouteSpy.mockReturnValue([ - { - pageName: SecurityPageName.timelines, - detailName: undefined, - tabName: undefined, - search: '', - pathName: '/timelines', - }, - ]); - - await waitFor(() => { - expect(mockSearch).toHaveBeenCalledTimes(2); - - expect(result.current).toEqual([ - DataLoadingState.loaded, - { - events: mockEvents, - id: TimelineId.active, - inspect: result.current[1].inspect, - loadPage: result.current[1].loadPage, - pageInfo: result.current[1].pageInfo, - refetch: result.current[1].refetch, - totalCount: 32, - refreshedAt: result.current[1].refreshedAt, - }, - ]); - }); + await waitFor(() => { + expect(result.current[0]).toEqual(DataLoadingState.loaded); }); + + expect(mockSearch).toHaveBeenCalledTimes(1); + + expect(result.current[1].events).toHaveLength(25); + + expect(result.current).toEqual([ + DataLoadingState.loaded, + { + events: expect.any(Array), + id: TimelineId.active, + inspect: result.current[1].inspect, + loadNextBatch: result.current[1].loadNextBatch, + pageInfo: result.current[1].pageInfo, + refetch: result.current[1].refetch, + totalCount: 32, + refreshedAt: result.current[1].refreshedAt, + }, + ]); }); test('Correlation pagination is calling search strategy when switching page', async () => { - await act(async () => { - const { result, waitForNextUpdate, rerender } = renderHook< - UseTimelineEventsProps, - [DataLoadingState, TimelineArgs] - >((args) => useTimelineEvents(args), { - initialProps: { - ...props, - language: 'eql', - eqlOptions: { - eventCategoryField: 'category', - tiebreakerField: '', - timestampField: '@timestamp', - query: 'find it EQL', - size: 100, - }, - }, - }); - - // useEffect on params request - await waitForNextUpdate(); - rerender({ + const { result, rerender } = renderHook< + [DataLoadingState, TimelineArgs], + UseTimelineEventsProps + >((args) => useTimelineEvents(args), { + initialProps: { ...props, - startDate, - endDate, language: 'eql', eqlOptions: { eventCategoryField: 'category', @@ -280,32 +244,102 @@ describe('useTimelineEvents', () => { query: 'find it EQL', size: 100, }, + }, + }); + + // useEffect on params request + await waitFor(() => new Promise((resolve) => resolve(null))); + rerender({ + ...props, + startDate, + endDate, + language: 'eql', + eqlOptions: { + eventCategoryField: 'category', + tiebreakerField: '', + timestampField: '@timestamp', + query: 'find it EQL', + size: 100, + }, + }); + // useEffect on params request + await waitFor(() => new Promise((resolve) => resolve(null))); + mockSearch.mockReset(); + act(() => { + result.current[1].loadNextBatch(); + }); + await waitFor(() => expect(mockSearch).toHaveBeenCalledTimes(1)); + }); + + describe('error/invalid states', () => { + const uniqueError = 'UNIQUE_ERROR'; + const onError = jest.fn(); + const mockSubscribeWithError = jest.fn(({ error }) => { + error(uniqueError); + }); + + beforeEach(() => { + onError.mockClear(); + mockSubscribeWithError.mockClear(); + + (useKibana as jest.Mock).mockReturnValue({ + services: { + data: { + search: { + search: () => ({ + subscribe: jest.fn().mockImplementation(({ error }) => { + const requestTimeout = setTimeout(() => { + mockSubscribeWithError({ error }); + }, 100); + + return { + unsubscribe: () => { + clearTimeout(requestTimeout); + }, + }; + }), + }), + showError: onError, + }, + }, + }, + }); + }); + + test('should broadcast correct loading state when request throws error', async () => { + const { result } = renderHook((args) => useTimelineEvents(args), { + initialProps: { ...props }, + }); + + expect(result.current[0]).toBe(DataLoadingState.loading); + + await waitFor(() => { + expect(onError).toHaveBeenCalledWith(uniqueError); + expect(result.current[0]).toBe(DataLoadingState.loaded); + }); + }); + test('should should not fire any request when indexName is empty', async () => { + const { result } = renderHook((args) => useTimelineEvents(args), { + initialProps: { ...props, indexNames: [] }, + }); + + await waitFor(() => { + expect(mockSearch).not.toHaveBeenCalled(); + expect(result.current[0]).toBe(DataLoadingState.loaded); }); - // useEffect on params request - await waitForNextUpdate(); - mockSearch.mockReset(); - result.current[1].loadPage(4); - await waitForNextUpdate(); - expect(mockSearch).toHaveBeenCalledTimes(1); }); }); - test('should query again when a new field is added', async () => { - await act(async () => { - const { waitForNextUpdate, rerender } = renderHook< - UseTimelineEventsProps, - [DataLoadingState, TimelineArgs] - >((args) => useTimelineEvents(args), { - initialProps: { ...props, startDate: '', endDate: '' }, + describe('fields', () => { + test('should query again when a new field is added', async () => { + const { rerender } = renderHook((args) => useTimelineEvents(args), { + initialProps: props, }); - // useEffect on params request - await waitForNextUpdate(); - rerender({ ...props, startDate, endDate }); - // useEffect on params request - await waitForNextUpdate(); + await waitFor(() => { + expect(mockSearch).toHaveBeenCalledTimes(1); + }); - expect(mockSearch).toHaveBeenCalledTimes(2); mockSearch.mockClear(); rerender({ @@ -315,83 +349,332 @@ describe('useTimelineEvents', () => { fields: ['@timestamp', 'event.kind', 'event.category'], }); + await waitFor(() => expect(mockSearch).toHaveBeenCalledTimes(1)); + }); + + test('should not query again when a field is removed', async () => { + const { rerender } = renderHook((args) => useTimelineEvents(args), { + initialProps: props, + }); + await waitFor(() => { expect(mockSearch).toHaveBeenCalledTimes(1); }); + mockSearch.mockClear(); + + rerender({ ...props, fields: ['@timestamp'] }); + + await waitFor(() => expect(mockSearch).toHaveBeenCalledTimes(0)); + }); + test('should not query again when a removed field is added back', async () => { + const { rerender } = renderHook((args) => useTimelineEvents(args), { + initialProps: props, + }); + + expect(mockSearch).toHaveBeenCalledTimes(1); + mockSearch.mockClear(); + + // remove `event.kind` from default fields + rerender({ ...props, fields: ['@timestamp'] }); + + await waitFor(() => expect(mockSearch).toHaveBeenCalledTimes(0)); + + // request default Fields + rerender({ ...props }); + + await waitFor(() => expect(mockSearch).toHaveBeenCalledTimes(0)); }); }); - test('should not query again when a field is removed', async () => { - await act(async () => { - const { waitForNextUpdate, rerender } = renderHook< - UseTimelineEventsProps, - [DataLoadingState, TimelineArgs] - >((args) => useTimelineEvents(args), { - initialProps: { ...props, startDate: '', endDate: '' }, + describe('batching', () => { + test('should broadcast correct loading state based on the batch being fetched', async () => { + const { result } = renderHook((args) => useTimelineEvents(args), { + initialProps: { ...props }, }); - // useEffect on params request - await waitForNextUpdate(); - rerender({ ...props, startDate, endDate }); - // useEffect on params request - await waitForNextUpdate(); + await waitFor(() => { + expect(result.current[0]).toBe(DataLoadingState.loading); + }); - expect(mockSearch).toHaveBeenCalledTimes(2); - mockSearch.mockClear(); + await waitFor(() => { + expect(result.current[0]).toBe(DataLoadingState.loaded); + }); - rerender({ ...props, startDate, endDate, fields: ['@timestamp'] }); + act(() => { + result.current[1].loadNextBatch(); + }); + + expect(result.current[0]).toBe(DataLoadingState.loadingMore); - expect(mockSearch).toHaveBeenCalledTimes(0); + await waitFor(() => { + expect(result.current[0]).toBe(DataLoadingState.loaded); + }); }); - }); - test('should not query again when a removed field is added back', async () => { - await act(async () => { - const { waitForNextUpdate, rerender } = renderHook< - UseTimelineEventsProps, - [DataLoadingState, TimelineArgs] - >((args) => useTimelineEvents(args), { - initialProps: { ...props, startDate: '', endDate: '' }, + test('should request incremental batches when next batch has been requested', async () => { + const { result } = renderHook((args) => useTimelineEvents(args), { + initialProps: { ...props }, }); - // useEffect on params request - await waitForNextUpdate(); - rerender({ ...props, startDate, endDate }); - // useEffect on params request - await waitForNextUpdate(); + await waitFor(() => { + expect(result.current[0]).toBe(DataLoadingState.loaded); + expect(mockSearch).toHaveBeenNthCalledWith( + 1, + expect.objectContaining({ pagination: { activePage: 0, querySize: 25 } }) + ); + }); - expect(mockSearch).toHaveBeenCalledTimes(2); mockSearch.mockClear(); - // remove `event.kind` from default fields - rerender({ ...props, startDate, endDate, fields: ['@timestamp'] }); + await loadNextBatch(result); - expect(mockSearch).toHaveBeenCalledTimes(0); + await waitFor(() => { + expect(mockSearch).toHaveBeenNthCalledWith( + 1, + expect.objectContaining({ pagination: { activePage: 1, querySize: 25 } }) + ); + }); - // request default Fields - rerender({ ...props, startDate, endDate }); + mockSearch.mockClear(); + + await loadNextBatch(result); - expect(mockSearch).toHaveBeenCalledTimes(0); + await waitFor(() => { + expect(mockSearch).toHaveBeenNthCalledWith( + 1, + expect.objectContaining({ pagination: { activePage: 2, querySize: 25 } }) + ); + }); }); - }); - test('should return the combined list of events for all the pages when multiple pages are queried', async () => { - await act(async () => { + test('should fetch new columns data for the all the batches ', async () => { + const { result, rerender } = renderHook((args) => useTimelineEvents(args), { + initialProps: { ...props }, + }); + + await waitFor(() => { + expect(result.current[0]).toBe(DataLoadingState.loaded); + }); + + //////// + // fetch 2 more batches before requesting new column + //////// + await loadNextBatch(result); + + await loadNextBatch(result); + /////// + + rerender({ ...props, fields: [...props.fields, 'new_column'] }); + + await waitFor(() => { + expect(result.current[0]).toBe(DataLoadingState.loaded); + expect(mockSearch).toHaveBeenCalledWith( + expect.objectContaining({ + fields: ['@timestamp', 'event.kind', 'new_column'], + pagination: { activePage: 0, querySize: 75 }, + }) + ); + }); + }); + + test('should reset batch to 0th when the data is `refetched`', async () => { const { result } = renderHook((args) => useTimelineEvents(args), { initialProps: { ...props }, }); + + await waitFor(() => { + expect(mockSearch).toHaveBeenCalledWith( + expect.objectContaining({ pagination: { activePage: 0, querySize: 25 } }) + ); + }); + + mockSearch.mockClear(); + + await loadNextBatch(result); + + await waitFor(() => { + expect(mockSearch).toHaveBeenCalledWith( + expect.objectContaining({ pagination: { activePage: 1, querySize: 25 } }) + ); + }); + + mockSearch.mockClear(); + + act(() => { + result.current[1].refetch(); + }); + + await waitFor(() => { + expect(mockSearch).toHaveBeenCalledTimes(1); + expect(mockSearch).toHaveBeenNthCalledWith( + 1, + expect.objectContaining({ pagination: { activePage: 0, querySize: 25 } }) + ); + }); + }); + + test('should query all batches when new column is added', async () => { + const { result, rerender } = renderHook((args) => useTimelineEvents(args), { + initialProps: { ...props }, + }); + + await waitFor(() => { + expect(mockSearch).toHaveBeenCalledWith( + expect.objectContaining({ pagination: { activePage: 0, querySize: 25 } }) + ); + }); + mockSearch.mockClear(); + + await loadNextBatch(result); + + await waitFor(() => { + expect(mockSearch).toHaveBeenCalledWith( + expect.objectContaining({ pagination: { activePage: 1, querySize: 25 } }) + ); + }); + + mockSearch.mockClear(); + + rerender({ ...props, fields: [...props.fields, 'new_column'] }); + + await waitFor(() => { + expect(mockSearch).toHaveBeenCalledWith( + expect.objectContaining({ pagination: { activePage: 0, querySize: 50 } }) + ); + }); + mockSearch.mockClear(); + + await loadNextBatch(result); + + await waitFor(() => { + expect(mockSearch).toHaveBeenCalledWith( + expect.objectContaining({ pagination: { activePage: 2, querySize: 25 } }) + ); + }); + }); + + test('should combine batches correctly when new column is added', async () => { + const { result, rerender } = renderHook((args) => useTimelineEvents(args), { + initialProps: { ...props, limit: 5 }, + }); + + await waitFor(() => { + expect(result.current[1].events.length).toBe(5); + }); + + ////////////////////// + // Batch 2 + await loadNextBatch(result); + await waitFor(() => { + expect(result.current[1].events.length).toBe(10); + }); + ////////////////////// + + ////////////////////// + // Batch 3 + await loadNextBatch(result); + await waitFor(() => { + expect(result.current[1].events.length).toBe(15); + }); + ////////////////////// + + /////////////////////////////////////////// + // add new column + // Fetch all 3 batches together + rerender({ ...props, limit: 5, fields: [...props.fields, 'new_column'] }); + + await waitFor(() => { + expect(result.current[0]).toBe(DataLoadingState.loadingMore); + }); + + // should fetch all the records together await waitFor(() => { - expect(result.current[1].events).toHaveLength(10); + expect(result.current[0]).toBe(DataLoadingState.loaded); + expect(result.current[1].events.length).toBe(15); + expect(result.current[1].pageInfo).toMatchObject({ + activePage: 0, + querySize: 15, + }); }); + /////////////////////////////////////////// + + ////////////////////// + // subsequent batch should be fetched incrementally + // Batch 4 + await loadNextBatch(result); + + await waitFor(() => { + expect(result.current[1].events.length).toBe(20); + expect(result.current[1].pageInfo).toMatchObject({ + activePage: 3, + querySize: 5, + }); + }); + ////////////////////// + + ////////////////////// + // Batch 5 + await loadNextBatch(result); + + await waitFor(() => { + expect(result.current[1].events.length).toBe(25); + expect(result.current[1].pageInfo).toMatchObject({ + activePage: 4, + querySize: 5, + }); + }); + ////////////////////// + }); + + test('should request 0th batch (refetch) when batchSize is changed', async () => { + const { result, rerender } = renderHook((args) => useTimelineEvents(args), { + initialProps: { ...props, limit: 5 }, + }); + + ////////////////////// + // Batch 2 + await loadNextBatch(result); + + ////////////////////// + // Batch 3 + await loadNextBatch(result); + + mockSearch.mockClear(); + + // change the batch size + rerender({ ...props, limit: 10 }); + + await waitFor(() => { + expect(result.current[0]).toBe(DataLoadingState.loaded); + expect(mockSearch).toHaveBeenCalledWith( + expect.objectContaining({ pagination: { activePage: 0, querySize: 10 } }) + ); + }); + }); + + test('should return correct list of events ( 0th batch ) when batchSize is changed', async () => { + const { result, rerender } = renderHook((args) => useTimelineEvents(args), { + initialProps: { ...props, limit: 5 }, + }); + + ////////////////////// + // Batch 2 + await loadNextBatch(result); + + ////////////////////// + // Batch 3 + await loadNextBatch(result); - result.current[1].loadPage(1); + // change the batch size + rerender({ ...props, limit: 10 }); await waitFor(() => { - expect(result.current[0]).toEqual(DataLoadingState.loadingMore); + expect(result.current[0]).toBe(DataLoadingState.loading); }); await waitFor(() => { - expect(result.current[1].events).toHaveLength(20); + expect(result.current[0]).toBe(DataLoadingState.loaded); + expect(result.current[1].events.length).toBe(10); }); }); }); diff --git a/x-pack/plugins/security_solution/public/timelines/containers/index.tsx b/x-pack/plugins/security_solution/public/timelines/containers/index.tsx index bd9d9e9bbb4df..ca0bea1be47d9 100644 --- a/x-pack/plugins/security_solution/public/timelines/containers/index.tsx +++ b/x-pack/plugins/security_solution/public/timelines/containers/index.tsx @@ -53,7 +53,7 @@ export interface TimelineArgs { inspect: InspectResponse; /** - * `loadPage` loads the next page/batch of records. + * `loadNextBatch` loads the next page/batch of records. * This is different from the data grid pages. Data grid pagination is only * client side and changing data grid pages does not impact this function. * @@ -61,7 +61,7 @@ export interface TimelineArgs { * irrespective of where user is in Data grid pagination. * */ - loadPage: LoadPage; + loadNextBatch: LoadPage; pageInfo: Pick; refetch: inputsModel.Refetch; totalCount: number; @@ -72,7 +72,7 @@ type OnNextResponseHandler = (response: TimelineArgs) => Promise | void; type TimelineEventsSearchHandler = (onNextResponse?: OnNextResponseHandler) => void; -type LoadPage = (newActivePage: number) => void; +type LoadPage = () => void; type TimelineRequest = T extends 'kuery' ? TimelineEventsAllOptionsInput @@ -167,7 +167,7 @@ export const useTimelineEventsHandler = ({ const abortCtrl = useRef(new AbortController()); const searchSubscription$ = useRef(new Subscription()); const [loading, setLoading] = useState(DataLoadingState.loaded); - const [activePage, setActivePage] = useState( + const [activeBatch, setActiveBatch] = useState( id === TimelineId.active ? activeTimeline.getActivePage() : 0 ); const [timelineRequest, setTimelineRequest] = useState | null>( @@ -184,7 +184,7 @@ export const useTimelineEventsHandler = ({ }, [dispatch, id]); /** - * `wrappedLoadPage` loads the next page/batch of records. + * `loadBatchHandler` loads the next batch of records. * This is different from the data grid pages. Data grid pagination is only * client side and changing data grid pages does not impact this function. * @@ -192,18 +192,23 @@ export const useTimelineEventsHandler = ({ * irrespective of where user is in Data grid pagination. * */ - const wrappedLoadPage = useCallback( - (newActivePage: number) => { + const loadBatchHandler = useCallback( + (newActiveBatch: number) => { clearSignalsState(); if (id === TimelineId.active) { - activeTimeline.setActivePage(newActivePage); + activeTimeline.setActivePage(newActiveBatch); } - setActivePage(newActivePage); + + setActiveBatch(newActiveBatch); }, [clearSignalsState, id] ); + const loadNextBatch = useCallback(() => { + loadBatchHandler(activeBatch + 1); + }, [activeBatch, loadBatchHandler]); + useEffect(() => { return () => { searchSubscription$.current?.unsubscribe(); @@ -214,8 +219,13 @@ export const useTimelineEventsHandler = ({ if (refetch.current != null) { refetch.current(); } - wrappedLoadPage(0); - }, [wrappedLoadPage]); + loadBatchHandler(0); + }, [loadBatchHandler]); + + useEffect(() => { + // when batch size changes, refetch DataGrid + setActiveBatch(0); + }, [limit]); const [timelineResponse, setTimelineResponse] = useState({ id, @@ -230,7 +240,7 @@ export const useTimelineEventsHandler = ({ querySize: 0, }, events: [], - loadPage: wrappedLoadPage, + loadNextBatch, refreshedAt: 0, }); @@ -246,7 +256,8 @@ export const useTimelineEventsHandler = ({ const asyncSearch = async () => { prevTimelineRequest.current = request; abortCtrl.current = new AbortController(); - if (activePage === 0) { + + if (activeBatch === 0) { setLoading(DataLoadingState.loading); } else { setLoading(DataLoadingState.loadingMore); @@ -317,7 +328,6 @@ export const useTimelineEventsHandler = ({ } else { prevTimelineRequest.current = activeTimeline.getRequest(); } - refetch.current = asyncSearch; setTimelineResponse((prevResp) => { const resp = @@ -325,11 +335,7 @@ export const useTimelineEventsHandler = ({ ? activeTimeline.getEqlResponse() : activeTimeline.getResponse(); if (resp != null) { - return { - ...resp, - refetch: refetchGrid, - loadPage: wrappedLoadPage, - }; + return resp; } return prevResp; }); @@ -343,19 +349,8 @@ export const useTimelineEventsHandler = ({ searchSubscription$.current.unsubscribe(); abortCtrl.current.abort(); await asyncSearch(); - refetch.current = asyncSearch; }, - [ - pageName, - skip, - id, - activePage, - startTracking, - data.search, - dataViewId, - refetchGrid, - wrappedLoadPage, - ] + [pageName, skip, id, activeBatch, startTracking, data.search, dataViewId] ); useEffect(() => { @@ -368,7 +363,6 @@ export const useTimelineEventsHandler = ({ const prevSearchParameters = { defaultIndex: prevRequest?.defaultIndex ?? [], filterQuery: prevRequest?.filterQuery ?? '', - querySize: prevRequest?.pagination?.querySize ?? 0, sort: prevRequest?.sort ?? initSortDefault, timerange: prevRequest?.timerange ?? {}, runtimeMappings: (prevRequest?.runtimeMappings ?? {}) as unknown as RunTimeMappings, @@ -382,16 +376,15 @@ export const useTimelineEventsHandler = ({ const currentSearchParameters = { defaultIndex: indexNames, filterQuery: createFilter(filterQuery), - querySize: limit, sort, - runtimeMappings, + runtimeMappings: runtimeMappings ?? {}, ...timerange, ...deStructureEqlOptions(eqlOptions), }; - const newActivePage = deepEqual(prevSearchParameters, currentSearchParameters) - ? activePage - : 0; + const areSearchParamsSame = deepEqual(prevSearchParameters, currentSearchParameters); + + const newActiveBatch = !areSearchParamsSame ? 0 : activeBatch; /* * optimization to avoid unnecessary network request when a field @@ -410,16 +403,32 @@ export const useTimelineEventsHandler = ({ finalFieldRequest = prevRequest?.fieldRequested ?? []; } + let newPagination = { + /* + * + * fetches data cumulatively for the batches upto the activeBatch + * This is needed because, we want to get incremental data as well for the old batches + * For example, newly requested fields + * + * */ + activePage: activeBatch, + querySize: limit, + }; + + if (newFieldsRequested.length > 0) { + newPagination = { + activePage: 0, + querySize: (newActiveBatch + 1) * limit, + }; + } + const currentRequest = { defaultIndex: indexNames, factoryQueryType: TimelineEventsQueries.all, fieldRequested: finalFieldRequest, fields: finalFieldRequest, filterQuery: createFilter(filterQuery), - pagination: { - activePage: newActivePage, - querySize: limit, - }, + pagination: newPagination, language, runtimeMappings, sort, @@ -427,10 +436,10 @@ export const useTimelineEventsHandler = ({ ...(eqlOptions ? eqlOptions : {}), } as const; - if (activePage !== newActivePage) { - setActivePage(newActivePage); + if (activeBatch !== newActiveBatch) { + setActiveBatch(newActiveBatch); if (id === TimelineId.active) { - activeTimeline.setActivePage(newActivePage); + activeTimeline.setActivePage(newActiveBatch); } } if (!deepEqual(prevRequest, currentRequest)) { @@ -441,7 +450,7 @@ export const useTimelineEventsHandler = ({ }, [ dispatch, indexNames, - activePage, + activeBatch, endDate, eqlOptions, filterQuery, @@ -454,19 +463,6 @@ export const useTimelineEventsHandler = ({ runtimeMappings, ]); - const timelineSearchHandler = useCallback( - async (onNextHandler?: OnNextResponseHandler) => { - if ( - id !== TimelineId.active || - timerangeKind === 'absolute' || - !deepEqual(prevTimelineRequest.current, timelineRequest) - ) { - await timelineSearch(timelineRequest, onNextHandler); - } - }, - [id, timelineRequest, timelineSearch, timerangeKind] - ); - /* cleanup timeline events response when the filters were removed completely to avoid displaying previous query results @@ -486,13 +482,34 @@ export const useTimelineEventsHandler = ({ querySize: 0, }, events: [], - loadPage: wrappedLoadPage, + loadNextBatch, refreshedAt: 0, }); } - }, [filterQuery, id, refetchGrid, wrappedLoadPage]); + }, [filterQuery, id, refetchGrid, loadNextBatch]); - return [loading, timelineResponse, timelineSearchHandler]; + const timelineSearchHandler = useCallback( + async (onNextHandler?: OnNextResponseHandler) => { + if ( + id !== TimelineId.active || + timerangeKind === 'absolute' || + !deepEqual(prevTimelineRequest.current, timelineRequest) + ) { + await timelineSearch(timelineRequest, onNextHandler); + } + }, + [id, timelineRequest, timelineSearch, timerangeKind] + ); + + const finalTimelineLineResponse = useMemo(() => { + return { + ...timelineResponse, + loadNextBatch, + refetch: refetchGrid, + }; + }, [timelineResponse, loadNextBatch, refetchGrid]); + + return [loading, finalTimelineLineResponse, timelineSearchHandler]; }; export const useTimelineEvents = ({ @@ -536,19 +553,32 @@ export const useTimelineEvents = ({ * the combined list of events can be supplied to DataGrid. * * */ + + if (dataLoadingState !== DataLoadingState.loaded) return; + + const { activePage, querySize } = timelineResponse.pageInfo; + setEventsPerPage((prev) => { - const result = [...prev]; - result[timelineResponse.pageInfo.activePage] = timelineResponse.events; + let result = [...prev]; + if (querySize === limit) { + result[activePage] = timelineResponse.events; + } else { + result = [timelineResponse.events]; + } return result; }); - }, [timelineResponse.events, timelineResponse.pageInfo.activePage]); + }, [timelineResponse.events, timelineResponse.pageInfo, dataLoadingState, limit]); useEffect(() => { if (!timelineSearchHandler) return; timelineSearchHandler(); }, [timelineSearchHandler]); - const combinedEvents = useMemo(() => eventsPerPage.flat(), [eventsPerPage]); + const combinedEvents = useMemo( + // exclude undefined values / empty slots + () => eventsPerPage.filter(Boolean).flat(), + [eventsPerPage] + ); const combinedResponse = useMemo( () => ({ diff --git a/x-pack/plugins/timelines/server/search_strategy/timeline/eql/helpers.ts b/x-pack/plugins/timelines/server/search_strategy/timeline/eql/helpers.ts index 645f6daa5727d..81012f0229dcf 100644 --- a/x-pack/plugins/timelines/server/search_strategy/timeline/eql/helpers.ts +++ b/x-pack/plugins/timelines/server/search_strategy/timeline/eql/helpers.ts @@ -20,7 +20,7 @@ import { inspectStringifyObject } from '../../../utils/build_query'; import { formatTimelineData } from '../factory/helpers/format_timeline_data'; export const buildEqlDsl = (options: TimelineEqlRequestOptions): Record => { - if (options.pagination && options.pagination.querySize >= DEFAULT_MAX_TABLE_QUERY_SIZE) { + if (options.pagination && options.pagination.querySize > DEFAULT_MAX_TABLE_QUERY_SIZE) { throw new Error(`No query size above ${DEFAULT_MAX_TABLE_QUERY_SIZE}`); } diff --git a/x-pack/plugins/timelines/server/search_strategy/timeline/factory/events/all/index.ts b/x-pack/plugins/timelines/server/search_strategy/timeline/factory/events/all/index.ts index 2ee2b64162c13..e54daafa3854c 100644 --- a/x-pack/plugins/timelines/server/search_strategy/timeline/factory/events/all/index.ts +++ b/x-pack/plugins/timelines/server/search_strategy/timeline/factory/events/all/index.ts @@ -22,7 +22,7 @@ import { formatTimelineData } from '../../helpers/format_timeline_data'; export const timelineEventsAll: TimelineFactory = { buildDsl: ({ authFilter, ...options }) => { - if (options.pagination && options.pagination.querySize >= DEFAULT_MAX_TABLE_QUERY_SIZE) { + if (options.pagination && options.pagination.querySize > DEFAULT_MAX_TABLE_QUERY_SIZE) { throw new Error(`No query size above ${DEFAULT_MAX_TABLE_QUERY_SIZE}`); } const { fieldRequested, ...queryOptions } = cloneDeep(options); diff --git a/x-pack/plugins/translations/translations/fr-FR.json b/x-pack/plugins/translations/translations/fr-FR.json index bfb08fb7efd10..a64f1accfd1b2 100644 --- a/x-pack/plugins/translations/translations/fr-FR.json +++ b/x-pack/plugins/translations/translations/fr-FR.json @@ -41137,7 +41137,6 @@ "xpack.securitySolution.flyout.user.closeButton": "fermer", "xpack.securitySolution.flyout.user.preview.viewDetailsLabel": "Afficher tous les détails de l'utilisateur", "xpack.securitySolution.footer.autoRefreshActiveDescription": "Actualisation automatique active", - "xpack.securitySolution.footer.autoRefreshActiveTooltip": "Lorsque l'actualisation automatique est activée, la chronologie vous montrera les {numberOfItems} derniers événements correspondant à votre recherche.", "xpack.securitySolution.footer.cancel": "Annuler", "xpack.securitySolution.footer.data": "données", "xpack.securitySolution.footer.events": "Événements", diff --git a/x-pack/plugins/translations/translations/ja-JP.json b/x-pack/plugins/translations/translations/ja-JP.json index 5d2da827ad783..2ca5590d2aeb7 100644 --- a/x-pack/plugins/translations/translations/ja-JP.json +++ b/x-pack/plugins/translations/translations/ja-JP.json @@ -41103,7 +41103,6 @@ "xpack.securitySolution.flyout.user.closeButton": "閉じる", "xpack.securitySolution.flyout.user.preview.viewDetailsLabel": "すべてのユーザー詳細を表示", "xpack.securitySolution.footer.autoRefreshActiveDescription": "自動更新アクション", - "xpack.securitySolution.footer.autoRefreshActiveTooltip": "自動更新が有効な間、タイムラインはクエリーに一致する最新の {numberOfItems} 件のイベントを表示します。", "xpack.securitySolution.footer.cancel": "キャンセル", "xpack.securitySolution.footer.data": "データ", "xpack.securitySolution.footer.events": "イベント", diff --git a/x-pack/plugins/translations/translations/zh-CN.json b/x-pack/plugins/translations/translations/zh-CN.json index 9b470b1a2455e..3663eb1129999 100644 --- a/x-pack/plugins/translations/translations/zh-CN.json +++ b/x-pack/plugins/translations/translations/zh-CN.json @@ -41171,7 +41171,6 @@ "xpack.securitySolution.flyout.user.closeButton": "关闭", "xpack.securitySolution.flyout.user.preview.viewDetailsLabel": "显示全部用户详情", "xpack.securitySolution.footer.autoRefreshActiveDescription": "自动刷新已启用", - "xpack.securitySolution.footer.autoRefreshActiveTooltip": "自动刷新已启用时，时间线将显示匹配查询的最近 {numberOfItems} 个事件。", "xpack.securitySolution.footer.cancel": "取消", "xpack.securitySolution.footer.data": "数据", "xpack.securitySolution.footer.events": "事件",