From b2766ece5e61f9aa6547432e91080cfa33ce29fe Mon Sep 17 00:00:00 2001 From: Georges Chaudy Date: Mon, 22 Jul 2024 15:07:12 +0200 Subject: [PATCH] Resource store sql backend watch (#90480) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Squashed commit of the following: commit 3ae9aea0fdc5eef70e16e0ba26f817ca1883ebc9 Author: Georges Chaudy Date: Wed Jul 17 23:21:07 2024 +0200 Add group and resource to ListOptions commit 638441a656d4098343fd72f1942c5845fb50f536 Author: Georges Chaudy Date: Wed Jul 17 22:25:35 2024 +0200 poll for resources independently commit 4c4c88e2d6b69a589a03fc4ea3b287ef557d67a0 Author: Georges Chaudy Date: Wed Jul 17 17:39:10 2024 +0200 Update pkg/storage/unified/sql/data/resource_history_poll.sql Co-authored-by: Dan Cech commit 67f42dbfcaa6b92056c7e6e25f21e870b7fa08e6 Author: Georges Chaudy Date: Wed Jul 17 16:01:13 2024 +0200 Update pkg/storage/unified/sql/backend.go Co-authored-by: Dan Cech commit e869e829d07d6fce6f79f18679680d32fa9491f5 Author: Georges Chaudy Date: Wed Jul 17 16:01:06 2024 +0200 Update pkg/storage/unified/sql/backend.go Co-authored-by: Dan Cech commit 6e498c0cd7434007649fd8de30a7d17e1c425c42 Author: Georges Chaudy Date: Tue Jul 16 16:47:35 2024 +0200 Add watch working for multiple resources commit 0da775e52148e0012c8cde5a839a43106344c65f Author: Georges Chaudy Date: Tue Jul 16 11:14:55 2024 +0200 indent commit e15d1d060c2d893b2c592c98f4d979670552f735 Author: Georges Chaudy Date: Mon Jul 15 14:41:15 2024 +0200 reindent sql commit a64f88b0ea60cf8ba2f8b96e34ff9e65c3bb1754 Author: Georges Chaudy Date: Fri Jul 12 18:32:13 2024 +0200 Update pkg/storage/unified/sql/backend.go Co-authored-by: Dan Cech commit dd383150bf984db4bce6f622cddc91999dc6e6de Author: Georges Chaudy Date: Wed Jul 10 10:45:56 2024 +0200 lint commit ccf12406a6f65030264be742827db6c859ca3b1b Author: Ryan McKinley Date: Tue Jul 9 15:25:54 2024 -0700 merge main commit 2b4b9f66d35450514b483d84256071f478c4cf01 Merge: df72a61a527 079f0715aa1 Author: Ryan McKinley Date: Tue Jul 9 15:12:51 2024 -0700 merge main commit df72a61a52755d4153f5a061503d385f1a909643 Author: Ryan McKinley Date: Tue Jul 9 15:06:32 2024 -0700 clearnup against smaller upstream commit 12257d9dfee1e57a97e908b0150ca0b8185b03a4 Author: Ryan McKinley Date: Tue Jul 9 15:03:50 2024 -0700 clearnup against smaller upstream commit 07b804457e7ddc4e9dc5d8b02c770270900f597e Author: Georges Chaudy Date: Tue Jul 9 21:41:30 2024 +0200 listAtRevision should always use the token from the RV if present commit 1fc1dd09146bc9aa04d8e6afda25791dabae8be8 Author: Georges Chaudy Date: Tue Jul 9 19:45:44 2024 +0200 Update pkg/storage/unified/sql/backend.go commit 2207ef10ded53120bbd5be526249b6d0306aacf2 Author: Georges Chaudy Date: Tue Jul 9 19:44:35 2024 +0200 Update pkg/storage/unified/sql/queries_test.go commit 36d9099d1a0888518a26fd52ad41ece9cd93d26f Author: Georges Chaudy Date: Tue Jul 9 15:57:55 2024 +0200 lint commit 37e0745d7a633fb520ccd9f58b577421c1d67403 Author: Georges Chaudy Date: Tue Jul 9 12:26:49 2024 +0200 Fix missing rows.Close() commit a6538fa8df6c717f69a7178bf83e2e6af53256b1 Author: Georges Chaudy Date: Mon Jul 8 23:54:33 2024 +0200 add resource_history_list test commit e8214869deaf275f0c7950c4eb808735d653fb7b Author: Georges Chaudy Date: Mon Jul 8 23:51:14 2024 +0200 add resource_history_list test commit f6b32c94bdc6d1c3331c7e11b3dd30e3cda8767b Author: Georges Chaudy Date: Mon Jul 8 23:42:46 2024 +0200 improve tests for pagination commit d305f1baf4665490d719369d89176de7e271dfea Author: Georges Chaudy Date: Mon Jul 8 22:57:11 2024 +0200 Add pagination commit c8f516fc551abc389f1e7e410d05bf419891ea0c Author: Georges Chaudy Date: Mon Jul 8 21:55:25 2024 +0200 pagination first page commit f3c554e668273e8d50154fca899faf685146b578 Author: Georges Chaudy Date: Mon Jul 8 12:16:16 2024 +0200 watch commit 68a0cf5d819d894e72177fc4e807d48e1579fabd Author: Georges Chaudy Date: Mon Jul 8 12:12:09 2024 +0200 remove testdataJSON commit d98da81a428b559ed5c45444a9ebd6416c5aa34b Author: Georges Chaudy Date: Mon Jul 8 11:49:05 2024 +0200 move sqltemplate commit 80bf28521639216520de35cdac7af7109c1bfc9e Author: Georges Chaudy Date: Mon Jul 8 10:49:25 2024 +0200 watch from head commit 396979bcf6d5043f8ab2e96868186651757f5db1 Author: Georges Chaudy Date: Mon Jul 8 10:19:01 2024 +0200 Cleanup commit 26a2f947e898cc695411e7a0ba9821307e86da62 Author: Georges Chaudy Date: Mon Jul 8 10:04:13 2024 +0200 basic list commit 4275a01bc2344a85390ef76f6b3880c96f80dd05 Author: Georges Chaudy Date: Mon Jul 8 00:54:33 2024 +0200 add watch commit 75c25a527dff566ada41cc25497eaa2ddad25da2 Author: Georges Chaudy Date: Mon Jul 8 00:51:40 2024 +0200 add watch commit 774ae238cb03e73306e40bb3109b517c956b77eb Author: Georges Chaudy Date: Sun Jul 7 21:49:44 2024 +0200 Implement SQL store commit 95ffa3486b364c9420d49d7bcef27393854a429f Author: Ryan McKinley Date: Wed Jul 3 20:30:42 2024 -0700 cleanup commit ab591cf4d1bdaa20e2d642dbaeaf58cbfb47c058 Author: Ryan McKinley Date: Wed Jul 3 20:25:12 2024 -0700 cleanup commit aa4901e2060505b5771d2e2e4d451e6997457806 Merge: 36f76100976 40cb4ac30f1 Author: Ryan McKinley Date: Wed Jul 3 20:16:34 2024 -0700 merge upstream commit 40cb4ac30f111f13b3992006735e63c6ce15e4b2 Merge: 8f4b3062d62 fe201b6bb21 Author: Ryan McKinley Date: Wed Jul 3 20:02:04 2024 -0700 Merge remote-tracking branch 'origin/main' into resource-store-bridge commit 8f4b3062d62863a52186873c7547f8ec1e7d9b69 Author: Ryan McKinley Date: Wed Jul 3 19:23:40 2024 -0700 fix lint commit 428b90c14b189971e1ccfacf096f232e4bf5e3b1 Merge: 9e5b88c6dd3 edc1f50c4e0 Author: Ryan McKinley Date: Wed Jul 3 18:46:30 2024 -0700 Merge remote-tracking branch 'origin/main' into resource-store-bridge commit 9e5b88c6dd39c86444fff8ecca86e5a611d95eb6 Author: Ryan McKinley Date: Wed Jul 3 15:56:28 2024 -0700 add field selectors commit 8dd8bc85a6c47bedbf3b8a8998282c05a2cffde8 Merge: 59a142095f5 de06762852d Author: Ryan McKinley Date: Wed Jul 3 15:44:27 2024 -0700 Merge remote-tracking branch 'origin/main' into resource-store-bridge commit 59a142095f5ab4322e5b273aa36ef368dcd2e006 Merge: 53f16521a7a f70f60efd02 Author: Ryan McKinley Date: Wed Jul 3 14:45:35 2024 -0700 Merge remote-tracking branch 'origin/main' into resource-store-bridge commit 53f16521a7af529d0ae590d03b2d9d39342b97ef Author: Ryan McKinley Date: Wed Jul 3 14:45:25 2024 -0700 add doc.go commit 9fa906ab809ca18f055f6b5d332b730dda5130ae Author: Ryan McKinley Date: Wed Jul 3 14:45:14 2024 -0700 add doc.go commit 22d3f163dfb35328f79707b6c824a83f2e3dbaae Merge: d09979b3326 411bab6d447 Author: Ryan McKinley Date: Wed Jul 3 13:55:27 2024 -0700 Merge remote-tracking branch 'origin/main' into resource-store-bridge commit d09979b3326633c87dbf00fe3f454500ec70458e Author: Ryan McKinley Date: Wed Jul 3 13:55:12 2024 -0700 check for deleted commit ddcbc753d3cf839da9c32d5c6a145f1d5d620305 Author: Ryan McKinley Date: Wed Jul 3 13:38:16 2024 -0700 fix tabs to spaces commit 2a4fedb6ee7c3e5518860453040452fafae7a529 Author: Ryan McKinley Date: Wed Jul 3 10:58:43 2024 -0700 remove sort parsing commit e728b12ec09ca697006543a036b9a37de46d5d39 Author: Ryan McKinley Date: Wed Jul 3 10:56:22 2024 -0700 remove watch tests -- key func complications commit dc163d10cc198c51df2a96f29855a1468ec09ee8 Author: Ryan McKinley Date: Wed Jul 3 08:59:09 2024 -0700 add resource store abstraction commit 36f76100976426b2c709a3f22db62a6ce6b692b9 Author: Ryan McKinley Date: Tue Jul 2 21:37:01 2024 -0700 using basic storage engine commit 26ef342b7e338d9494c2fd5ccb6f85a77322ae26 Author: Ryan McKinley Date: Tue Jul 2 21:24:53 2024 -0700 renamed to basic commit ddb5f125f04d3eb05c18bad96480acbff49081c6 Author: Ryan McKinley Date: Tue Jul 2 18:21:59 2024 -0700 add sqlobj basic implementation commit 6a08224725b04d907773c107799463ce01bcd17c Merge: 04d57a99e5b 5b6edc96d91 Author: Ryan McKinley Date: Tue Jul 2 17:47:36 2024 -0700 Merge remote-tracking branch 'origin/main' into resource-store commit 04d57a99e5bf4463a8b4a414ba9c1d6cf5cbe7ee Author: Ryan McKinley Date: Tue Jul 2 14:47:02 2024 -0700 merge main commit 33917141f0a50b5c5ef41dbc3ab1cd7572b4dbc6 Merge: 924c42f7077 2a1b620c383 Author: Ryan McKinley Date: Tue Jul 2 14:45:45 2024 -0700 Merge remote-tracking branch 'origin/main' into resource-store commit 924c42f7077ad8c7d1acdf88e292bbfe45cd3e69 Author: Ryan McKinley Date: Tue Jul 2 14:37:35 2024 -0700 make list preparelist commit 0db0e132487620d65bb8ff4fd912aeacf323542a Author: Ryan McKinley Date: Tue Jul 2 14:37:28 2024 -0700 make list preparelist commit a3d9f8aab23a3ef9e2fac47faeea0c892545cb78 Author: Ryan McKinley Date: Tue Jul 2 14:08:42 2024 -0700 remove eventID commit 8c1f60aa8b7de8acfdf228b655064e711f7fa80a Author: Ryan McKinley Date: Tue Jul 2 14:08:34 2024 -0700 remove eventID commit abacd9aef24789bdd65f527d53a62865c3221b02 Author: Ryan McKinley Date: Tue Jul 2 07:36:07 2024 -0700 split off blob store services commit ddae2b49698fddfd7fd2efb713da7089077ea069 Author: Ryan McKinley Date: Tue Jul 2 07:23:53 2024 -0700 rename search service to index service commit a984a4e4f909cdbf8db49ae56e56cfbfb53979b8 Author: Ryan McKinley Date: Mon Jul 1 17:03:23 2024 -0700 more lint commit b25e735c0223bcf2adf3912c5449a841d9906cbd Author: Ryan McKinley Date: Mon Jul 1 16:04:53 2024 -0700 frontend refactors commit 1fb97cf4170ec96b603e11158b3d033adbd8e59c Merge: fe8a05c648b c3b5cabb143 Author: Ryan McKinley Date: Mon Jul 1 15:02:34 2024 -0700 minor lint fixes commit fe8a05c648bdda7e41578ea9d497b9ebfc80dfca Author: Ryan McKinley Date: Mon Jul 1 08:12:28 2024 -0700 minor lint fixes commit 9f2aae1c489ac7a8aa14b4b8efd697cf6764b79d Merge: ece478e12de 86466aec61f Author: Ryan McKinley Date: Mon Jul 1 08:03:18 2024 -0700 Merge remote-tracking branch 'origin/main' into resource-store commit ece478e12de7709031f42b0db430f2cee21b98e7 Merge: f472e95f92f 55ba32bda75 Author: Ryan McKinley Date: Mon Jul 1 01:41:29 2024 -0700 Merge remote-tracking branch 'origin/main' into resource-store commit f472e95f92f739a3be6a002c210a83de8ac3266b Author: Andres Martinez Gotor Date: Mon Jul 1 09:31:05 2024 +0200 Adapt docs/sources/developers/plugins/plugin.schema.json codeowners (#89832) commit 8674d705eab8062614fe886a312a11c79f6494a4 Author: Sebastian Hiebl Date: Sat Jun 29 01:13:34 2024 +0200 TimeSeries: Expose option for "Bar width factor" (#77338) Co-authored-by: Leon Sorokin commit 52d476db05a9b308b7e79e78bae84ccb8bb76541 Author: Leon Sorokin Date: Thu Jun 27 18:58:49 2024 -0500 Table: Don't error if first frame is field-less (empty response) (#89867) commit cc5b4d6a94aa5e894c221754db543c1ca8102ed9 Author: Simon Podlipsky Date: Fri Jun 28 00:53:43 2024 +0300 Dockerfile: Fix go deps (#89857) Add missing deps commit 7c86705b9b3b4a1a682518cafad35bfb66558af7 Author: Adela Almasan <88068998+adela-almasan@users.noreply.github.com> Date: Thu Jun 27 15:49:20 2024 -0600 VizTooltip: Hide tooltip options from when tooltip is hidden (#89864) commit aa9299c4d705735b6f279cb2ad56d4ab8e0eed47 Author: Isabel Matwawana <76437239+imatwawana@users.noreply.github.com> Date: Thu Jun 27 17:16:57 2024 -0400 Docs: Add tooltip options to histogram v11.2 (#89863) Co-authored-by: Adela Almasan <88068998+adela-almasan@users.noreply.github.com> commit f90168f5002912fb255a2cccd3938600fd50b55a Author: Nathan Marrs Date: Thu Jun 27 14:43:10 2024 -0600 Chore: Lint json files in .github directory (#89861) commit 0f65c06cfe6d1757e99467cbe996ebfa59d192fa Author: Nathan Marrs Date: Thu Jun 27 14:22:15 2024 -0600 Chore: Update DataViz github automation (#89852) commit 058538287ff854d3c5cd37f08efcd91cf2550146 Author: Lucy Chen <140550297+lucychen-grafana@users.noreply.github.com> Date: Thu Jun 27 15:36:23 2024 -0400 ShareModal: Export options (JSON) (#87082) * Adding new export button * Create Export as JSON drawer * update scene drawer and add css * update css * Update ExportAsJson to be regular react component * add tests to export menu and button * add tests * prettier and lint * fix translations * update translation * Apply suggestions from code review Co-authored-by: Juan Cabanas * delete extra file * Update to use SceneObject * add spinner * Rename ExportAsJSON.tsx to ExportAsJson.tsx * update i18n * Upate texts * small fixes from code review * add space * i18n * fix build issues * changes from review feedback * update test * update test --------- Co-authored-by: Juan Cabanas commit 79092ebc6a8468825fcfa7fbe21af39cb9267809 Author: Juan Cabanas Date: Thu Jun 27 14:30:33 2024 -0300 ShareDrawer: Schedule Report (#89802) commit d842178c6a7058dd226ce9da3a5954f242fcc411 Author: linoman <2051016+linoman@users.noreply.github.com> Date: Thu Jun 27 18:29:16 2024 +0200 SAML: Add Azure AD configuration for SAML integration (#89767) * Add Azure AD configuration for SAML integration commit 920756f7977311a4d3277b7826c29c851dab5805 Author: Ashley Harrison Date: Thu Jun 27 16:43:43 2024 +0100 Storybook: Add basic e2e verification test (#89779) * add very basic tests to verify storybook builds correctly * add storybook step to drone * reorder steps * drone tweaks * don't need host since it's set in env * don't need to wait * format build.star and readd wait * install netcat in CI * do a yarn install here to get correct bindings * refactoring to hopefully work better in CI * add wait-on * add verbose logging * localhost? * more logging * specify storybook host * ... * back to grafana-server * does this work? :thinking: * run storybook e2e test after rgm-package so the backround process is running for less time * split into separate step * format commit a760dad1e3e0613f61282b30609829bf0a83dbc1 Author: Ivan Ortega Alba Date: Thu Jun 27 17:26:38 2024 +0200 VizPanelManager: Adapt color palete after plugin change (#89790) * VizPanelManager: Adapt color palete after plugin change * Upgrade scenes * Remove broken code * Upgrade scenes version * linter commit c8ce20a807819626979fdf3facce1db53ec511d9 Author: Ida Štambuk Date: Thu Jun 27 17:10:28 2024 +0200 Cloudwatch: Round up endTime in GetMetricData to next minute (#89341) * Add cloudWatchRoundUpEndTime feature toggle commit 17ebc9614b019ebff86fbdaa9ae86744cc31c6ca Author: Kristina Date: Thu Jun 27 09:46:49 2024 -0500 Explore / Query Library : Show user data (#88258) * Add basic button for adding a query template * Add hook to create a template * Handle notifications * Add tags to invalidate cache * Generate translations * Updates types * Add tests * Simplify code * Add user to type * Add a better default title * bring in piotrs pr and try to add user data * Move out of metadata (reserved in k8s) and make new values exportable * Show user data * Fix bad merge * WIP * Add annotation data to FE * add (failing) test * Fix types and test * Cleanup * Enhance user data and send to component for display * Fix type * Fix expected values * fix betterer * Fix test * Remove user lookup * testing slug usage for api * Revert "testing slug usage for api" This reverts commit cc4556c3b781488cd88a586df834ff337fbf7fd5. * change types, display userid if login isnt returned * Simply display whatever is in property * skip test on removed logic * Try waiting for query to finish before eval * Revert "Try waiting for query to finish before eval" This reverts commit 6220cabd17d71004074ae9807be97e1892dde177. * Handle attribute not existing when storage type is file --------- Co-authored-by: Piotr Jamroz commit 91895e4835f657f6c898482bc26cd9f53201193b Author: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu Jun 27 17:43:59 2024 +0300 Chore(deps): Bump github.com/alicebob/miniredis/v2 from 2.30.1 to 2.33.0 (#88588) Bumps [github.com/alicebob/miniredis/v2](https://github.com/alicebob/miniredis) from 2.30.1 to 2.33.0. - [Release notes](https://github.com/alicebob/miniredis/releases) - [Changelog](https://github.com/alicebob/miniredis/blob/master/CHANGELOG.md) - [Commits](https://github.com/alicebob/miniredis/compare/v2.30.1...v2.33.0) --- updated-dependencies: - dependency-name: github.com/alicebob/miniredis/v2 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> commit ed6c948db078bc81d12691d03357f29306358112 Author: kay delaney <45561153+kaydelaney@users.noreply.github.com> Date: Thu Jun 27 14:17:21 2024 +0100 Dashboards: Change datemath parse logic so parsed dates always take timezone into account (#88182) * Dashboards: Change datemath parse logic so parsed dates always take timezone into account commit 523d99941450f46a9fd927ddf90a6d38335ab8a3 Author: Sonia Aguilar <33540275+soniaAguilarPeiron@users.noreply.github.com> Date: Thu Jun 27 14:00:42 2024 +0200 Alerting: Use runtime data source for getting events from alert state history in the bar chart (#89307) * Use runtime data source for getting events from alert state history in the bar chart * extract translations * refactor * More refactor * Update events limit * Add info icon with tooltip info for label querying filter * Add translations * Create new useRuleHistoryRecords hook skipping extraction of common labels as they are not used * Fix test * update limit value for the events in the api to 5000 * Use state for rows key * remove React import * Address review comments * Address review comments * run prettier * Remove duplicated handlers commit c3337c39b18fde14fd27bdd03127430bcefcd562 Author: Laura Benz <48948963+L-M-K-B@users.noreply.github.com> Date: Thu Jun 27 14:00:12 2024 +0200 RestoreDashboards: Add search (#89692) * feat: add search * feat: run i18n extraction * refactor: styling issue for smaller screen sizes commit 06084f0ed15db44f6b03e1fc72e9f24ecb441bc0 Author: Alexander Zobnin Date: Thu Jun 27 13:57:06 2024 +0200 Zanzana: Initial schema loading (#89492) * Zanzana: Dummy schema loading * Load authorzation model for client --------- Co-authored-by: Karl Persson commit ddea4ba8b2540c46da44084430103001a8473e7a Author: ismail simsek Date: Thu Jun 27 13:33:35 2024 +0200 Chore: Remove CancelablePromise (#89564) * Remove CancelablePromise.ts as it wasn't used in anywhere * Move cancelable-promise to the place where it is used commit 177747ecdaeefdbd3f31b32dc66cc88d2294d547 Author: Jack Westbrook Date: Thu Jun 27 12:42:42 2024 +0200 Grafana/data: Remove barrel files part 1 (#89556) * chore(grafana-data): remove dataframe barrel file and update all imports and exports * chore(grafana-data): remove field barrel file and update imports n exports * chore(grafana-data): remove utils barrel file and update imports and exports * chore(grafana-data): remove tranformations barrel file and update imports n exports * chore(grafana-data): remove imports from root index.ts * chore(grafana-data): remove valueFormats barrel file and update imports n exports * chore(grafana-data): don't import from root barrel file commit c3b16639bc9f82c73f05a352e3dd66aebc7f1a19 Author: Ashley Harrison Date: Thu Jun 27 11:15:15 2024 +0100 Keybinds E2E: simplify keybinds e2e even further (#89819) simplify keybinds e2e even further commit 8b7a891db4e9b47ee98afaab88c3979aca034f89 Author: AHeinlein Date: Thu Jun 27 12:07:30 2024 +0200 Docs: Update LDAP doc for debug view (#89522) Update LDAP doc LDAP debug view does not work with single bind since there is no way to bind to the LDAP server and perform the search. commit cb31a1b6ff42b966c4afe67b492d1d10a9463411 Author: Tom Ratcliffe Date: Thu Jun 27 10:33:40 2024 +0100 Alerting: Update warning message for Telegram parse_mode and default to empty value (#89630) commit 3ef048cdd74e30e258ec2a3842a7b5e24453edd7 Author: Sergej-Vlasov <37613182+Sergej-Vlasov@users.noreply.github.com> Date: Thu Jun 27 12:06:07 2024 +0300 DashboardScenes: Disable flaky new-datasource-variable e2e test (#89817) disable flaky new-datasource-variable e2e test commit e09c1c7515c3007d9fe6651cd6b3923970ddfe70 Author: brendamuir <100768211+brendamuir@users.noreply.github.com> Date: Thu Jun 27 10:10:34 2024 +0200 Alerting docs: adds silence RBAC 11.1 (#89176) * Alerting docs: adds silence RBAC 11.1 * ran prettier * Improve docs with new rule-specific silence RBAC information * Apply suggestions from code review Co-authored-by: Jack Baldry * Apply suggestions from code review Co-authored-by: Jack Baldry * prettier --------- Co-authored-by: Matt Jacobson Co-authored-by: Jack Baldry commit 1aeceb9d9adac46a84bd2316ebe199117079097b Author: Alexander Akhmetov Date: Thu Jun 27 09:45:15 2024 +0200 Alerting: Add setting for maximum allowed rule evaluation results (#89468) * Alerting: Add setting for maximum allowed rule evaluation results Added a new configuration setting `quota.alerting_rule_evaluation_results` to set the maximum number of alert rule evaluation results per rule. If the limit is exceeded, the evaluation will result in an error. commit e96b1c0b4291fc50a111a07a0465e62b7223f91a Author: Georges Chaudy Date: Mon Jul 1 10:32:37 2024 +0200 Add unified-next-grpc (#89891) commit afe0848d166af2feb8a300385879b535600f3691 Author: Ryan McKinley Date: Thu Jun 27 13:11:10 2024 +0300 history UI commit 2fab168fb1f26bf10c9a1791bac6a65c59e0bcf3 Author: Ryan McKinley Date: Thu Jun 27 12:08:46 2024 +0300 history UI commit 1043efe3db962ecdb031ec157ddb80f30fef063d Author: Ryan McKinley Date: Thu Jun 27 11:26:42 2024 +0300 rename to legacy commit d452e53ce39be39227c4517510f2158bce0527f4 Author: Ryan McKinley Date: Thu Jun 27 10:53:08 2024 +0300 avoid key from context commit 19496ceed53b8f59c06cc94a0bcbf6c3c18e9707 Merge: b71329a8c33 be1414b36b9 Author: Ryan McKinley Date: Thu Jun 27 10:19:58 2024 +0300 Merge remote-tracking branch 'origin/main' into resource-store commit b71329a8c331062beafcd49c30f3b872af2c98f6 Author: Ryan McKinley Date: Thu Jun 27 10:09:59 2024 +0300 history in resource package commit 37c6b56e4ea5e09d154f50a31ea23f9ab4cf7d64 Merge: 80fa46b2250 9056ff73f6f Author: Ryan McKinley Date: Thu Jun 27 08:49:46 2024 +0300 Merge remote-tracking branch 'origin/main' into resource-store commit 80fa46b22505a9b6a30203be8afbf48c5bcccde7 Author: Ryan McKinley Date: Thu Jun 27 08:49:24 2024 +0300 history from SQL query commit 4d49cc31185e7085a289bba7f3263db47033ddd5 Merge: 999cd506c0e ed13959e336 Author: Ryan McKinley Date: Thu Jun 27 07:35:23 2024 +0300 Merge remote-tracking branch 'origin/main' into resource-store commit 999cd506c0ecc8997bee1eb10026ea9ad312976e Author: Ryan McKinley Date: Thu Jun 27 01:20:29 2024 +0300 now with dashboard history commit 7345ece8ef3df2632aa65ad5ff9a58931fc52264 Author: Ryan McKinley Date: Thu Jun 27 00:10:30 2024 +0300 with dashboard service commit 86a706433495739ae21ba62a6c94248338db8be0 Merge: f68390bebf9 ba4f2713c53 Author: Ryan McKinley Date: Wed Jun 26 16:04:23 2024 +0300 Merge remote-tracking branch 'origin/main' into resource-store commit f68390bebf991775b28d221ad497cfef922f3818 Author: Ryan McKinley Date: Wed Jun 26 16:04:08 2024 +0300 not rest.Storage commit cfc192a2b58c33583223a54fad896c40360a953d Author: Ryan McKinley Date: Wed Jun 26 12:53:57 2024 +0300 more utils commit 197a837e1ee8f8f00466ea75b331b0acce49a6fe Merge: 44a134f72b5 4651506319f Author: Ryan McKinley Date: Wed Jun 26 08:48:10 2024 +0300 merge main commit 44a134f72b56f0e338161723f5a1d0cd31376b02 Merge: bac63d0edb6 f5d9c247d9a Author: Ryan McKinley Date: Mon Jun 24 20:55:15 2024 +0300 Merge remote-tracking branch 'origin/main' into resource-store commit bac63d0edb60e1facd7ced0fb9f5e04dad0100a8 Author: Ryan McKinley Date: Fri Jun 21 17:39:19 2024 +0300 remove old comment commit 37edc450a7a03e25752befbd4e260cc6f68a7e66 Merge: af38567ea54 b62f8c0f19b Author: Ryan McKinley Date: Fri Jun 21 17:38:20 2024 +0300 Merge remote-tracking branch 'origin/main' into resource-store commit af38567ea5429f5e8846488f91793038faeeb200 Author: Ryan McKinley Date: Fri Jun 21 14:55:40 2024 +0300 remove unimplemented functions commit ec925bab80a8852051f532abe8a5ac0b24c18336 Author: Ryan McKinley Date: Fri Jun 21 14:53:23 2024 +0300 has a watch bridge... but not working commit b771d38d7892b63819d7a465e702345d1b454d76 Author: Ryan McKinley Date: Fri Jun 21 11:13:35 2024 +0300 with build changes commit 37c2dade15718026171159ee1b5fd40ef79a92e0 Author: Ryan McKinley Date: Fri Jun 21 09:58:38 2024 +0300 writing to SQL commit bb7d6d44cd4e8ece6060f8b101f43335c1c5fb26 Author: Ryan McKinley Date: Fri Jun 21 09:24:44 2024 +0300 with setup flags commit c2c94496297bc7ff3db9e0dcb943610d90cf2565 Author: Ryan McKinley Date: Fri Jun 21 09:09:01 2024 +0300 working commit d67f83a1f36745bcdbfc4eec50218c4f23881109 Merge: b07c4e42106 61a102274d2 Author: Ryan McKinley Date: Fri Jun 21 07:49:24 2024 +0300 Merge remote-tracking branch 'origin/main' into resource-store commit b07c4e421067cda7784a754e14b69fd213c4c6e0 Merge: 09741a3f49e b0759262027 Author: Ryan McKinley Date: Fri Jun 21 00:52:10 2024 +0300 Merge remote-tracking branch 'origin/main' into resource-store commit 09741a3f49efdf08945aee688f5c1f36e39708ca Author: Ryan McKinley Date: Thu Jun 20 23:05:13 2024 +0300 merge main commit 8a6d2e9b6e31dcb13a158723a97d2490876a3beb Author: Ryan McKinley Date: Thu Jun 20 22:50:08 2024 +0300 merge main commit 63ebf7b12eed775f67faf1d0104115a80098b6bb Merge: 75d5ad465e0 5e95c1bdf86 Author: Ryan McKinley Date: Thu Jun 20 22:34:38 2024 +0300 merge main commit 75d5ad465e08b6c03ccd23dcccc46353e563c04e Merge: 83df3bdec8b d988f5c3b06 Author: Ryan McKinley Date: Thu Jun 20 22:26:33 2024 +0300 merge main commit 83df3bdec8b84b7f97387101e73379eae06d17e7 Merge: a115bb62146 27e800768ed Author: Ryan McKinley Date: Thu Jun 20 18:07:48 2024 +0300 Merge remote-tracking branch 'origin/main' into resource-store commit a115bb62146e20cce2e1f841ca10e57c796ba618 Merge: c17140f263e 13e069b5aff Author: Ryan McKinley Date: Thu Jun 20 17:05:36 2024 +0300 merge main commit c17140f263e2e7467f31528853c592f97e045550 Merge: 3a6e7dc3f0f 5d328983a1f Author: Ryan McKinley Date: Wed Jun 19 22:59:26 2024 +0300 Merge remote-tracking branch 'origin/main' into resource-store commit 3a6e7dc3f0f166eb9556f4ca70ebeed26a689957 Author: Ryan McKinley Date: Wed Jun 19 22:09:33 2024 +0300 go mod tidy commit 040f3920186da3fd963221251d88bbc669d30531 Author: Ryan McKinley Date: Wed Jun 19 22:01:10 2024 +0300 Storage: Add cdk blob support to ResourceStore (#89408) commit 3e85f87db69aa978ca398175fbd0924bdd0837fe Author: Ryan McKinley Date: Wed Jun 19 15:15:37 2024 +0300 Storage(NG): Move message and operation from protobuf to annotations (#89359) commit 0b29ca5eac3e2b16f85d1c351dd6efc522192e8f Merge: 0717e39ec2c b3907ca5ec3 Author: Ryan McKinley Date: Wed Jun 19 14:57:21 2024 +0300 Merge remote-tracking branch 'origin/main' into resource-store commit 0717e39ec2c1fad001f34695add512214cf762da Author: Ryan McKinley Date: Tue Jun 18 22:27:16 2024 +0300 K8s: Rename origin.key to origin.hash (#89337) commit 4b4afa1fa76038e1fae053b4536192e4244bc6d6 Author: Ryan McKinley Date: Tue Jun 18 21:19:21 2024 +0300 more cleanup commit 9f6709c167a6218d2eeef7539c40d6c0fb5e7b65 Merge: 7e171702f45 b0c043de5fe Author: Ryan McKinley Date: Tue Jun 18 21:17:30 2024 +0300 Merge remote-tracking branch 'origin/main' into resource-store commit 7e171702f45308a4a685e3649bd49772d5d98bed Merge: 760848d2037 890482052a4 Author: Ryan McKinley Date: Tue Jun 18 16:08:30 2024 +0300 Merge remote-tracking branch 'origin/main' into resource-store commit 760848d2037cd8b57247b5933c258265115c9279 Author: Ryan McKinley Date: Tue Jun 18 16:07:55 2024 +0300 Storage (resource): Mutate object in storage, not k8s (#89331) commit 5e4fa5d735145407a657fe0fabdefdb75dca7057 Merge: 4cde5bd59f4 ea7f6ea32f5 Author: Ryan McKinley Date: Tue Jun 18 08:44:04 2024 +0300 Merge remote-tracking branch 'origin/main' into resource-store commit 4cde5bd59f414ae1d8e701e187f538a3208d188e Merge: 549ecfe053e 5f83fdef2c3 Author: Ryan McKinley Date: Tue Jun 18 07:27:05 2024 +0300 Merge remote-tracking branch 'origin/main' into resource-store commit 549ecfe053ec3f2e72542037b605cd89e02fd0cd Author: Ryan McKinley Date: Sat Jun 15 00:49:06 2024 +0300 lint fix commit 1a5de75c34171b0a5e46a6a265f8ded3077b303a Author: Ryan McKinley Date: Sat Jun 15 00:42:19 2024 +0300 lint fix commit 31a05d566628ce0df64109160902d182a980671e Author: Ryan McKinley Date: Sat Jun 15 00:26:45 2024 +0300 hack hack hack... but working from kubectl commit 2f64556cf16be328f42e541bc50709b1882db80d Author: Ryan McKinley Date: Fri Jun 14 23:34:04 2024 +0300 almost... but stuck on requester commit c1b786c71830560ed20b47bd86d38b932a6f72d2 Author: Ryan McKinley Date: Fri Jun 14 23:10:59 2024 +0300 cleanup commit d8abf8d851fec91ce896437a7ebe29fcbd89229f Author: Ryan McKinley Date: Fri Jun 14 22:53:10 2024 +0300 entity bridge commit d5f3038421c06ca4b9a5b900f7e3174b5f7b3dd2 Merge: df9baddf53a 8491e02cafa Author: Ryan McKinley Date: Fri Jun 14 21:48:25 2024 +0300 merge main commit df9baddf53a3393fb0aa3905dc6957d212545ed3 Author: Ryan McKinley Date: Fri Jun 14 21:44:33 2024 +0300 move test files commit 617dd1b40ee5195d8a9b577707d8f4dca2ac751e Author: Ryan McKinley Date: Fri Jun 14 21:30:41 2024 +0300 add basic list commit 8108e90fe2c63830fd0ba388db8a073b701fb48c Merge: ef3a6011090 15c81b2a8b3 Author: Ryan McKinley Date: Fri Jun 14 17:42:18 2024 +0300 Merge remote-tracking branch 'origin/main' into resource-store commit ef3a60110901dae082532eaae3b0ff40104d2631 Author: Ryan McKinley Date: Fri Jun 14 17:42:15 2024 +0300 update the event type messages commit 55d1e8ad1d8cc3481253d8671eea3665a4fe2acf Author: Ryan McKinley Date: Fri Jun 14 17:41:16 2024 +0300 update the event type messages commit a01f8dd0994f02b1253f2c85bd99266c277f2c55 Author: Ryan McKinley Date: Fri Jun 14 16:52:17 2024 +0300 remove rv from key and update watch signature commit e78a5122ceea32229795f02b3ab7eb5e56a84763 Merge: 504d84b8b2b 90c3d74955d Author: Ryan McKinley Date: Fri Jun 14 16:19:53 2024 +0300 Merge remote-tracking branch 'origin/main' into resource-store commit 504d84b8b2b33632f3cce648ba8b463c0ac68f6d Author: Ryan McKinley Date: Fri Jun 14 15:21:28 2024 +0300 with storage wrapper commit 4fa60b6546c39d7d01ca149ba82141e6b5317df2 Author: Ryan McKinley Date: Fri Jun 14 15:21:17 2024 +0300 with storage wrapper commit d97d59ab3866f89b58f8117c83cf62f27fd09ad2 Merge: f66768c67d8 2dd44e2f58d Author: Ryan McKinley Date: Fri Jun 14 14:55:52 2024 +0300 Merge remote-tracking branch 'origin/main' into resource-store commit f66768c67d895cc681b0d182ad672eba189c2b83 Author: Ryan McKinley Date: Fri Jun 14 14:24:36 2024 +0300 now with a base server implementation commit 4a41f7d0ddc199aac52e94dbc03d044699af7eb6 Author: Ryan McKinley Date: Fri Jun 14 14:24:26 2024 +0300 now with a base server implementation commit 68003738fdee0fcc668abe3a46b58b48bbd25776 Author: Ryan McKinley Date: Fri Jun 14 11:28:15 2024 +0300 use partial object metadata commit fc8208998725c33db54f37bbee97ad1d09618079 Merge: 9478befbcd8 dd3c3b5857d Author: Ryan McKinley Date: Fri Jun 14 10:57:16 2024 +0300 Merge remote-tracking branch 'origin/main' into resource-store commit 9478befbcd8b7ba0a612d93d298b3697c441cacb Merge: 19481990d11 2173429ce2b Author: Ryan McKinley Date: Fri Jun 14 09:08:14 2024 +0300 merge main commit 19481990d11d03b1efe4761f7beae913c8df7a68 Author: Ryan McKinley Date: Fri Jun 14 09:06:44 2024 +0300 merge main commit 29714d87fe4a9d319a7f93652f949b463b10b883 Author: Ryan McKinley Date: Fri Jun 14 02:12:35 2024 +0300 use a path hash for the blob table commit 382d5d4e01f8deffb7678fedb25e43d9c410718a Author: Ryan McKinley Date: Fri Jun 14 02:04:21 2024 +0300 add fs based store commit 15b958b2d1d20a35d82a1bcd5a527b0acd33d29b Merge: a03a978eb48 e2f65dda0cf Author: Ryan McKinley Date: Thu Jun 13 22:56:34 2024 +0300 Merge remote-tracking branch 'origin/main' into resource-store commit a03a978eb48d91457d7a4b8fbfe4ea8288e2d19a Author: Ryan McKinley Date: Thu Jun 13 22:37:29 2024 +0300 maybe happier commit 79aed54c2c22225ca78010f365369568e74d33ae Author: Ryan McKinley Date: Thu Jun 13 22:12:55 2024 +0300 more body prep commit 1540c98fe1145313b06b2dc7760088a612813469 Merge: f54af3d3718 9d3a4e236d2 Author: Ryan McKinley Date: Thu Jun 13 21:19:51 2024 +0300 Merge remote-tracking branch 'origin/main' into resource-store commit f54af3d3718de8c42089854d196ae8d3645504bb Author: Ryan McKinley Date: Thu Jun 13 20:12:53 2024 +0300 sum cleanup commit a0f682b315cd43ed82f1ec8b98c95df26127fb3b Merge: 2197a0e5e8d eb535e163d7 Author: Ryan McKinley Date: Thu Jun 13 20:05:01 2024 +0300 merge main commit 2197a0e5e8d3cbf13515662bcebe9f30bf4f2c22 Author: Ryan McKinley Date: Thu Jun 13 16:02:06 2024 +0300 update workspace commit b696b6a32a934d705f75f15f8edca5f361c49e8b Author: Ryan McKinley Date: Thu Jun 13 15:53:06 2024 +0300 update go.mod commit 8f44e1a3498628754868c740cb8214e6e2796497 Author: Ryan McKinley Date: Thu Jun 13 15:49:41 2024 +0300 lint fixes commit c1798320d2b39c2fc170978463765c76340b28a6 Merge: b44333730f0 afcb5a855c2 Author: Ryan McKinley Date: Thu Jun 13 14:27:42 2024 +0300 Merge remote-tracking branch 'origin/main' into resource-store commit b44333730f081d4fa1bc5fe3a4952e63538f55e6 Author: Ryan McKinley Date: Thu Jun 13 14:26:57 2024 +0300 now with basic test commit f481f11d3a4e59c276a2699ceb9c282e17ea1600 Author: Ryan McKinley Date: Thu Jun 13 09:11:10 2024 +0300 change tree structure commit e7c15b0e9673a56ab4a497eac82b84e88355badf Author: Ryan McKinley Date: Thu Jun 13 08:42:00 2024 +0300 merged main and moved folder structure commit 09e8605f14ed670bcdda19df98ac3da006951203 Merge: 40829d645c5 99d8025829e Author: Ryan McKinley Date: Thu Jun 13 07:14:54 2024 +0300 Merge remote-tracking branch 'origin/main' into resource-store commit 40829d645c566421d5edb86a58a4c21fc54698a2 Author: Ryan McKinley Date: Thu Jun 13 07:08:54 2024 +0300 include client wrapper commit 1857690bd08e134e4c46f24345266fef04bad0b6 Merge: c9c18717f55 ed400f0bbfe Author: Ryan McKinley Date: Wed Jun 12 22:02:53 2024 +0300 Merge remote-tracking branch 'origin/main' into resource-store commit c9c18717f5528cc6b8d1f6e7376037ca510d72e0 Merge: ad79d44db8a 3ae95a6eb11 Author: Ryan McKinley Date: Wed Jun 12 15:26:12 2024 +0300 packaging commit ad79d44db8abbcfcbf919e376843a7bf94791dcd Author: Ryan McKinley Date: Wed Jun 12 14:44:21 2024 +0300 adding package commit f8a2a83d599a2e4f5e43e328c9c7a1a174cbc34d Author: Ryan McKinley Date: Wed Jun 12 14:44:18 2024 +0300 adding package commit 7b6e4d5da4db3d037c743a6b110872f8d8d8fe7e Merge: 16471fa057e 58f7032b398 Author: Ryan McKinley Date: Wed Jun 12 14:21:41 2024 +0300 Merge remote-tracking branch 'origin/main' into resource-store commit 16471fa057e8c0c6b1cfe665429a4bbbbdd62874 Author: Ryan McKinley Date: Wed Jun 12 13:02:06 2024 +0300 now with event setup commit d83aa7f8652be78fa52e056c2e2b65884f52e4b3 Author: Ryan McKinley Date: Wed Jun 12 02:44:49 2024 +0300 is_current commit df148ca9eaa42c6ddaa7662170fb6989a7242577 Author: Ryan McKinley Date: Wed Jun 12 02:21:05 2024 +0300 use reflection commit 0118c7666a5fc54f49619728e3778d9417459cbb Author: Ryan McKinley Date: Wed Jun 12 02:20:47 2024 +0300 use reflection commit 5c91e52922a41a22af2ff449285f24da419bc50e Author: Ryan McKinley Date: Wed Jun 12 02:15:05 2024 +0300 get metadata with reflection commit af86293d87633237781b07480793f0e826e6aa44 Merge: 4dc3d0ee7f5 465efbf8020 Author: Ryan McKinley Date: Wed Jun 12 00:48:13 2024 +0300 Merge remote-tracking branch 'origin/main' into resource-store commit 4dc3d0ee7f58090dbf962f05c8ecce53b0f48922 Author: Ryan McKinley Date: Wed Jun 12 00:34:40 2024 +0300 update commit 835879aa7a48e012a84d39e8b2383bfd0e2f0bd9 Author: Ryan McKinley Date: Wed Jun 12 00:29:52 2024 +0300 revert hash key issues commit ac39953d0ee63703f8407deba1d4fad8fac5dee3 Author: Ryan McKinley Date: Tue Jun 11 23:01:40 2024 +0300 basic queries * remove old file --- pkg/storage/unified/sql/backend.go | 126 +++++++++++++----- pkg/storage/unified/sql/backend_test.go | 45 ++++++- .../sql/data/resource_history_poll.sql | 6 +- .../sql/data/resource_version_list.sql | 6 + pkg/storage/unified/sql/queries.go | 18 +++ 5 files changed, 164 insertions(+), 37 deletions(-) create mode 100644 pkg/storage/unified/sql/data/resource_version_list.sql diff --git a/pkg/storage/unified/sql/backend.go b/pkg/storage/unified/sql/backend.go index 99d3c656371..d2e62e5d400 100644 --- a/pkg/storage/unified/sql/backend.go +++ b/pkg/storage/unified/sql/backend.go @@ -355,6 +355,10 @@ func (b *backend) PrepareList(ctx context.Context, req *resource.ListRequest) (* _, span := b.tracer.Start(ctx, trace_prefix+"List") defer span.End() + if req.Options == nil || req.Options.Key.Group == "" || req.Options.Key.Resource == "" { + return nil, fmt.Errorf("missing group or resource") + } + // TODO: think about how to handler VersionMatch. We should be able to use latest for the first page (only). if req.ResourceVersion > 0 || req.NextPageToken != "" { @@ -373,9 +377,7 @@ func (b *backend) listLatest(ctx context.Context, req *resource.ListRequest) (*r err := b.sqlDB.WithTx(ctx, ReadCommitted, func(ctx context.Context, tx db.Tx) error { var err error - // TODO: Here the lastest RV might be lower than the actual latest RV - // because delete events are not included in the resource table. - out.ResourceVersion, err = fetchLatestRV(ctx, tx) + out.ResourceVersion, err = fetchLatestRV(ctx, tx, b.sqlDialect, req.Options.Key.Group, req.Options.Key.Resource) if err != nil { return err } @@ -500,10 +502,10 @@ func (b *backend) WatchWriteEvents(ctx context.Context) (<-chan *resource.Writte if err := b.Init(); err != nil { return nil, err } - // Fetch the lastest RV - since, err := fetchLatestRV(ctx, b.sqlDB) + // Get the latest RV + since, err := b.listLatestRVs(ctx) if err != nil { - return nil, err + return nil, fmt.Errorf("get the latest resource version: %w", err) } // Start the poller stream := make(chan *resource.WrittenEvent) @@ -511,9 +513,7 @@ func (b *backend) WatchWriteEvents(ctx context.Context) (<-chan *resource.Writte return stream, nil } -func (b *backend) poller(ctx context.Context, since int64, stream chan<- *resource.WrittenEvent) { - var err error - +func (b *backend) poller(ctx context.Context, since groupResourceRV, stream chan<- *resource.WrittenEvent) { interval := 100 * time.Millisecond // TODO make this configurable t := time.NewTicker(interval) defer close(stream) @@ -524,63 +524,123 @@ func (b *backend) poller(ctx context.Context, since int64, stream chan<- *resour case <-b.ctx.Done(): return case <-t.C: - since, err = b.poll(ctx, since, stream) + // List the latest RVs + grv, err := b.listLatestRVs(ctx) if err != nil { - b.log.Error("watch error", "err", err) + b.log.Error("get the latest resource version", "err", err) + t.Reset(interval) + continue } + for group, items := range grv { + for resource := range items { + // If we haven't seen this resource before, we start from 0 + if _, ok := since[group]; !ok { + since[group] = make(map[string]int64) + } + if _, ok := since[group][resource]; !ok { + since[group][resource] = 0 + } + + // Poll for new events + next, err := b.poll(ctx, group, resource, since[group][resource], stream) + if err != nil { + b.log.Error("polling for resource", "err", err) + t.Reset(interval) + continue + } + since[group][resource] = next + } + } + t.Reset(interval) } } } -// fetchLatestRV returns the current maxium RV in the resource table -func fetchLatestRV(ctx context.Context, db db.ContextExecer) (int64, error) { - // Fetch the lastest RV - rows, err := db.QueryContext(ctx, `SELECT COALESCE(max("resource_version"), 0) FROM "resource";`) +// listLatestRVs returns the latest resource version for each (Group, Resource) pair. +func (b *backend) listLatestRVs(ctx context.Context) (groupResourceRV, error) { + since := groupResourceRV{} + reqRVs := sqlResourceVersionListRequest{ + SQLTemplate: sqltemplate.New(b.sqlDialect), + groupResourceVersion: new(groupResourceVersion), + } + query, err := sqltemplate.Execute(sqlResourceVersionList, reqRVs) if err != nil { - return 0, fmt.Errorf("fetch latest rv: %w", err) + return nil, fmt.Errorf("execute SQL template to get the latest resource version: %w", err) + } + rows, err := b.sqlDB.QueryContext(ctx, query, reqRVs.GetArgs()...) + if err != nil { + return nil, fmt.Errorf("fetching recent resource versions: %w", err) } defer func() { _ = rows.Close() }() - if rows.Next() { - rv := new(int64) - if err := rows.Scan(&rv); err != nil { - return 0, fmt.Errorf("scan since resource version: %w", err) + + for rows.Next() { + if err := rows.Scan(reqRVs.GetScanDest()...); err != nil { + return nil, err } - return *rv, nil + if _, ok := since[reqRVs.Group]; !ok { + since[reqRVs.Group] = map[string]int64{} + } + if _, ok := since[reqRVs.Group][reqRVs.Resource]; !ok { + since[reqRVs.Group] = map[string]int64{} + } + since[reqRVs.Group][reqRVs.Resource] = reqRVs.ResourceVersion } - return 0, fmt.Errorf("no rows") + return since, nil } -func (b *backend) poll(ctx context.Context, since int64, stream chan<- *resource.WrittenEvent) (int64, error) { +// fetchLatestRV returns the current maximum RV in the resource table +func fetchLatestRV(ctx context.Context, x db.ContextExecer, d sqltemplate.Dialect, group, resource string) (int64, error) { + res, err := queryRow(ctx, x, sqlResourceVersionGet, sqlResourceVersionRequest{ + SQLTemplate: sqltemplate.New(d), + Group: group, + Resource: resource, + resourceVersion: new(resourceVersion), + }) + if errors.Is(err, sql.ErrNoRows) { + return 0, fmt.Errorf("now row for the provided resource version") + } else if err != nil { + return 0, fmt.Errorf("get resource version: %w", err) + } + return res.ResourceVersion, nil +} + +func (b *backend) poll(ctx context.Context, grp string, res string, since int64, stream chan<- *resource.WrittenEvent) (int64, error) { ctx, span := b.tracer.Start(ctx, trace_prefix+"poll") defer span.End() pollReq := sqlResourceHistoryPollRequest{ SQLTemplate: sqltemplate.New(b.sqlDialect), + Resource: res, + Group: grp, SinceResourceVersion: since, - Response: new(historyPollResponse), + Response: &historyPollResponse{}, } query, err := sqltemplate.Execute(sqlResourceHistoryPoll, pollReq) if err != nil { - return 0, fmt.Errorf("execute SQL template to poll for resource history: %w", err) + return since, fmt.Errorf("execute SQL template to poll for resource history: %w", err) } + rows, err := b.sqlDB.QueryContext(ctx, query, pollReq.GetArgs()...) if err != nil { - return 0, fmt.Errorf("poll for resource history: %w", err) + return since, fmt.Errorf("poll for resource history: %w", err) } + defer func() { _ = rows.Close() }() - next := since - for i := 1; rows.Next(); i++ { + nextRV := since + for rows.Next() { // check if the context is done if ctx.Err() != nil { - return 0, ctx.Err() + return nextRV, ctx.Err() } if err := rows.Scan(pollReq.GetScanDest()...); err != nil { - return 0, fmt.Errorf("scan row #%d polling for resource history: %w", i, err) + return nextRV, fmt.Errorf("scan row polling for resource history: %w", err) } resp := pollReq.Response - next = resp.ResourceVersion - + if resp.Key.Group == "" || resp.Key.Resource == "" || resp.Key.Name == "" { + return nextRV, fmt.Errorf("missing key in response") + } + nextRV = resp.ResourceVersion stream <- &resource.WrittenEvent{ WriteEvent: resource.WriteEvent{ Value: resp.Value, @@ -596,7 +656,7 @@ func (b *backend) poll(ctx context.Context, since int64, stream chan<- *resource // Timestamp: , // TODO: add timestamp } } - return next, nil + return nextRV, nil } // resourceVersionAtomicInc atomically increases the version of a kind within a diff --git a/pkg/storage/unified/sql/backend_test.go b/pkg/storage/unified/sql/backend_test.go index 42aa24e8034..8727885ae1b 100644 --- a/pkg/storage/unified/sql/backend_test.go +++ b/pkg/storage/unified/sql/backend_test.go @@ -77,12 +77,20 @@ func TestBackendHappyPath(t *testing.T) { }) t.Run("PrepareList latest", func(t *testing.T) { - resp, err := store.PrepareList(ctx, &resource.ListRequest{}) + resp, err := store.PrepareList(ctx, &resource.ListRequest{ + Options: &resource.ListOptions{ + Key: &resource.ResourceKey{ + Namespace: "namespace", + Group: "group", + Resource: "resource", + }, + }, + }) assert.NoError(t, err) assert.Len(t, resp.Items, 2) assert.Equal(t, "item2 MODIFIED", string(resp.Items[0].Value)) assert.Equal(t, "item3 ADDED", string(resp.Items[1].Value)) - assert.Equal(t, int64(4), resp.ResourceVersion) + assert.Equal(t, int64(5), resp.ResourceVersion) }) t.Run("Watch events", func(t *testing.T) { @@ -162,7 +170,14 @@ func TestBackendPrepareList(t *testing.T) { _, _ = writeEvent(ctx, store, "item3", resource.WatchEvent_DELETED) // rv=7 _, _ = writeEvent(ctx, store, "item6", resource.WatchEvent_ADDED) // rv=8 t.Run("fetch all latest", func(t *testing.T) { - res, err := store.PrepareList(ctx, &resource.ListRequest{}) + res, err := store.PrepareList(ctx, &resource.ListRequest{ + Options: &resource.ListOptions{ + Key: &resource.ResourceKey{ + Group: "group", + Resource: "resource", + }, + }, + }) assert.NoError(t, err) assert.Len(t, res.Items, 5) assert.Empty(t, res.NextPageToken) @@ -171,6 +186,12 @@ func TestBackendPrepareList(t *testing.T) { t.Run("list latest first page ", func(t *testing.T) { res, err := store.PrepareList(ctx, &resource.ListRequest{ Limit: 3, + Options: &resource.ListOptions{ + Key: &resource.ResourceKey{ + Group: "group", + Resource: "resource", + }, + }, }) assert.NoError(t, err) assert.Len(t, res.Items, 3) @@ -183,6 +204,12 @@ func TestBackendPrepareList(t *testing.T) { t.Run("list at revision", func(t *testing.T) { res, err := store.PrepareList(ctx, &resource.ListRequest{ ResourceVersion: 4, + Options: &resource.ListOptions{ + Key: &resource.ResourceKey{ + Group: "group", + Resource: "resource", + }, + }, }) assert.NoError(t, err) assert.Len(t, res.Items, 4) @@ -197,6 +224,12 @@ func TestBackendPrepareList(t *testing.T) { res, err := store.PrepareList(ctx, &resource.ListRequest{ Limit: 3, ResourceVersion: 7, + Options: &resource.ListOptions{ + Key: &resource.ResourceKey{ + Group: "group", + Resource: "resource", + }, + }, }) assert.NoError(t, err) assert.Len(t, res.Items, 3) @@ -218,6 +251,12 @@ func TestBackendPrepareList(t *testing.T) { res, err := store.PrepareList(ctx, &resource.ListRequest{ NextPageToken: continueToken.String(), Limit: 2, + Options: &resource.ListOptions{ + Key: &resource.ResourceKey{ + Group: "group", + Resource: "resource", + }, + }, }) assert.NoError(t, err) assert.Len(t, res.Items, 2) diff --git a/pkg/storage/unified/sql/data/resource_history_poll.sql b/pkg/storage/unified/sql/data/resource_history_poll.sql index d2b6dca8e2b..bebfab9286d 100644 --- a/pkg/storage/unified/sql/data/resource_history_poll.sql +++ b/pkg/storage/unified/sql/data/resource_history_poll.sql @@ -8,5 +8,9 @@ SELECT {{ .Ident "action" | .Into .Response.Action }} FROM {{ .Ident "resource_history" }} - WHERE {{ .Ident "resource_version" }} > {{ .Arg .SinceResourceVersion }} + WHERE 1 = 1 + AND {{ .Ident "group" }} = {{ .Arg .Group }} + AND {{ .Ident "resource" }} = {{ .Arg .Resource }} + AND {{ .Ident "resource_version" }} > {{ .Arg .SinceResourceVersion }} + ORDER BY {{ .Ident "resource_version" }} ASC ; diff --git a/pkg/storage/unified/sql/data/resource_version_list.sql b/pkg/storage/unified/sql/data/resource_version_list.sql new file mode 100644 index 00000000000..a793b30ca7e --- /dev/null +++ b/pkg/storage/unified/sql/data/resource_version_list.sql @@ -0,0 +1,6 @@ +SELECT + {{ .Ident "resource_version" | .Into .ResourceVersion }}, + {{ .Ident "group" | .Into .Group }}, + {{ .Ident "resource" | .Into .Resource }} + FROM {{ .Ident "resource_version" }} +; diff --git a/pkg/storage/unified/sql/queries.go b/pkg/storage/unified/sql/queries.go index 5c7bdf108ef..5b766de1b67 100644 --- a/pkg/storage/unified/sql/queries.go +++ b/pkg/storage/unified/sql/queries.go @@ -43,6 +43,7 @@ var ( sqlResourceVersionGet = mustTemplate("resource_version_get.sql") sqlResourceVersionInc = mustTemplate("resource_version_inc.sql") sqlResourceVersionInsert = mustTemplate("resource_version_insert.sql") + sqlResourceVersionList = mustTemplate("resource_version_list.sql") ) // TxOptions. @@ -104,8 +105,11 @@ func (r *historyPollResponse) Results() (*historyPollResponse, error) { return r, nil } +type groupResourceRV map[string]map[string]int64 type sqlResourceHistoryPollRequest struct { *sqltemplate.SQLTemplate + Resource string + Group string SinceResourceVersion int64 Response *historyPollResponse } @@ -176,6 +180,11 @@ type resourceVersion struct { ResourceVersion int64 } +type groupResourceVersion struct { + Group, Resource string + ResourceVersion int64 +} + func (r *resourceVersion) Results() (*resourceVersion, error) { return r, nil } @@ -189,3 +198,12 @@ type sqlResourceVersionRequest struct { func (r sqlResourceVersionRequest) Validate() error { return nil // TODO } + +type sqlResourceVersionListRequest struct { + *sqltemplate.SQLTemplate + *groupResourceVersion +} + +func (r sqlResourceVersionListRequest) Validate() error { + return nil // TODO +}