diff --git a/dev_notes.md b/dev_notes.md index ed48fb96..5d7aa643 100644 --- a/dev_notes.md +++ b/dev_notes.md @@ -2,6 +2,8 @@ ## CURRENT WORK +- dynamic server side pagination for users in admin ui + ## Stage 1 - essentials [x] finished @@ -17,12 +19,13 @@ ### Notes for performance optimizations -- all the `get_`s on the `Client` will probably be good with returning slices instead of real Strings -> less memory - allocations +- all the `get_`s on the `Client` will probably be good with returning slices instead of real Strings + -> less memory allocations ## Stage 3 - Possible nice to haves - respect `display=popup` and / or `display=touch` on `/authorize` - impl experimental `dilithium` alg for token signing to become quantum safe -- 'rauthy-migrate' project to help migrating to rauthy? +- 'rauthy-migrate' project to help migrating to rauthy? probably when doing benchmarks anyway and use it + for dummy data? - custom event listener template to build own implementation? -> only if NATS will be implemented maybe? diff --git a/migrations/postgres/22_users_created_at_index.sql b/migrations/postgres/22_users_created_at_index.sql new file mode 100644 index 00000000..2b033738 --- /dev/null +++ b/migrations/postgres/22_users_created_at_index.sql @@ -0,0 +1,2 @@ +create index users_created_at_index + on users (created_at); \ No newline at end of file diff --git a/migrations/sqlite/22_users_created_at_index.sql b/migrations/sqlite/22_users_created_at_index.sql new file mode 100644 index 00000000..2b033738 --- /dev/null +++ b/migrations/sqlite/22_users_created_at_index.sql @@ -0,0 +1,2 @@ +create index users_created_at_index + on users (created_at); \ No newline at end of file diff --git a/rauthy-common/src/constants.rs b/rauthy-common/src/constants.rs index bf40d895..3fe342d9 100644 --- a/rauthy-common/src/constants.rs +++ b/rauthy-common/src/constants.rs @@ -68,6 +68,7 @@ pub const IDX_SCOPES: &str = "scopes_"; pub const IDX_SESSION: &str = "session_"; pub const IDX_SESSIONS: &str = "sessions"; pub const IDX_USERS: &str = "users_"; +pub const USER_COUNT_IDX: &str = "users_count_total"; pub const IDX_USERS_VALUES: &str = "users_values_"; pub const IDX_USER_ATTR_CONFIG: &str = "user_attrs_"; pub const IDX_WEBAUTHN: &str = "webauthn_"; @@ -348,6 +349,11 @@ lazy_static! { .parse::() .expect("SWAGGER_UI_EXTERNAL cannot be parsed to bool - bad format"); + pub static ref SSP_THRESHOLD: u16 = env::var("SSP_THRESHOLD") + .unwrap_or_else(|_| String::from("1000")) + .parse::() + .expect("SSP_THRESHOLD cannot be parsed to u16 - bad format"); + pub static ref UNSAFE_NO_RESET_BINDING: bool = env::var("UNSAFE_NO_RESET_BINDING") .unwrap_or_else(|_| String::from("false")) .parse::() diff --git a/rauthy-handlers/src/generic.rs b/rauthy-handlers/src/generic.rs index 67e035a0..082c6d95 100644 --- a/rauthy-handlers/src/generic.rs +++ b/rauthy-handlers/src/generic.rs @@ -543,9 +543,10 @@ pub async fn get_search( ) -> Result { principal.validate_admin_session()?; + let limit = params.limit.unwrap_or(100) as i64; match params.ty { SearchParamsType::User => { - let res = User::search(&data, ¶ms.idx, ¶ms.q).await?; + let res = User::search(&data, ¶ms.idx, ¶ms.q, limit).await?; Ok(HttpResponse::Ok().json(res)) } } diff --git a/rauthy-handlers/src/users.rs b/rauthy-handlers/src/users.rs index 380302a0..7577cbe0 100644 --- a/rauthy-handlers/src/users.rs +++ b/rauthy-handlers/src/users.rs @@ -2,16 +2,17 @@ use crate::ReqPrincipal; use actix_web::http::header::LOCATION; use actix_web::http::StatusCode; use actix_web::{cookie, delete, get, post, put, web, HttpRequest, HttpResponse, ResponseError}; -use actix_web_validator::Json; +use actix_web_validator::{Json, Query}; use rauthy_common::constants::{ COOKIE_MFA, ENABLE_WEB_ID, HEADER_ALLOW_ALL_ORIGINS, HEADER_HTML, OPEN_USER_REG, - PWD_RESET_COOKIE, TEXT_TURTLE, USER_REG_DOMAIN_RESTRICTION, + PWD_RESET_COOKIE, SSP_THRESHOLD, TEXT_TURTLE, USER_REG_DOMAIN_RESTRICTION, }; use rauthy_common::error_response::{ErrorResponse, ErrorResponseType}; use rauthy_common::utils::real_ip_from_req; use rauthy_models::app_state::AppState; use rauthy_models::entity::api_keys::{AccessGroup, AccessRights}; use rauthy_models::entity::colors::ColorEntity; +use rauthy_models::entity::continuation_token::ContinuationToken; use rauthy_models::entity::password::PasswordPolicy; use rauthy_models::entity::pow::PowEntity; use rauthy_models::entity::user_attr::{UserAttrConfigEntity, UserAttrValueEntity}; @@ -23,14 +24,14 @@ use rauthy_models::entity::webids::WebId; use rauthy_models::events::event::Event; use rauthy_models::language::Language; use rauthy_models::request::{ - MfaPurpose, NewUserRegistrationRequest, NewUserRequest, PasswordResetRequest, + MfaPurpose, NewUserRegistrationRequest, NewUserRequest, PaginationParams, PasswordResetRequest, RequestResetRequest, UpdateUserRequest, UpdateUserSelfRequest, UserAttrConfigRequest, UserAttrValuesUpdateRequest, WebIdRequest, WebauthnAuthFinishRequest, WebauthnAuthStartRequest, WebauthnRegFinishRequest, WebauthnRegStartRequest, }; use rauthy_models::response::{ PasskeyResponse, UserAttrConfigResponse, UserAttrValueResponse, UserAttrValuesResponse, - UserResponse, UserResponseSimple, WebIdResponse, + UserResponse, WebIdResponse, }; use rauthy_models::templates::{Error1Html, Error3Html, ErrorHtml, UserRegisterHtml}; use rauthy_service::password_reset; @@ -57,17 +58,46 @@ use tracing::{error, warn}; pub async fn get_users( data: web::Data, principal: ReqPrincipal, + params: Query, ) -> Result { principal.validate_api_key_or_admin_session(AccessGroup::Users, AccessRights::Read)?; - let users = User::find_all(&data).await?; - let mut res = Vec::new(); - users - .into_iter() - // return a simplified version to decrease payload for big deployments - .for_each(|u| res.push(UserResponseSimple::from(u))); + let user_count = User::count(&data).await?; - Ok(HttpResponse::Ok().json(res)) + if user_count >= *SSP_THRESHOLD as i64 || params.page_size.is_some() { + let page_size = params.page_size.unwrap_or(15) as i64; + let offset = params.offset.unwrap_or(0) as i64; + let backwards = params.backwards.unwrap_or(false); + let continuation_token = if let Some(token) = ¶ms.continuation_token { + Some(ContinuationToken::try_from(token.as_str())?) + } else { + None + }; + + let (users, continuation_token) = + User::find_paginated(&data, continuation_token, page_size, offset, backwards).await?; + let x_page_count = (user_count as f64 / page_size as f64).ceil() as u32; + + if let Some(token) = continuation_token { + Ok(HttpResponse::PartialContent() + .insert_header(("x-user-count", user_count)) + .insert_header(("x-page-count", x_page_count)) + .insert_header(("x-page-size", page_size as u32)) + .insert_header(token.into_header_pair()) + .json(users)) + } else { + Ok(HttpResponse::PartialContent() + .insert_header(("x-user-count", user_count)) + .insert_header(("x-page-count", x_page_count)) + .insert_header(("x-page-size", page_size as u32)) + .json(users)) + } + } else { + let users = User::find_all_simple(&data).await?; + Ok(HttpResponse::Ok() + .insert_header(("x-user-count", user_count)) + .json(users)) + } } /// Adds a new user to the database diff --git a/rauthy-models/src/entity/continuation_token.rs b/rauthy-models/src/entity/continuation_token.rs new file mode 100644 index 00000000..37431f66 --- /dev/null +++ b/rauthy-models/src/entity/continuation_token.rs @@ -0,0 +1,55 @@ +use actix_web::http::header::{HeaderName, HeaderValue}; +use rauthy_common::error_response::{ErrorResponse, ErrorResponseType}; +use serde::{Deserialize, Serialize}; +use std::str::FromStr; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ContinuationToken { + pub id: String, + pub ts: i64, +} + +impl ToString for ContinuationToken { + fn to_string(&self) -> String { + format!("{}{}", self.ts, self.id) + } +} + +impl TryFrom<&str> for ContinuationToken { + type Error = ErrorResponse; + + fn try_from(value: &str) -> Result { + if value.len() < 16 { + return Err(ErrorResponse::new( + ErrorResponseType::BadRequest, + "Invalid continuation_token".to_string(), + )); + } + + let (ts, id) = value.split_at(10); + let ts = i64::from_str(ts).map_err(|err| { + ErrorResponse::new( + ErrorResponseType::BadRequest, + format!("timestamp str cannot be parsed into i64: {:?}", err), + ) + })?; + + Ok(Self { + id: id.to_string(), + ts, + }) + } +} + +impl ContinuationToken { + pub fn new(id: String, ts: i64) -> Self { + Self { id, ts } + } + + pub fn into_header_pair(self) -> (HeaderName, HeaderValue) { + // these header values will always be valid + let name = HeaderName::from_str("x-continuation-token").unwrap(); + let value = HeaderValue::from_str(&self.to_string()).unwrap(); + (name, value) + } +} diff --git a/rauthy-models/src/entity/mod.rs b/rauthy-models/src/entity/mod.rs index 3aed5431..d8e9866a 100644 --- a/rauthy-models/src/entity/mod.rs +++ b/rauthy-models/src/entity/mod.rs @@ -9,6 +9,7 @@ pub mod clients; pub mod clients_dyn; pub mod colors; pub mod config; +pub mod continuation_token; pub mod db_version; pub mod dpop_proof; pub mod groups; diff --git a/rauthy-models/src/entity/users.rs b/rauthy-models/src/entity/users.rs index 2e0f9e4c..cfe31d2b 100644 --- a/rauthy-models/src/entity/users.rs +++ b/rauthy-models/src/entity/users.rs @@ -1,6 +1,7 @@ use crate::app_state::{AppState, Argon2Params, DbTxn}; use crate::email::{send_email_change_info_new, send_email_confirm_change, send_pwd_reset}; use crate::entity::colors::ColorEntity; +use crate::entity::continuation_token::ContinuationToken; use crate::entity::groups::Group; use crate::entity::magic_links::{MagicLink, MagicLinkUsage}; use crate::entity::password::PasswordPolicy; @@ -21,7 +22,8 @@ use crate::templates::UserEmailChangeConfirmHtml; use actix_web::{web, HttpRequest}; use argon2::PasswordHash; use rauthy_common::constants::{ - CACHE_NAME_USERS, IDX_USERS, RAUTHY_ADMIN_ROLE, WEBAUTHN_NO_PASSWORD_EXPIRY, + CACHE_NAME_12HR, CACHE_NAME_USERS, IDX_USERS, RAUTHY_ADMIN_ROLE, USER_COUNT_IDX, + WEBAUTHN_NO_PASSWORD_EXPIRY, }; use rauthy_common::error_response::{ErrorResponse, ErrorResponseType}; use rauthy_common::password_hasher::{ComparePasswords, HashPassword}; @@ -73,6 +75,73 @@ pub struct User { // CRUD impl User { + pub async fn count(data: &web::Data) -> Result { + if let Some(count) = cache_get!( + i64, + CACHE_NAME_12HR.to_string(), + USER_COUNT_IDX.to_string(), + &data.caches.ha_cache_config, + false + ) + .await? + { + return Ok(count); + } + + let res = sqlx::query!("SELECT COUNT (*) count FROM users") + .fetch_one(&data.db) + .await?; + + // sqlite returns an i32 for count while postgres returns an Option + #[cfg(feature = "postgres")] + let count = res.count.unwrap_or_default(); + #[cfg(not(feature = "postgres"))] + let count = res.count as i64; + + cache_insert( + CACHE_NAME_12HR.to_string(), + USER_COUNT_IDX.to_string(), + &data.caches.ha_cache_config, + &count, + AckLevel::Once, + ) + .await?; + + Ok(count) + } + + async fn count_inc(data: &web::Data) -> Result<(), ErrorResponse> { + let mut count = Self::count(data).await?; + // theoretically, we could have overlaps here, but we don't really care + // -> used for dynamic pagination only and SQLite has limited query features + count += 1; + cache_insert( + CACHE_NAME_12HR.to_string(), + USER_COUNT_IDX.to_string(), + &data.caches.ha_cache_config, + &count, + AckLevel::Once, + ) + .await?; + Ok(()) + } + + async fn count_dec(data: &web::Data) -> Result<(), ErrorResponse> { + let mut count = Self::count(data).await?; + // theoretically, we could have overlaps here, but we don't really care + // -> used for dynamic pagination only and SQLite has limited query features + count -= 1; + cache_insert( + CACHE_NAME_12HR.to_string(), + USER_COUNT_IDX.to_string(), + &data.caches.ha_cache_config, + &count, + AckLevel::Once, + ) + .await?; + Ok(()) + } + // Inserts a user into the database pub async fn create( data: &web::Data, @@ -155,6 +224,8 @@ impl User { ) .await?; + Self::count_dec(data).await?; + Ok(()) } @@ -260,12 +331,24 @@ impl User { } pub async fn find_all(data: &web::Data) -> Result, ErrorResponse> { - let res = sqlx::query_as!(Self, "select * from users") + let res = sqlx::query_as!(Self, "SELECT * FROM users ORDER BY created_at ASC") .fetch_all(&data.db) .await?; Ok(res) } + pub async fn find_all_simple( + data: &web::Data, + ) -> Result, ErrorResponse> { + let res = sqlx::query_as!( + UserResponseSimple, + "SELECT id, email FROM users ORDER BY created_at ASC" + ) + .fetch_all(&data.db) + .await?; + Ok(res) + } + pub async fn find_expired(data: &web::Data) -> Result, ErrorResponse> { let now = OffsetDateTime::now_utc() .add(time::Duration::seconds(10)) @@ -276,6 +359,95 @@ impl User { Ok(res) } + pub async fn find_paginated( + data: &web::Data, + continuation_token: Option, + page_size: i64, + offset: i64, + backwards: bool, + ) -> Result<(Vec, Option), ErrorResponse> { + let mut res = Vec::with_capacity(page_size as usize); + let mut latest_ts = 0; + + if let Some(token) = continuation_token { + if backwards { + let rows = sqlx::query!( + r#"SELECT id, email, created_at + FROM users + WHERE created_at <= $1 AND id != $2 + ORDER BY created_at DESC + LIMIT $3 + OFFSET $4"#, + token.ts, + token.id, + page_size, + offset, + ) + .fetch_all(&data.db) + .await?; + + for row in rows { + res.push(UserResponseSimple { + id: row.id, + email: row.email, + }); + latest_ts = row.created_at; + } + res.reverse(); + } else { + let rows = sqlx::query!( + r#"SELECT id, email, created_at + FROM users + WHERE created_at >= $1 AND id != $2 + ORDER BY created_at ASC + LIMIT $3 + OFFSET $4"#, + token.ts, + token.id, + page_size, + offset, + ) + .fetch_all(&data.db) + .await?; + + for row in rows { + res.push(UserResponseSimple { + id: row.id, + email: row.email, + }); + latest_ts = row.created_at; + } + }; + } else { + // there is no "backwards" without a continuation token + let rows = sqlx::query!( + r#"SELECT id, email, created_at + FROM users + ORDER BY created_at ASC + LIMIT $1 + OFFSET $2"#, + page_size, + offset, + ) + .fetch_all(&data.db) + .await?; + + for row in rows { + res.push(UserResponseSimple { + id: row.id, + email: row.email, + }); + latest_ts = row.created_at; + } + }; + + let token = res + .last() + .map(|entry| ContinuationToken::new(entry.id.clone(), latest_ts)); + + Ok((res, token)) + } + async fn insert(data: &web::Data, new_user: User) -> Result { let lang = new_user.language.as_str(); sqlx::query!( @@ -301,6 +473,8 @@ impl User { .execute(&data.db) .await?; + Self::count_inc(data).await?; + Ok(new_user) } @@ -413,6 +587,7 @@ impl User { data: &web::Data, idx: &SearchParamsIdx, q: &str, + limit: i64, ) -> Result, ErrorResponse> { let q = format!("%{}%", q); @@ -420,18 +595,20 @@ impl User { SearchParamsIdx::Id => { query_as!( UserResponseSimple, - "SELECT id, email FROM users WHERE id LIKE $1", - q + "SELECT id, email FROM users WHERE id LIKE $1 ORDER BY created_at ASC LIMIT $2", + q, + limit ) .fetch_all(&data.db) .await? } SearchParamsIdx::Email => { query_as!( - UserResponseSimple, - "SELECT id, email FROM users WHERE email LIKE $1", - q - ) + UserResponseSimple, + "SELECT id, email FROM users WHERE email LIKE $1 ORDER BY created_at ASC LIMIT $2", + q, + limit + ) .fetch_all(&data.db) .await? } diff --git a/rauthy-models/src/request.rs b/rauthy-models/src/request.rs index e0a9b9bb..780a2598 100644 --- a/rauthy-models/src/request.rs +++ b/rauthy-models/src/request.rs @@ -647,6 +647,17 @@ pub struct NewRoleRequest { pub role: String, } +#[derive(Debug, Deserialize, Validate, ToSchema)] +pub struct PaginationParams { + pub page: Option, + pub page_size: Option, + pub offset: Option, + pub backwards: Option, + /// Validation: `[a-zA-Z0-9]` + #[validate(regex(path = "RE_ALNUM", code = "[a-zA-Z0-9]"))] + pub continuation_token: Option, +} + #[derive(Debug, Serialize, Deserialize, Validate, ToSchema)] pub struct PasskeyRequest { /// Validation: `[a-zA-Z0-9À-ÿ-\\s]{2,32}` @@ -676,6 +687,7 @@ pub struct SearchParams { /// The actual search query - validation: `[a-zA-Z0-9,.:/_\-&?=~#!$'()*+%@]+` #[validate(regex(path = "RE_SEARCH", code = "[a-zA-Z0-9,.:/_\\-&?=~#!$'()*+%@]+"))] pub q: String, + pub limit: Option, } #[derive(Debug, PartialEq, Deserialize, ToSchema)] diff --git a/rauthy.cfg b/rauthy.cfg index f0ae68bc..380d459f 100644 --- a/rauthy.cfg +++ b/rauthy.cfg @@ -792,6 +792,13 @@ SWAGGER_UI_EXTERNAL=true # default: 30 #SSE_KEEP_ALIVE=30 +# Dynamic server side pagination threshold +# If the total users count exceeds this value, Rauthy will dynamically +# change search and pagination for users in the Admin UI from client +# side to server side to not have a degradation in performance. +# default: 1000 +SSP_THRESHOLD=1000 + ##################################### ############ TEMPLATES ############## #####################################