Merge pull request #362 from sebadob/dynamic-server-side-pagination

feat: dynamic server side pagination + search for users

dev_notes.md

@@ -2,6 +2,8 @@
 
 ## CURRENT WORK
 
+- dynamic server side pagination for users in admin ui
+
 ## Stage 1 - essentials
 
 [x] finished
@@ -17,12 +19,13 @@
 
 ### Notes for performance optimizations
 
-- all the `get_`s on the `Client` will probably be good with returning slices instead of real Strings -> less memory
-  allocations
+- all the `get_`s on the `Client` will probably be good with returning slices instead of real Strings
+  -> less memory allocations
 
 ## Stage 3 - Possible nice to haves
 
 - respect `display=popup` and / or `display=touch` on `/authorize`
 - impl experimental `dilithium` alg for token signing to become quantum safe
-- 'rauthy-migrate' project to help migrating to rauthy?
+- 'rauthy-migrate' project to help migrating to rauthy? probably when doing benchmarks anyway and use it
+  for dummy data?
 - custom event listener template to build own implementation? -> only if NATS will be implemented maybe?

migrations/postgres/22_users_created_at_index.sql

@@ -0,0 +1,2 @@
+create index users_created_at_index
+    on users (created_at);

migrations/sqlite/22_users_created_at_index.sql

@@ -0,0 +1,2 @@
+create index users_created_at_index
+    on users (created_at);

rauthy-common/src/constants.rs

@@ -68,6 +68,7 @@ pub const IDX_SCOPES: &str = "scopes_";
 pub const IDX_SESSION: &str = "session_";
 pub const IDX_SESSIONS: &str = "sessions";
 pub const IDX_USERS: &str = "users_";
+pub const USER_COUNT_IDX: &str = "users_count_total";
 pub const IDX_USERS_VALUES: &str = "users_values_";
 pub const IDX_USER_ATTR_CONFIG: &str = "user_attrs_";
 pub const IDX_WEBAUTHN: &str = "webauthn_";
@@ -348,6 +349,11 @@ lazy_static! {
         .parse::<bool>()
         .expect("SWAGGER_UI_EXTERNAL cannot be parsed to bool - bad format");
 
+     pub static ref SSP_THRESHOLD: u16 = env::var("SSP_THRESHOLD")
+        .unwrap_or_else(|_| String::from("1000"))
+        .parse::<u16>()
+        .expect("SSP_THRESHOLD cannot be parsed to u16 - bad format");
+
     pub static ref UNSAFE_NO_RESET_BINDING: bool = env::var("UNSAFE_NO_RESET_BINDING")
         .unwrap_or_else(|_| String::from("false"))
         .parse::<bool>()

rauthy-handlers/src/generic.rs

@@ -543,9 +543,10 @@ pub async fn get_search(
 ) -> Result<HttpResponse, ErrorResponse> {
     principal.validate_admin_session()?;
 
+    let limit = params.limit.unwrap_or(100) as i64;
     match params.ty {
         SearchParamsType::User => {
-            let res = User::search(&data, &params.idx, &params.q).await?;
+            let res = User::search(&data, &params.idx, &params.q, limit).await?;
             Ok(HttpResponse::Ok().json(res))
         }
     }

rauthy-handlers/src/users.rs

@@ -2,16 +2,17 @@ use crate::ReqPrincipal;
 use actix_web::http::header::LOCATION;
 use actix_web::http::StatusCode;
 use actix_web::{cookie, delete, get, post, put, web, HttpRequest, HttpResponse, ResponseError};
-use actix_web_validator::Json;
+use actix_web_validator::{Json, Query};
 use rauthy_common::constants::{
     COOKIE_MFA, ENABLE_WEB_ID, HEADER_ALLOW_ALL_ORIGINS, HEADER_HTML, OPEN_USER_REG,
-    PWD_RESET_COOKIE, TEXT_TURTLE, USER_REG_DOMAIN_RESTRICTION,
+    PWD_RESET_COOKIE, SSP_THRESHOLD, TEXT_TURTLE, USER_REG_DOMAIN_RESTRICTION,
 };
 use rauthy_common::error_response::{ErrorResponse, ErrorResponseType};
 use rauthy_common::utils::real_ip_from_req;
 use rauthy_models::app_state::AppState;
 use rauthy_models::entity::api_keys::{AccessGroup, AccessRights};
 use rauthy_models::entity::colors::ColorEntity;
+use rauthy_models::entity::continuation_token::ContinuationToken;
 use rauthy_models::entity::password::PasswordPolicy;
 use rauthy_models::entity::pow::PowEntity;
 use rauthy_models::entity::user_attr::{UserAttrConfigEntity, UserAttrValueEntity};
@@ -23,14 +24,14 @@ use rauthy_models::entity::webids::WebId;
 use rauthy_models::events::event::Event;
 use rauthy_models::language::Language;
 use rauthy_models::request::{
-    MfaPurpose, NewUserRegistrationRequest, NewUserRequest, PasswordResetRequest,
+    MfaPurpose, NewUserRegistrationRequest, NewUserRequest, PaginationParams, PasswordResetRequest,
     RequestResetRequest, UpdateUserRequest, UpdateUserSelfRequest, UserAttrConfigRequest,
     UserAttrValuesUpdateRequest, WebIdRequest, WebauthnAuthFinishRequest, WebauthnAuthStartRequest,
     WebauthnRegFinishRequest, WebauthnRegStartRequest,
 };
 use rauthy_models::response::{
     PasskeyResponse, UserAttrConfigResponse, UserAttrValueResponse, UserAttrValuesResponse,
-    UserResponse, UserResponseSimple, WebIdResponse,
+    UserResponse, WebIdResponse,
 };
 use rauthy_models::templates::{Error1Html, Error3Html, ErrorHtml, UserRegisterHtml};
 use rauthy_service::password_reset;
@@ -57,17 +58,46 @@ use tracing::{error, warn};
 pub async fn get_users(
     data: web::Data<AppState>,
     principal: ReqPrincipal,
+    params: Query<PaginationParams>,
 ) -> Result<HttpResponse, ErrorResponse> {
     principal.validate_api_key_or_admin_session(AccessGroup::Users, AccessRights::Read)?;
 
-    let users = User::find_all(&data).await?;
-    let mut res = Vec::new();
-    users
-        .into_iter()
-        // return a simplified version to decrease payload for big deployments
-        .for_each(|u| res.push(UserResponseSimple::from(u)));
+    let user_count = User::count(&data).await?;
 
-    Ok(HttpResponse::Ok().json(res))
+    if user_count >= *SSP_THRESHOLD as i64 || params.page_size.is_some() {
+        let page_size = params.page_size.unwrap_or(15) as i64;
+        let offset = params.offset.unwrap_or(0) as i64;
+        let backwards = params.backwards.unwrap_or(false);
+        let continuation_token = if let Some(token) = &params.continuation_token {
+            Some(ContinuationToken::try_from(token.as_str())?)
+        } else {
+            None
+        };
+
+        let (users, continuation_token) =
+            User::find_paginated(&data, continuation_token, page_size, offset, backwards).await?;
+        let x_page_count = (user_count as f64 / page_size as f64).ceil() as u32;
+
+        if let Some(token) = continuation_token {
+            Ok(HttpResponse::PartialContent()
+                .insert_header(("x-user-count", user_count))
+                .insert_header(("x-page-count", x_page_count))
+                .insert_header(("x-page-size", page_size as u32))
+                .insert_header(token.into_header_pair())
+                .json(users))
+        } else {
+            Ok(HttpResponse::PartialContent()
+                .insert_header(("x-user-count", user_count))
+                .insert_header(("x-page-count", x_page_count))
+                .insert_header(("x-page-size", page_size as u32))
+                .json(users))
+        }
+    } else {
+        let users = User::find_all_simple(&data).await?;
+        Ok(HttpResponse::Ok()
+            .insert_header(("x-user-count", user_count))
+            .json(users))
+    }
 }
 
 /// Adds a new user to the database

rauthy-models/src/entity/continuation_token.rs

@@ -0,0 +1,55 @@
+use actix_web::http::header::{HeaderName, HeaderValue};
+use rauthy_common::error_response::{ErrorResponse, ErrorResponseType};
+use serde::{Deserialize, Serialize};
+use std::str::FromStr;
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct ContinuationToken {
+    pub id: String,
+    pub ts: i64,
+}
+
+impl ToString for ContinuationToken {
+    fn to_string(&self) -> String {
+        format!("{}{}", self.ts, self.id)
+    }
+}
+
+impl TryFrom<&str> for ContinuationToken {
+    type Error = ErrorResponse;
+
+    fn try_from(value: &str) -> Result<Self, Self::Error> {
+        if value.len() < 16 {
+            return Err(ErrorResponse::new(
+                ErrorResponseType::BadRequest,
+                "Invalid continuation_token".to_string(),
+            ));
+        }
+
+        let (ts, id) = value.split_at(10);
+        let ts = i64::from_str(ts).map_err(|err| {
+            ErrorResponse::new(
+                ErrorResponseType::BadRequest,
+                format!("timestamp str cannot be parsed into i64: {:?}", err),
+            )
+        })?;
+
+        Ok(Self {
+            id: id.to_string(),
+            ts,
+        })
+    }
+}
+
+impl ContinuationToken {
+    pub fn new(id: String, ts: i64) -> Self {
+        Self { id, ts }
+    }
+
+    pub fn into_header_pair(self) -> (HeaderName, HeaderValue) {
+        // these header values will always be valid
+        let name = HeaderName::from_str("x-continuation-token").unwrap();
+        let value = HeaderValue::from_str(&self.to_string()).unwrap();
+        (name, value)
+    }
+}

rauthy-models/src/entity/mod.rs

@@ -9,6 +9,7 @@ pub mod clients;
 pub mod clients_dyn;
 pub mod colors;
 pub mod config;
+pub mod continuation_token;
 pub mod db_version;
 pub mod dpop_proof;
 pub mod groups;