Merge pull request #30 from Walther/2023-10-30-more

2023 10 30 more

README.md

@@ -67,8 +67,8 @@ sequenceDiagram
     postgres->>batch: ;
     note over batch: rendering
     batch->>postgres: delete render_task
-    batch->>postgres: save render_result id
     batch->>bucket: save image and thumbnail
+    batch->>postgres: save render_result id
     front-->batch: ;
     front->>back: GET /renders
     back->>postgres: get render_result ids
@@ -154,6 +154,6 @@ sequenceDiagram
 | 2023-10-26 | 5.5   | Improvement: major UI overhaul! New layout, remove flex, grid only. Improved responsiveness & breakpoints. |
 | 2023-10-27 | 4.5   | Feature: scene options handling, add One Weekend scene. Improvements: further layout and breakpoint polish |
 | 2023-10-29 | 12.5  | Feature: images and thumbnails in S3 compatible storage. Ridiculously long work & debug with the SDK.      |
-| 2023-10-30 | 1.5   | Feature: finish the S3 compatible work and add basic ratelimiting to the backend                           |
+| 2023-10-30 | 5.5   | Feature: finish the S3 compatible work                                                                     |
 | ---        | ---   | ---                                                                                                        |
-| total      | 180.0 |                                                                                                            |
+| total      | 184.0 |                                                                                                            |

clovers-back/Cargo.toml

@@ -22,7 +22,6 @@ sqlx = { version = "0.6", features = [
   "json",
 ] }
 tokio = { version = "1.32", features = ["full"] }
-tower = "0.4.13"
 tower-http = { version = "0.4.3", features = ["trace", "cors"] }
 tower_governor = "0.1.0"
 tracing = "0.1"

clovers-back/src/main.rs

@@ -4,22 +4,19 @@ mod rest;
 mod ws;
 
 use axum::{
-    error_handling::HandleErrorLayer,
     extract::FromRef,
     http::{
         header::{self},
         HeaderValue, Method, StatusCode,
     },
     response::IntoResponse,
     routing::{get, post},
-    BoxError, Json, Router,
+    Json, Router,
 };
 use clovers_svc_common::load_configs;
 use redis::aio::ConnectionManager;
 use sqlx::{postgres::PgPoolOptions, Pool, Postgres};
 use tokio::sync::Mutex;
-use tower::ServiceBuilder;
-use tower_governor::{errors::display_error, governor::GovernorConfigBuilder, GovernorLayer};
 use tower_http::{
     cors::CorsLayer,
     trace::{DefaultOnRequest, DefaultOnResponse, TraceLayer},
@@ -74,10 +71,6 @@ async fn main() -> anyhow::Result<()> {
         s3: s3client,
     };
 
-    // set up ratelimiting
-    // TODO: endpoint specific ratelimits, especially for POST /render
-    let governor_conf = Box::new(GovernorConfigBuilder::default().finish().unwrap());
-
     // assemble the application
     let app = Router::new()
         // `GET /` goes to `hello`
@@ -122,17 +115,6 @@ async fn main() -> anyhow::Result<()> {
                         .level(Level::INFO)
                         .latency_unit(LatencyUnit::Micros),
                 ),
-        )
-        .layer(
-            ServiceBuilder::new()
-                // this middleware goes above `GovernorLayer` because it will receive
-                // errors returned by `GovernorLayer` below
-                .layer(HandleErrorLayer::new(|e: BoxError| async move {
-                    display_error(e)
-                }))
-                .layer(GovernorLayer {
-                    config: Box::leak(governor_conf),
-                }),
         );
 
     // run the app

clovers-front/fly.toml

@@ -7,7 +7,7 @@ kill_timeout = 5
 processes = []
 
 [build]
-builder = "heroku/buildpacks:20"
+builder = "heroku/builder:22"
 
 [build.args]
 REACT_APP_BACKEND = "https://clovers-back.fly.dev"
@@ -26,6 +26,9 @@ internal_port = 8080
 processes = ["app"]
 protocol = "tcp"
 script_checks = []
+auto_stop_machines = true
+auto_start_machines = true
+min_machines_running = 0
 [services.concurrency]
 hard_limit = 25
 soft_limit = 20
@@ -41,7 +44,14 @@ handlers = ["tls", "http"]
 port = 443
 
 [[services.tcp_checks]]
-grace_period = "1s"
-interval = "15s"
+grace_period = "10s"
+interval = "30s"
 restart_limit = 0
-timeout = "2s"
+timeout = "5s"
+
+[[http_service.checks]]
+grace_period = "10s"
+interval = "30s"
+method = "GET"
+timeout = "5s"
+path = "/"

clovers-svc-common/src/handlers/render_result.rs

@@ -15,30 +15,36 @@ pub async fn save_render_result(
     postgres_pool: &Pool<Postgres>,
     s3: &aws_sdk_s3::Client,
 ) -> Result<Uuid> {
-    let id: Uuid = match sqlx::query(
+    let uuid = Uuid::new_v4();
+
+    // first store the image in the S3 compatible storage
+    let path = format!("images/{uuid}");
+    put_object(s3, path, render_result.image).await?;
+    let path = format!("thumbs/{uuid}");
+    put_object(s3, path, render_result.thumb).await?;
+
+    // then insert into database
+    // this prevents the frontend race condition for the thumbnails
+    let _id: Uuid = match sqlx::query(
         r#"
-INSERT INTO render_results
-VALUES ( default )
+INSERT INTO render_results ( id )
+VALUES ( $1 )
 RETURNING id
       "#,
     )
+    .bind(uuid)
     .fetch_one(postgres_pool)
     .await
     {
         Ok(row) => row.try_get("id")?,
         Err(e) => {
-            let error_message = format!("Error saving rendertask to postgres: {e}");
+            let error_message = format!("Error saving render_result to postgres: {e}");
             tracing::error!("{error_message}");
             return Err(anyhow!("{error_message}"));
         }
     };
 
-    let path = format!("images/{id}");
-    put_object(s3, path, render_result.image).await?;
-    let path = format!("thumbs/{id}");
-    put_object(s3, path, render_result.thumb).await?;
-
-    Ok(id)
+    Ok(uuid)
 }
 
 pub async fn put_object(