From 61bec279163dda2a31726483df1cefa19282034a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sa=C5=A1a=20Tomi=C4=87?= Date: Thu, 20 Jun 2024 18:11:34 +0200 Subject: [PATCH] feat(dre): Cache public dashboard API response for 1h (#506) --- rs/cli/src/registry_dump.rs | 2 +- rs/cli/src/runner.rs | 4 +- .../src/public_dashboard.rs | 55 ++++++++++++++----- rs/ic-management-backend/src/registry.rs | 11 ++-- rs/np-notifications/src/health_check.rs | 4 +- 5 files changed, 54 insertions(+), 22 deletions(-) diff --git a/rs/cli/src/registry_dump.rs b/rs/cli/src/registry_dump.rs index 449707eb4..f19fbb1bd 100644 --- a/rs/cli/src/registry_dump.rs +++ b/rs/cli/src/registry_dump.rs @@ -65,7 +65,7 @@ async fn get_registry(path: &Option, network: &Network, version: &i64) let elected_host_os_versions = get_elected_host_os_versions(&local_registry, version)?; let node_provider_names: HashMap = HashMap::from_iter( - query_ic_dashboard_list::("v3/node-providers") + query_ic_dashboard_list::(network, "v3/node-providers") .await? .node_providers .iter() diff --git a/rs/cli/src/runner.rs b/rs/cli/src/runner.rs index f87c20afd..a81b811d2 100644 --- a/rs/cli/src/runner.rs +++ b/rs/cli/src/runner.rs @@ -191,7 +191,7 @@ impl Runner { let dashboard_backend_client = DashboardBackendClient::new_with_backend_url(backend_url); let mut registry = registry::RegistryState::new(network, true).await; - let node_providers = query_ic_dashboard_list::("v3/node-providers") + let node_providers = query_ic_dashboard_list::(network, "v3/node-providers") .await? .node_providers; registry.update_node_details(&node_providers).await?; @@ -211,7 +211,7 @@ impl Runner { let dashboard_backend_client = DashboardBackendClient::new_with_backend_url(backend_url); let mut registry = registry::RegistryState::new(network, true).await; - let node_providers = query_ic_dashboard_list::("v3/node-providers") + let node_providers = query_ic_dashboard_list::(network, "v3/node-providers") .await? .node_providers; registry.update_node_details(&node_providers).await?; diff --git a/rs/ic-management-backend/src/public_dashboard.rs b/rs/ic-management-backend/src/public_dashboard.rs index 16842b819..c2b207c7d 100644 --- a/rs/ic-management-backend/src/public_dashboard.rs +++ b/rs/ic-management-backend/src/public_dashboard.rs @@ -1,17 +1,46 @@ +use crate::registry::local_cache_path; +use ic_management_types::Network; use serde::de::DeserializeOwned; +use std::path::PathBuf; +use tokio::io::{AsyncReadExt, BufReader}; +use tokio::{fs::File, io::AsyncWriteExt}; -pub async fn query_ic_dashboard_list(path: &str) -> anyhow::Result { - let client = reqwest::Client::new(); - let result = client - .get(format!("https://ic-api.internetcomputer.org/api/{}", &path)) - .send() - .await - .and_then(|r| r.error_for_status()); - match result { - Ok(response) => match response.json::().await { - Ok(data) => Ok(data), - Err(e) => Err(anyhow::format_err!("failed to parse response: {}", e)), - }, - Err(e) => Err(anyhow::format_err!("failed to fetch response: {}", e)), +const IC_DASHBOARD_API: &str = "https://ic-api.internetcomputer.org/api/"; +const IC_API_REFRESH_INTERVAL_SECONDS: u64 = 60 * 60; // 1h + +pub async fn query_ic_dashboard_list(network: &Network, query_what: &str) -> anyhow::Result { + let local_cache_file_path = local_cache_path() + .join(PathBuf::from(query_what).file_name().unwrap()) + .with_extension("json"); + // if there is a local cache, use it if either: + // - file was last updated before IC_API_REFRESH_INTERVAL_SECONDS, or + // - we target a network other than the mainnet: IC dashboard has data only for the mainnet, so data does not matter + let data = if local_cache_file_path.exists() + && (!network.is_mainnet() + || local_cache_file_path.metadata().unwrap().modified().unwrap().elapsed().unwrap().as_secs() < IC_API_REFRESH_INTERVAL_SECONDS) + { + let file = File::open(&local_cache_file_path).await?; + let mut buf = Vec::new(); + BufReader::new(file).read_to_end(&mut buf).await?; + buf + } else { + let client = reqwest::Client::new(); + client + .get(format!("{}/{}", IC_DASHBOARD_API, &query_what)) + .send() + .await + .and_then(|r| r.error_for_status()) + .map_err(|e| anyhow::format_err!("failed to fetch response: {}", e))? + .bytes() + .await? + .to_vec() + }; + match serde_json::from_slice(data.as_slice()) { + Ok(result) => { + let mut file = File::create(&local_cache_file_path).await?; + file.write_all(&data).await?; + Ok(result) + } + Err(e) => Err(anyhow::format_err!("failed to parse as json: {}", e)), } } diff --git a/rs/ic-management-backend/src/registry.rs b/rs/ic-management-backend/src/registry.rs index 29e6d7638..e34de9925 100644 --- a/rs/ic-management-backend/src/registry.rs +++ b/rs/ic-management-backend/src/registry.rs @@ -871,7 +871,7 @@ fn node_ip_addr(nr: &NodeRecord) -> Ipv6Addr { Ipv6Addr::from_str(&nr.http.clone().expect("missing ipv6 address").ip_addr).expect("invalid ipv6 address") } -pub fn local_registry_path(network: &Network) -> PathBuf { +pub fn local_cache_path() -> PathBuf { match std::env::var("LOCAL_REGISTRY_PATH") { Ok(path) => PathBuf::from(path), Err(_) => match dirs::cache_dir() { @@ -880,8 +880,10 @@ pub fn local_registry_path(network: &Network) -> PathBuf { }, } .join("ic-registry-cache") - .join(Path::new(network.name.as_str())) - .join("local_registry") +} + +pub fn local_registry_path(network: &Network) -> PathBuf { + local_cache_path().join(Path::new(network.name.as_str())).join("local_registry") } pub async fn nns_public_key(registry_canister: &RegistryCanister) -> anyhow::Result { @@ -1061,7 +1063,8 @@ async fn fetch_and_add_node_labels_guests_to_registry(target_network: &Network, } pub async fn update_node_details(registry_state: &Arc>) { - match query_ic_dashboard_list::("v3/node-providers").await { + let network = registry_state.read().await.network(); + match query_ic_dashboard_list::(&network, "v3/node-providers").await { Ok(node_providers_response) => { let mut registry_state = registry_state.write().await; let update = registry_state.update_node_details(&node_providers_response.node_providers).await; diff --git a/rs/np-notifications/src/health_check.rs b/rs/np-notifications/src/health_check.rs index dda36f8f5..fca7448a9 100644 --- a/rs/np-notifications/src/health_check.rs +++ b/rs/np-notifications/src/health_check.rs @@ -34,7 +34,7 @@ pub async fn start_health_check_loop(config: HealthCheckLoopConfig) { let network = ic_management_types::Network::new("mainnet", &vec![]) .await .expect("failed to create mainnet network"); - let hc = HealthClient::new(network); + let hc = HealthClient::new(network.clone()); let mut nodes_status = NodesStatus::from(hc.nodes().await.unwrap()); let mut rs = config.registry_state; @@ -47,7 +47,7 @@ pub async fn start_health_check_loop(config: HealthCheckLoopConfig) { // reboots. In the worst case, if a provider is not up to date in the list, // the program will crash, then restart and update the list, which should // fix the issue. - let node_providers = query_ic_dashboard_list::("v3/node-providers") + let node_providers = query_ic_dashboard_list::(&network, "v3/node-providers") .await .unwrap() .node_providers;