feat(dre): Cache public dashboard API response for 1h (#506)

rs/cli/src/registry_dump.rs

@@ -65,7 +65,7 @@ async fn get_registry(path: &Option<PathBuf>, network: &Network, version: &i64)
     let elected_host_os_versions = get_elected_host_os_versions(&local_registry, version)?;
 
     let node_provider_names: HashMap<PrincipalId, String> = HashMap::from_iter(
-        query_ic_dashboard_list::<NodeProvidersResponse>("v3/node-providers")
+        query_ic_dashboard_list::<NodeProvidersResponse>(network, "v3/node-providers")
             .await?
             .node_providers
             .iter()

rs/cli/src/runner.rs

@@ -191,7 +191,7 @@ impl Runner {
 
         let dashboard_backend_client = DashboardBackendClient::new_with_backend_url(backend_url);
         let mut registry = registry::RegistryState::new(network, true).await;
-        let node_providers = query_ic_dashboard_list::<NodeProvidersResponse>("v3/node-providers")
+        let node_providers = query_ic_dashboard_list::<NodeProvidersResponse>(network, "v3/node-providers")
             .await?
             .node_providers;
         registry.update_node_details(&node_providers).await?;
@@ -211,7 +211,7 @@ impl Runner {
         let dashboard_backend_client = DashboardBackendClient::new_with_backend_url(backend_url);
 
         let mut registry = registry::RegistryState::new(network, true).await;
-        let node_providers = query_ic_dashboard_list::<NodeProvidersResponse>("v3/node-providers")
+        let node_providers = query_ic_dashboard_list::<NodeProvidersResponse>(network, "v3/node-providers")
             .await?
             .node_providers;
         registry.update_node_details(&node_providers).await?;

rs/ic-management-backend/src/public_dashboard.rs

@@ -1,17 +1,46 @@
+use crate::registry::local_cache_path;
+use ic_management_types::Network;
 use serde::de::DeserializeOwned;
+use std::path::PathBuf;
+use tokio::io::{AsyncReadExt, BufReader};
+use tokio::{fs::File, io::AsyncWriteExt};
 
-pub async fn query_ic_dashboard_list<T: DeserializeOwned>(path: &str) -> anyhow::Result<T> {
-    let client = reqwest::Client::new();
-    let result = client
-        .get(format!("https://ic-api.internetcomputer.org/api/{}", &path))
-        .send()
-        .await
-        .and_then(|r| r.error_for_status());
-    match result {
-        Ok(response) => match response.json::<T>().await {
-            Ok(data) => Ok(data),
-            Err(e) => Err(anyhow::format_err!("failed to parse response: {}", e)),
-        },
-        Err(e) => Err(anyhow::format_err!("failed to fetch response: {}", e)),
+const IC_DASHBOARD_API: &str = "https://ic-api.internetcomputer.org/api/";
+const IC_API_REFRESH_INTERVAL_SECONDS: u64 = 60 * 60; // 1h
+
+pub async fn query_ic_dashboard_list<T: DeserializeOwned>(network: &Network, query_what: &str) -> anyhow::Result<T> {
+    let local_cache_file_path = local_cache_path()
+        .join(PathBuf::from(query_what).file_name().unwrap())
+        .with_extension("json");
+    // if there is a local cache, use it if either:
+    // - file was last updated before IC_API_REFRESH_INTERVAL_SECONDS, or
+    // - we target a network other than the mainnet: IC dashboard has data only for the mainnet, so data does not matter
+    let data = if local_cache_file_path.exists()
+        && (!network.is_mainnet()
+            || local_cache_file_path.metadata().unwrap().modified().unwrap().elapsed().unwrap().as_secs() < IC_API_REFRESH_INTERVAL_SECONDS)
+    {
+        let file = File::open(&local_cache_file_path).await?;
+        let mut buf = Vec::new();
+        BufReader::new(file).read_to_end(&mut buf).await?;
+        buf
+    } else {
+        let client = reqwest::Client::new();
+        client
+            .get(format!("{}/{}", IC_DASHBOARD_API, &query_what))
+            .send()
+            .await
+            .and_then(|r| r.error_for_status())
+            .map_err(|e| anyhow::format_err!("failed to fetch response: {}", e))?
+            .bytes()
+            .await?
+            .to_vec()
+    };
+    match serde_json::from_slice(data.as_slice()) {
+        Ok(result) => {
+            let mut file = File::create(&local_cache_file_path).await?;
+            file.write_all(&data).await?;
+            Ok(result)
+        }
+        Err(e) => Err(anyhow::format_err!("failed to parse as json: {}", e)),
     }
 }

rs/ic-management-backend/src/registry.rs

@@ -871,7 +871,7 @@ fn node_ip_addr(nr: &NodeRecord) -> Ipv6Addr {
     Ipv6Addr::from_str(&nr.http.clone().expect("missing ipv6 address").ip_addr).expect("invalid ipv6 address")
 }
 
-pub fn local_registry_path(network: &Network) -> PathBuf {
+pub fn local_cache_path() -> PathBuf {
     match std::env::var("LOCAL_REGISTRY_PATH") {
         Ok(path) => PathBuf::from(path),
         Err(_) => match dirs::cache_dir() {
@@ -880,8 +880,10 @@ pub fn local_registry_path(network: &Network) -> PathBuf {
         },
     }
     .join("ic-registry-cache")
-    .join(Path::new(network.name.as_str()))
-    .join("local_registry")
+}
+
+pub fn local_registry_path(network: &Network) -> PathBuf {
+    local_cache_path().join(Path::new(network.name.as_str())).join("local_registry")
 }
 
 pub async fn nns_public_key(registry_canister: &RegistryCanister) -> anyhow::Result<ThresholdSigPublicKey> {
@@ -1061,7 +1063,8 @@ async fn fetch_and_add_node_labels_guests_to_registry(target_network: &Network,
 }
 
 pub async fn update_node_details(registry_state: &Arc<RwLock<RegistryState>>) {
-    match query_ic_dashboard_list::<NodeProvidersResponse>("v3/node-providers").await {
+    let network = registry_state.read().await.network();
+    match query_ic_dashboard_list::<NodeProvidersResponse>(&network, "v3/node-providers").await {
         Ok(node_providers_response) => {
             let mut registry_state = registry_state.write().await;
             let update = registry_state.update_node_details(&node_providers_response.node_providers).await;

rs/np-notifications/src/health_check.rs

@@ -34,7 +34,7 @@ pub async fn start_health_check_loop(config: HealthCheckLoopConfig) {
     let network = ic_management_types::Network::new("mainnet", &vec![])
         .await
         .expect("failed to create mainnet network");
-    let hc = HealthClient::new(network);
+    let hc = HealthClient::new(network.clone());
     let mut nodes_status = NodesStatus::from(hc.nodes().await.unwrap());
 
     let mut rs = config.registry_state;
@@ -47,7 +47,7 @@ pub async fn start_health_check_loop(config: HealthCheckLoopConfig) {
     // reboots. In the worst case, if a provider is not up to date in the list,
     // the program will crash, then restart and update the list, which should
     // fix the issue.
-    let node_providers = query_ic_dashboard_list::<NodeProvidersResponse>("v3/node-providers")
+    let node_providers = query_ic_dashboard_list::<NodeProvidersResponse>(&network, "v3/node-providers")
         .await
         .unwrap()
         .node_providers;