Function call success and error metrics (#7613)

Provides visibility into the success rate of processing function call actions.
In case of errors, this PR also counts the error types and sub-types where it makes sense.

Would be useful to have `shard_id` in these metrics, but the runtime is not supposed to know anything about epochs, shard layouts, etc.

```
# HELP near_function_call_processed The number of function calls processed since starting this node
# TYPE near_function_call_processed counter
near_function_call_processed{result="FunctionCallError"} 17
near_function_call_processed{result="ok"} 1088
# HELP near_function_call_processed_function_call_errors The number of function calls resulting in function call errors, since starting this node
# TYPE near_function_call_processed_function_call_errors counter
near_function_call_processed_function_call_errors{error_type="HostError"} 17
# HELP near_function_call_processed_host_errors The number of function calls resulting in host errors, since starting this node
# TYPE near_function_call_processed_host_errors counter
near_function_call_processed_host_errors{error_type="GasLimitExceeded"} 5
near_function_call_processed_host_errors{error_type="GuestPanic"} 12
```

runtime/near-vm-errors/Cargo.toml

@@ -15,12 +15,13 @@ Error that can occur inside Near Runtime encapsulated in a separate crate. Might
 """
 
 [dependencies]
-serde = { version = "1", features = ["derive"] }
+near-account-id = { path = "../../core/account-id" }
+near-rpc-error-macro = { path = "../../tools/rpctypegen/macro" }
 
 borsh = "0.9"
 deepsize = { version = "0.2.0", optional = true }
-near-account-id = { path = "../../core/account-id" }
-near-rpc-error-macro = { path = "../../tools/rpctypegen/macro" }
+serde = { version = "1", features = ["derive"] }
+strum = { version = "0.24", features = ["derive"] }
 
 [features]
 dump_errors_schema = ["near-rpc-error-macro/dump_errors_schema"]

runtime/near-vm-errors/src/lib.rs

@@ -7,7 +7,7 @@ use serde::{Deserialize, Serialize};
 use std::any::Any;
 use std::fmt::{self, Error, Formatter};
 
-#[derive(Debug, PartialEq, Eq)]
+#[derive(Debug, PartialEq, Eq, strum::IntoStaticStr)]
 pub enum VMError {
     FunctionCallError(FunctionCallError),
     /// Type erased error from `External` trait implementation.
@@ -19,7 +19,7 @@ pub enum VMError {
     CacheError(CacheError),
 }
 
-#[derive(Debug, PartialEq, Eq)]
+#[derive(Debug, PartialEq, Eq, strum::IntoStaticStr)]
 pub enum FunctionCallError {
     /// Wasm compilation error
     CompilationError(CompilationError),
@@ -67,7 +67,7 @@ pub enum FunctionCallErrorSer {
     ExecutionError(String),
 }
 
-#[derive(Debug, PartialEq, Eq)]
+#[derive(Debug, PartialEq, Eq, strum::IntoStaticStr)]
 pub enum CacheError {
     ReadError,
     WriteError,
@@ -77,7 +77,16 @@ pub enum CacheError {
 /// A kind of a trap happened during execution of a binary
 #[cfg_attr(feature = "deepsize_feature", derive(deepsize::DeepSizeOf))]
 #[derive(
-    Debug, Clone, PartialEq, Eq, BorshDeserialize, BorshSerialize, Deserialize, Serialize, RpcError,
+    Debug,
+    Clone,
+    PartialEq,
+    Eq,
+    BorshDeserialize,
+    BorshSerialize,
+    Deserialize,
+    Serialize,
+    RpcError,
+    strum::IntoStaticStr,
 )]
 pub enum WasmTrap {
     /// An `unreachable` opcode was executed.
@@ -102,7 +111,16 @@ pub enum WasmTrap {
 
 #[cfg_attr(feature = "deepsize_feature", derive(deepsize::DeepSizeOf))]
 #[derive(
-    Debug, Clone, PartialEq, Eq, BorshDeserialize, BorshSerialize, Deserialize, Serialize, RpcError,
+    Debug,
+    Clone,
+    PartialEq,
+    Eq,
+    BorshDeserialize,
+    BorshSerialize,
+    Deserialize,
+    Serialize,
+    RpcError,
+    strum::IntoStaticStr,
 )]
 pub enum MethodResolveError {
     MethodEmptyName,
@@ -112,7 +130,16 @@ pub enum MethodResolveError {
 
 #[cfg_attr(feature = "deepsize_feature", derive(deepsize::DeepSizeOf))]
 #[derive(
-    Debug, Clone, PartialEq, Eq, BorshDeserialize, BorshSerialize, Deserialize, Serialize, RpcError,
+    Debug,
+    Clone,
+    PartialEq,
+    Eq,
+    BorshDeserialize,
+    BorshSerialize,
+    Deserialize,
+    Serialize,
+    RpcError,
+    strum::IntoStaticStr,
 )]
 pub enum CompilationError {
     CodeDoesNotExist { account_id: AccountId },
@@ -156,7 +183,16 @@ pub enum PrepareError {
 
 #[cfg_attr(feature = "deepsize_feature", derive(deepsize::DeepSizeOf))]
 #[derive(
-    Debug, Clone, PartialEq, Eq, BorshDeserialize, BorshSerialize, Deserialize, Serialize, RpcError,
+    Debug,
+    Clone,
+    PartialEq,
+    Eq,
+    BorshDeserialize,
+    BorshSerialize,
+    Deserialize,
+    Serialize,
+    RpcError,
+    strum::IntoStaticStr,
 )]
 pub enum HostError {
     /// String encoding is bad UTF-16 sequence

runtime/runtime/src/actions.rs

@@ -1,6 +1,6 @@
 use crate::config::{safe_add_gas, RuntimeConfig};
 use crate::ext::{ExternalError, RuntimeExt};
-use crate::{ActionResult, ApplyState};
+use crate::{metrics, ActionResult, ApplyState};
 use borsh::{BorshDeserialize, BorshSerialize};
 use near_crypto::PublicKey;
 use near_primitives::account::{AccessKey, AccessKeyPermission, Account};
@@ -168,45 +168,79 @@ pub(crate) fn action_function_call(
         None,
     )
     .outcome_error();
+
+    match &err {
+        None => {
+            metrics::FUNCTION_CALL_PROCESSED.with_label_values(&["ok"]).inc();
+        }
+        Some(err) => {
+            metrics::FUNCTION_CALL_PROCESSED.with_label_values(&[err.into()]).inc();
+        }
+    }
+
     let execution_succeeded = match err {
-        Some(VMError::FunctionCallError(err)) => match err {
-            FunctionCallError::Nondeterministic(msg) => {
-                panic!("Contract runner returned non-deterministic error '{}', aborting", msg)
-            }
-            FunctionCallError::WasmUnknownError { debug_message } => {
-                panic!("Wasmer returned unknown message: {}", debug_message)
-            }
-            FunctionCallError::CompilationError(err) => {
-                result.result = Err(ActionErrorKind::FunctionCallError(
-                    ContractCallError::CompilationError(err).into(),
-                )
-                .into());
-                false
-            }
-            FunctionCallError::LinkError { msg } => {
-                result.result = Err(ActionErrorKind::FunctionCallError(
-                    ContractCallError::ExecutionError { msg: format!("Link Error: {}", msg) }
-                        .into(),
-                )
-                .into());
-                false
-            }
-            FunctionCallError::MethodResolveError(err) => {
-                result.result = Err(ActionErrorKind::FunctionCallError(
-                    ContractCallError::MethodResolveError(err).into(),
-                )
-                .into());
-                false
-            }
-            FunctionCallError::WasmTrap(_) | FunctionCallError::HostError(_) => {
-                result.result = Err(ActionErrorKind::FunctionCallError(
-                    ContractCallError::ExecutionError { msg: err.to_string() }.into(),
-                )
-                .into());
-                false
+        Some(VMError::FunctionCallError(err)) => {
+            metrics::FUNCTION_CALL_PROCESSED_FUNCTION_CALL_ERRORS
+                .with_label_values(&[(&err).into()])
+                .inc();
+            match err {
+                FunctionCallError::Nondeterministic(msg) => {
+                    panic!("Contract runner returned non-deterministic error '{}', aborting", msg)
+                }
+                FunctionCallError::WasmUnknownError { debug_message } => {
+                    panic!("Wasmer returned unknown message: {}", debug_message)
+                }
+                FunctionCallError::CompilationError(err) => {
+                    metrics::FUNCTION_CALL_PROCESSED_COMPILATION_ERRORS
+                        .with_label_values(&[(&err).into()])
+                        .inc();
+                    result.result = Err(ActionErrorKind::FunctionCallError(
+                        ContractCallError::CompilationError(err).into(),
+                    )
+                    .into());
+                    false
+                }
+                FunctionCallError::LinkError { msg } => {
+                    result.result = Err(ActionErrorKind::FunctionCallError(
+                        ContractCallError::ExecutionError { msg: format!("Link Error: {}", msg) }
+                            .into(),
+                    )
+                    .into());
+                    false
+                }
+                FunctionCallError::MethodResolveError(err) => {
+                    metrics::FUNCTION_CALL_PROCESSED_METHOD_RESOLVE_ERRORS
+                        .with_label_values(&[(&err).into()])
+                        .inc();
+                    result.result = Err(ActionErrorKind::FunctionCallError(
+                        ContractCallError::MethodResolveError(err).into(),
+                    )
+                    .into());
+                    false
+                }
+                FunctionCallError::WasmTrap(ref inner_err) => {
+                    metrics::FUNCTION_CALL_PROCESSED_WASM_TRAP_ERRORS
+                        .with_label_values(&[inner_err.into()])
+                        .inc();
+                    result.result = Err(ActionErrorKind::FunctionCallError(
+                        ContractCallError::ExecutionError { msg: err.to_string() }.into(),
+                    )
+                    .into());
+                    false
+                }
+                FunctionCallError::HostError(ref inner_err) => {
+                    metrics::FUNCTION_CALL_PROCESSED_HOST_ERRORS
+                        .with_label_values(&[inner_err.into()])
+                        .inc();
+                    result.result = Err(ActionErrorKind::FunctionCallError(
+                        ContractCallError::ExecutionError { msg: err.to_string() }.into(),
+                    )
+                    .into());
+                    false
+                }
+                FunctionCallError::_EVMError => unreachable!(),
             }
-            FunctionCallError::_EVMError => unreachable!(),
-        },
+        }
         Some(VMError::ExternalError(any_err)) => {
             let err: ExternalError =
                 any_err.downcast().expect("Downcasting AnyError should not fail");
@@ -219,6 +253,7 @@ pub(crate) fn action_function_call(
             return Err(StorageError::StorageInconsistentState(err.to_string()).into());
         }
         Some(VMError::CacheError(err)) => {
+            metrics::FUNCTION_CALL_PROCESSED_CACHE_ERRORS.with_label_values(&[(&err).into()]).inc();
             let message = match err {
                 CacheError::DeserializationError => "Cache deserialization error",
                 CacheError::SerializationError { hash: _hash } => "Cache serialization error",

runtime/runtime/src/metrics.rs

@@ -33,3 +33,59 @@ pub static TRANSACTION_PROCESSED_FAILED_TOTAL: Lazy<IntCounter> = Lazy::new(|| {
     )
     .unwrap()
 });
+pub static FUNCTION_CALL_PROCESSED: Lazy<IntCounterVec> = Lazy::new(|| {
+    try_create_int_counter_vec(
+        "near_function_call_processed",
+        "The number of function calls processed since starting this node",
+        &["result"],
+    )
+    .unwrap()
+});
+pub static FUNCTION_CALL_PROCESSED_FUNCTION_CALL_ERRORS: Lazy<IntCounterVec> = Lazy::new(|| {
+    try_create_int_counter_vec(
+        "near_function_call_processed_function_call_errors",
+        "The number of function calls resulting in function call errors, since starting this node",
+        &["error_type"],
+    )
+    .unwrap()
+});
+pub static FUNCTION_CALL_PROCESSED_COMPILATION_ERRORS: Lazy<IntCounterVec> = Lazy::new(|| {
+    try_create_int_counter_vec(
+        "near_function_call_processed_compilation_errors",
+        "The number of function calls resulting in compilation errors, since starting this node",
+        &["error_type"],
+    )
+    .unwrap()
+});
+pub static FUNCTION_CALL_PROCESSED_METHOD_RESOLVE_ERRORS: Lazy<IntCounterVec> = Lazy::new(|| {
+    try_create_int_counter_vec(
+        "near_function_call_processed_method_resolve_errors",
+        "The number of function calls resulting in method resolve errors, since starting this node",
+        &["error_type"],
+    )
+    .unwrap()
+});
+pub static FUNCTION_CALL_PROCESSED_WASM_TRAP_ERRORS: Lazy<IntCounterVec> = Lazy::new(|| {
+    try_create_int_counter_vec(
+        "near_function_call_processed_wasm_trap_errors",
+        "The number of function calls resulting in wasm trap errors, since starting this node",
+        &["error_type"],
+    )
+    .unwrap()
+});
+pub static FUNCTION_CALL_PROCESSED_HOST_ERRORS: Lazy<IntCounterVec> = Lazy::new(|| {
+    try_create_int_counter_vec(
+        "near_function_call_processed_host_errors",
+        "The number of function calls resulting in host errors, since starting this node",
+        &["error_type"],
+    )
+    .unwrap()
+});
+pub static FUNCTION_CALL_PROCESSED_CACHE_ERRORS: Lazy<IntCounterVec> = Lazy::new(|| {
+    try_create_int_counter_vec(
+        "near_function_call_processed_cache_errors",
+        "The number of function calls resulting in VM cache errors, since starting this node",
+        &["error_type"],
+    )
+    .unwrap()
+});