diff --git a/ci/install-check.sh b/ci/install-check.sh index 628d6d229..95d17c9e0 100755 --- a/ci/install-check.sh +++ b/ci/install-check.sh @@ -23,12 +23,28 @@ CHECK_VERSION=0.12.0 CHECK_TARBALL="check-${CHECK_VERSION}.tar.gz" CHECK_DIR="check-${CHECK_VERSION}" +echo "building and installing check" >&2 + ( cd "$TEMP" && wget "https://github.com/libcheck/check/releases/download/${CHECK_VERSION}/${CHECK_TARBALL}" && - tar xvfz "${CHECK_TARBALL}" && + tar xfz "${CHECK_TARBALL}" && cd "${CHECK_DIR}" && ./configure --prefix="$CHECK_PREFIX" && make && make install -) || die "check build failed" +) >$TEMP/cmake-build.log 2>&1 + +RESULT=$? +if [[ $RESULT -ne 0 ]]; then + cat >&2 <&2 +fi + +exit $RESULT diff --git a/ci/local-run.sh b/ci/local-run.sh new file mode 100755 index 000000000..36b7146d9 --- /dev/null +++ b/ci/local-run.sh @@ -0,0 +1,55 @@ +#!/bin/bash + +## This file is potentially useful for doing a clean environment build on MacOS ## +## This is my personal way of running the build, YMMV - jsimms ## + +set -euo pipefail +IFS=$'\n\t' + +die() { echo "fatal: $*" >&2; exit 1; } + +TOPLEVEL=$(git -C "$(cd "$(dirname "$0")" >/dev/null || exit 1; pwd)" rev-parse --show-toplevel) || die "TOPLEVEL fail" + +cd "$TOPLEVEL" + +TEMP="$(mktemp -d -t TEMP.XXXXXXX)" || die "failed to make tmpdir" +cleanup() { [[ -n "${TEMP:-}" ]] && rm -rf "${TEMP}"; } +trap cleanup EXIT + +BUILD_PATH=( + "$HOME/.cargo/bin" + "/usr/local/bin" + "/usr/local/sbin" + "/usr/bin" + "/usr/sbin" + "/bin" + "/sbin" + "/opt/X11/bin" + "/usr/X11R6/bin" +) + +PATH=$(echo "${BUILD_PATH[@]}"|tr ' ' ':') + +cat >&2 < +#include + +/* NOTE: for documentation see ccommon/rust/ccommon_rs/src/log.rs */ + +typedef enum log_level_rs { + LOG_LEVEL_ERROR = 1, + LOG_LEVEL_WARN, + LOG_LEVEL_INFO, + LOG_LEVEL_DEBUG, + LOG_LEVEL_TRACE, +} log_level_rs_e; + + +typedef enum log_status_rs { + /* Good work! */ + LOG_STATUS_OK = 0, + /* An action that requires log_rs_is_setup() to be true, but it isn't + * i.e. you need to call log_rs_setup() before whatever you just tried to do. */ + LOG_STATUS_NOT_SETUP_ERROR, + /* We could not register as the backend for the log crate . + * This state is unrecoverable. */ + LOG_STATUS_REGISTRATION_FAIL, + /* Returned when there is already a logger set up for rust. */ + LOG_STATUS_ALREADY_SET_ERROR, + /* Data was expected to be valid UTF8 but was not */ + LOG_STATUS_INVALID_UTF8, + /* Failed to create a logger instance */ + LOG_STATUS_CREATION_ERROR, + /* An unexpected error occurred, check stderr */ + LOG_STATUS_OTHER_FAILURE, + /* You suck at programming */ + LOG_STATUS_NULL_POINTER_ERROR, +} log_status_rs_e; + +struct log_config_rs { + uint32_t buf_size; + log_level_rs_e level; + struct bstring path; + struct bstring prefix; +}; + +struct log_handle_rs; + +struct log_handle_rs* log_create_handle_rs(struct log_config_rs *cfg); +log_status_rs_e log_shutdown_rs(struct log_handle_rs *handle, uint32_t timeout_ms); +void log_destroy_handle_rs(struct log_handle_rs **h); + +bool log_is_setup_rs(struct log_handle_rs *handle); + +#ifdef __cplusplus +} +#endif diff --git a/include/rust/cc_util_rs.h b/include/rust/cc_util_rs.h new file mode 100644 index 000000000..bb354c129 --- /dev/null +++ b/include/rust/cc_util_rs.h @@ -0,0 +1,33 @@ +/* + * ccommon - a cache common library. + * Copyright (C) 2018 Twitter, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#ifdef __cplusplus +extern "C" { +#endif + +/* recursively remove all content under path then unlinks path. + * returns 0 on success, -1 on failure and sets errno. +*/ +int +cc_util_rm_rf_rs(const char *path); + + +#ifdef __cplusplus +} +#endif diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 5ae870301..d605d009e 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -1,6 +1,6 @@ [[package]] name = "aho-corasick" -version = "0.6.5" +version = "0.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "memchr 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)", @@ -41,7 +41,7 @@ name = "backtrace-sys" version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "cc 1.0.17 (registry+https://github.com/rust-lang/crates.io-index)", + "cc 1.0.18 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -55,7 +55,7 @@ dependencies = [ "clang-sys 0.23.0 (registry+https://github.com/rust-lang/crates.io-index)", "clap 2.32.0 (registry+https://github.com/rust-lang/crates.io-index)", "env_logger 0.5.10 (registry+https://github.com/rust-lang/crates.io-index)", - "lazy_static 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)", "peeking_take_while 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", "proc-macro2 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", @@ -71,7 +71,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "cc" -version = "1.0.17" +version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] @@ -87,9 +87,18 @@ name = "ccommon_rs" version = "0.1.0" dependencies = [ "cc_binding 0.1.0", + "chrono 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)", + "crossbeam 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", "failure 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "failure_derive 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)", + "rusty-fork 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "tempfile 3.0.3 (registry+https://github.com/rust-lang/crates.io-index)", + "thread-id 3.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "thread_local 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", + "time 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -105,6 +114,16 @@ name = "cfg-if" version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "chrono" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "num-integer 0.1.39 (registry+https://github.com/rust-lang/crates.io-index)", + "num-traits 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)", + "time 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "clang-sys" version = "0.23.0" @@ -129,6 +148,19 @@ dependencies = [ "vec_map 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "cloudabi" +version = "0.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "bitflags 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "crossbeam" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "env_logger" version = "0.5.10" @@ -160,6 +192,25 @@ dependencies = [ "synstructure 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "fnv" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "fuchsia-zircon" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "bitflags 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)", + "fuchsia-zircon-sys 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "fuchsia-zircon-sys" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "glob" version = "0.2.11" @@ -175,7 +226,7 @@ dependencies = [ [[package]] name = "lazy_static" -version = "1.0.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] @@ -188,7 +239,7 @@ name = "libloading" version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "cc 1.0.17 (registry+https://github.com/rust-lang/crates.io-index)", + "cc 1.0.18 (registry+https://github.com/rust-lang/crates.io-index)", "winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -224,6 +275,19 @@ dependencies = [ "memchr 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "num-integer" +version = "0.1.39" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "num-traits 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "num-traits" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "peeking_take_while" version = "0.1.2" @@ -255,6 +319,23 @@ dependencies = [ "proc-macro2 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "rand" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cloudabi 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)", + "fuchsia-zircon 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_core 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rand_core" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "redox_syscall" version = "0.1.40" @@ -273,7 +354,7 @@ name = "regex" version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "aho-corasick 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)", + "aho-corasick 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)", "memchr 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)", "regex-syntax 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)", "thread_local 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", @@ -288,11 +369,30 @@ dependencies = [ "ucd-util 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "remove_dir_all" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "rustc-demangle" version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "rusty-fork" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", + "quick-error 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "tempfile 3.0.3 (registry+https://github.com/rust-lang/crates.io-index)", + "wait-timeout 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "strsim" version = "0.7.0" @@ -325,6 +425,18 @@ dependencies = [ "syn 0.11.11 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "tempfile" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)", + "rand 0.5.4 (registry+https://github.com/rust-lang/crates.io-index)", + "redox_syscall 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)", + "remove_dir_all 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "termcolor" version = "0.3.6" @@ -351,15 +463,35 @@ dependencies = [ "unicode-width 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "thread-id" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)", + "redox_syscall 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "thread_local" version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "lazy_static 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", "unreachable 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "time" +version = "0.1.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)", + "redox_syscall 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "ucd-util" version = "0.1.1" @@ -403,6 +535,14 @@ name = "void" version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "wait-timeout" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "which" version = "1.0.5" @@ -439,48 +579,63 @@ dependencies = [ ] [metadata] -"checksum aho-corasick 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)" = "f0ba20154ea1f47ce2793322f049c5646cc6d0fa9759d5f333f286e507bf8080" +"checksum aho-corasick 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)" = "c1c6d463cbe7ed28720b5b489e7c083eeb8f90d08be2a0d6bb9e1ffea9ce1afa" "checksum ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b" "checksum atty 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)" = "2fc4a1aa4c24c0718a250f0681885c1af91419d242f29eb8f2ab28502d80dbd1" "checksum backtrace 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)" = "89a47830402e9981c5c41223151efcced65a0510c13097c769cede7efb34782a" "checksum backtrace-sys 0.1.23 (registry+https://github.com/rust-lang/crates.io-index)" = "bff67d0c06556c0b8e6b5f090f0eac52d950d9dfd1d35ba04e4ca3543eaf6a7e" "checksum bindgen 0.37.4 (registry+https://github.com/rust-lang/crates.io-index)" = "1b25ab82877ea8fe6ce1ce1f8ac54361f0218bad900af9eb11803994bf67c221" "checksum bitflags 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "d0c54bb8f454c567f21197eefcdbf5679d0bd99f2ddbe52e84c77061952e6789" -"checksum cc 1.0.17 (registry+https://github.com/rust-lang/crates.io-index)" = "49ec142f5768efb5b7622aebc3fdbdbb8950a4b9ba996393cb76ef7466e8747d" +"checksum cc 1.0.18 (registry+https://github.com/rust-lang/crates.io-index)" = "2119ea4867bd2b8ed3aecab467709720b2d55b1bcfe09f772fd68066eaf15275" "checksum cexpr 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "42aac45e9567d97474a834efdee3081b3c942b2205be932092f53354ce503d6c" "checksum cfg-if 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "efe5c877e17a9c717a0bf3613b2709f723202c4e4675cc8f12926ded29bcb17e" +"checksum chrono 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)" = "6962c635d530328acc53ac6a955e83093fedc91c5809dfac1fa60fa470830a37" "checksum clang-sys 0.23.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d7f7c04e52c35222fffcc3a115b5daf5f7e2bfb71c13c4e2321afe1fc71859c2" "checksum clap 2.32.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b957d88f4b6a63b9d70d5f454ac8011819c6efa7727858f458ab71c756ce2d3e" +"checksum cloudabi 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f" +"checksum crossbeam 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "24ce9782d4d5c53674646a6a4c1863a21a8fc0cb649b3c94dfc16e45071dea19" "checksum env_logger 0.5.10 (registry+https://github.com/rust-lang/crates.io-index)" = "0e6e40ebb0e66918a37b38c7acab4e10d299e0463fe2af5d29b9cc86710cfd2a" "checksum failure 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "934799b6c1de475a012a02dab0ace1ace43789ee4b99bcfbf1a2e3e8ced5de82" "checksum failure_derive 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "c7cdda555bb90c9bb67a3b670a0f42de8e73f5981524123ad8578aafec8ddb8b" +"checksum fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)" = "2fad85553e09a6f881f739c29f0b00b0f01357c743266d478b68951ce23285f3" +"checksum fuchsia-zircon 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "2e9763c69ebaae630ba35f74888db465e49e259ba1bc0eda7d06f4a067615d82" +"checksum fuchsia-zircon-sys 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7" "checksum glob 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)" = "8be18de09a56b60ed0edf84bc9df007e30040691af7acd1c41874faac5895bfb" "checksum humantime 1.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0484fda3e7007f2a4a0d9c3a703ca38c71c54c55602ce4660c419fd32e188c9e" -"checksum lazy_static 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "e6412c5e2ad9584b0b8e979393122026cdd6d2a80b933f890dcd694ddbe73739" +"checksum lazy_static 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "fb497c35d362b6a331cfd94956a07fc2c78a4604cdbee844a81170386b996dd3" "checksum libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)" = "b685088df2b950fccadf07a7187c8ef846a959c142338a48f9dc0b94517eb5f1" "checksum libloading 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "9c3ad660d7cb8c5822cd83d10897b0f1f1526792737a179e73896152f85b88c2" "checksum log 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)" = "61bd98ae7f7b754bc53dca7d44b604f733c6bba044ea6f41bc8d89272d8161d2" "checksum memchr 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "148fab2e51b4f1cfc66da2a7c32981d1d3c083a803978268bb11fe4b86925e7a" "checksum memchr 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "796fba70e76612589ed2ce7f45282f5af869e0fdd7cc6199fa1aa1f1d591ba9d" "checksum nom 3.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "05aec50c70fd288702bcd93284a8444607f3292dbdf2a30de5ea5dcdbe72287b" +"checksum num-integer 0.1.39 (registry+https://github.com/rust-lang/crates.io-index)" = "e83d528d2677f0518c570baf2b7abdcf0cd2d248860b68507bdcb3e91d4c0cea" +"checksum num-traits 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)" = "630de1ef5cc79d0cdd78b7e33b81f083cbfe90de0f4b2b2f07f905867c70e9fe" "checksum peeking_take_while 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099" "checksum proc-macro2 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "77997c53ae6edd6d187fec07ec41b207063b5ee6f33680e9fa86d405cdd313d4" "checksum quick-error 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "9274b940887ce9addde99c4eee6b5c44cc494b182b97e73dc8ffdcb3397fd3f0" "checksum quote 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)" = "7a6e920b65c65f10b2ae65c831a81a073a89edd28c7cce89475bff467ab4167a" "checksum quote 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "9949cfe66888ffe1d53e6ec9d9f3b70714083854be20fd5e271b232a017401e8" +"checksum rand 0.5.4 (registry+https://github.com/rust-lang/crates.io-index)" = "12397506224b2f93e6664ffc4f664b29be8208e5157d3d90b44f09b5fae470ea" +"checksum rand_core 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "edecf0f94da5551fc9b492093e30b041a891657db7940ee221f9d2f66e82eef2" "checksum redox_syscall 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)" = "c214e91d3ecf43e9a4e41e578973adeb14b474f2bee858742d127af75a0112b1" "checksum redox_termios 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "7e891cfe48e9100a70a3b6eb652fef28920c117d366339687bd5576160db0f76" "checksum regex 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "13c93d55961981ba9226a213b385216f83ab43bd6ac53ab16b2eeb47e337cf4e" "checksum regex-syntax 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)" = "05b06a75f5217880fc5e905952a42750bf44787e56a6c6d6852ed0992f5e1d54" +"checksum remove_dir_all 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "3488ba1b9a2084d38645c4c08276a1752dcbf2c7130d74f1569681ad5d2799c5" "checksum rustc-demangle 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)" = "76d7ba1feafada44f2d38eed812bd2489a03c0f5abb975799251518b68848649" +"checksum rusty-fork 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ea98d8d2644fd8b4946a2be90e8c6dc52b652e03079c46e134d9815062b9082d" "checksum strsim 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "bb4f380125926a99e52bc279241539c018323fab05ad6368b56f93d9369ff550" "checksum syn 0.11.11 (registry+https://github.com/rust-lang/crates.io-index)" = "d3b891b9015c88c576343b9b3e41c2c11a51c219ef067b264bd9c8aa9b441dad" "checksum synom 0.11.3 (registry+https://github.com/rust-lang/crates.io-index)" = "a393066ed9010ebaed60b9eafa373d4b1baac186dd7e008555b0f702b51945b6" "checksum synstructure 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)" = "3a761d12e6d8dcb4dcf952a7a89b475e3a9d69e4a69307e01a470977642914bd" +"checksum tempfile 3.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "c4b103c6d08d323b92ff42c8ce62abcd83ca8efa7fd5bf7927efefec75f58c76" "checksum termcolor 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "adc4587ead41bf016f11af03e55a624c06568b5a19db4e90fde573d805074f83" "checksum termion 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "689a3bdfaab439fd92bc87df5c4c78417d3cbe537487274e9b0b2dce76e92096" "checksum textwrap 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "307686869c93e71f94da64286f9a9524c0f308a9e1c87a583de8e9c9039ad3f6" +"checksum thread-id 3.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "c7fbf4c9d56b320106cd64fd024dadfa0be7cb4706725fc44a7d7ce952d820c1" "checksum thread_local 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "279ef31c19ededf577bfd12dfae728040a21f635b06a24cd670ff510edd38963" +"checksum time 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)" = "d825be0eb33fda1a7e68012d51e9c7f451dc1a69391e7fdc197060bb8c56667b" "checksum ucd-util 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "fd2be2d6639d0f8fe6cdda291ad456e23629558d466e2789d2c3e9892bda285d" "checksum unicode-width 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "882386231c45df4700b275c7ff55b6f3698780a650026380e72dabe76fa46526" "checksum unicode-xid 0.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "8c1f860d7d29cf02cb2f3f359fd35991af3d30bac52c57d265a3c461074cb4dc" @@ -489,6 +644,7 @@ dependencies = [ "checksum utf8-ranges 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "662fab6525a98beff2921d7f61a39e7d59e0b425ebc7d0d9e66d316e55124122" "checksum vec_map 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)" = "05c78687fb1a80548ae3250346c3db86a80a7cdd77bda190189f2d0a0987c81a" "checksum void 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" +"checksum wait-timeout 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "b9f3bf741a801531993db6478b95682117471f76916f5e690dd8d45395b09349" "checksum which 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)" = "e84a603e7e0b1ce1aa1ee2b109c7be00155ce52df5081590d1ffb93f4f515cb2" "checksum winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "773ef9dcc5f24b7d850d0ff101e542ff24c3b090a9768e03ff889fdef41f00fd" "checksum winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" diff --git a/rust/cc_binding/build.rs b/rust/cc_binding/build.rs index 4e6dc243d..29ff8e627 100644 --- a/rust/cc_binding/build.rs +++ b/rust/cc_binding/build.rs @@ -49,6 +49,9 @@ fn get_cmake_cache_value(binary_dir: &Path, key: &str) -> Result> fn main() { println!("cargo:rustc-link-lib=static=ccommon-1.2.0"); + if cfg!(target_os = "macos") { + println!("cargo:rustc-link-lib=framework=Security"); + } let include_path = fs::canonicalize("./../../include").unwrap(); @@ -76,6 +79,10 @@ fn main() { println!("cargo:rustc-link-search=native={}", lib_dir); + if cfg!(target_os = "macos") { + println!("cargo:rustc-link-search=framework=/System/Library/Frameworks"); + } + let bindings = bindgen::Builder::default() .clang_args(vec![ "-I", include_path.to_str().unwrap(), diff --git a/rust/cc_binding/src/lib.rs b/rust/cc_binding/src/lib.rs index 2cf071191..458966182 100644 --- a/rust/cc_binding/src/lib.rs +++ b/rust/cc_binding/src/lib.rs @@ -1,3 +1,26 @@ +// ccommon - a cache common library. +// Copyright (C) 2018 Twitter, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! This module contains the bindgen created classes. +//! PRO-TIP: If you want to look at the generated code, you can find it with: +//! +//! ```ignore +//! $ find . -name bindgen.rs +//! ``` +//! + #![allow(unknown_lints)] #![allow(clippy)] #![allow(clippy_pedantic)] @@ -6,3 +29,4 @@ #![allow(non_snake_case)] #![allow(dead_code)] include!(concat!(env!("OUT_DIR"), "/bindings.rs")); + diff --git a/rust/cc_binding/wrapper.h b/rust/cc_binding/wrapper.h index c5b21b148..02962be00 100644 --- a/rust/cc_binding/wrapper.h +++ b/rust/cc_binding/wrapper.h @@ -1,2 +1,20 @@ +/* ccommon - a cache common library. + * Copyright (C) 2018 Twitter, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + #include #include +#include +#include diff --git a/rust/ccommon_rs/Cargo.toml b/rust/ccommon_rs/Cargo.toml index 382bf2286..69de89ef3 100644 --- a/rust/ccommon_rs/Cargo.toml +++ b/rust/ccommon_rs/Cargo.toml @@ -3,10 +3,22 @@ name = "ccommon_rs" version = "0.1.0" authors = ["Jonathan Simms "] +[lib] +name = "ccommon_rs" +crate-type = ["lib", "dylib", "rlib", "staticlib"] + [dependencies] cc_binding = { path = "../cc_binding" } - +chrono = "~0.4.4" +crossbeam = "~0.3.2" +failure = "~0.1" +failure_derive = "~0.1" +lazy_static = "~1.0" +libc = "~0.2" log = "~0.4" -libc = "~0.2.42" -failure = "~0.1.1" +rusty-fork = "~0.2.0" +tempfile = "~3.0" +thread-id = "~3.3" +thread_local = "~0.3.5" +time = "~0.1" diff --git a/rust/ccommon_rs/src/bstring.rs b/rust/ccommon_rs/src/bstring.rs index f8588d608..5a33f8078 100644 --- a/rust/ccommon_rs/src/bstring.rs +++ b/rust/ccommon_rs/src/bstring.rs @@ -91,8 +91,21 @@ impl BStr { pub fn as_ptr(&self) -> *mut CCbstring { self as *const _ as *mut _ } + + pub fn from_ref<'a>(ccb: &'a CCbstring) -> &'a Self { + unsafe { Self::from_ptr(ccb as *const CCbstring as *mut _) } + } + + pub fn to_utf8_str<'a>(&'a self) -> super::Result<&'a str> { + str::from_utf8(&self[..]).map_err(|e| e.into()) + } + + pub fn to_utf8_string(&self) -> super::Result { + self.to_utf8_str().map(|x| x.to_owned()) + } } + impl Deref for BStr { type Target = [u8]; @@ -263,6 +276,7 @@ impl BString { /// # Panics /// /// This method will panic if `src.len() != self.len()` + #[allow(dead_code)] #[inline] #[allow(dead_code)] fn copy_from_slice(&mut self, src: &[u8]) { @@ -271,12 +285,12 @@ impl BString { } #[inline] - fn as_bytes(&self) -> &[u8] { + pub fn as_bytes(&self) -> &[u8] { unsafe { raw_ptr_to_bytes(self.0) } } #[inline] - fn as_bytes_mut(&mut self) -> &mut [u8] { + pub fn as_bytes_mut(&mut self) -> &mut [u8] { unsafe { raw_ptr_to_bytes_mut(self.0) } } @@ -284,6 +298,14 @@ impl BString { fn len(&self) -> usize { unsafe { (*self.0).len as usize } } + + pub fn to_utf8_str<'a>(&'a self) -> super::Result<&'a str> { + str::from_utf8(self.as_bytes()).map_err(|e| e.into()) + } + + pub fn to_utf8_string(&self) -> super::Result { + self.to_utf8_str().map(|x| x.to_owned()) + } } impl Debug for BString { diff --git a/rust/ccommon_rs/src/lib.rs b/rust/ccommon_rs/src/lib.rs index 6d035da4d..639b47540 100644 --- a/rust/ccommon_rs/src/lib.rs +++ b/rust/ccommon_rs/src/lib.rs @@ -1,3 +1,44 @@ +// ccommon - a cache common library. +// Copyright (C) 2018 Twitter, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + extern crate cc_binding; +extern crate chrono; +extern crate crossbeam; +#[macro_use] extern crate failure; +#[macro_use] +extern crate failure_derive; +extern crate lazy_static; +#[macro_use] +extern crate log as rslog; +extern crate tempfile; +extern crate time; +extern crate thread_local; +extern crate thread_id; + +#[cfg(test)] +#[macro_use] +extern crate rusty_fork; + +use std::result; + pub mod bstring; +pub mod log; +pub mod util; + +// like how guava provides enhancements for Int as "Ints" +pub mod ptrs; + +pub type Result = result::Result; diff --git a/rust/ccommon_rs/src/log/mod.rs b/rust/ccommon_rs/src/log/mod.rs new file mode 100644 index 000000000..96650c368 --- /dev/null +++ b/rust/ccommon_rs/src/log/mod.rs @@ -0,0 +1,856 @@ +// ccommon - a cache common library. +// Copyright (C) 2018 Twitter, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Threadsafe glue between the `log` crate and `cc_log`. +//! +//! The C side configures this module with a directory and base filename. +//! When a Rust thread calls one of the logging macros, a new logger is +//! created with a unique filename (based either on the thread's name or +//! its posix unique id) and stored in a thread local variable. At shutdown, +//! the struct that refers to the thread-local loggers is atomically +//! swapped out for a no-op logger, and the thread-local loggers are flushed +//! and shut down cleanly. +//! +//! This configuration is a shared-nothing lockless design...for _SPEED_. +//! +//! # Example +//! +//! Basic setup in an app that makes use of ccommon: +//! +//! ```ignore +//! #include +//! #include +//! #include +//! +//! #include +//! +//! #define PATH "/var/log/appname" +//! +//! static struct log_handle_rs *log_handle; +//! static struct log_config_rs log_config; +//! +//! +//! void +//! log_setup() +//! { +//! log_config.buf_size = 1024; +//! bstring_set_raw(&log_config.prefix, "templog"); +//! bstring_set_raw(&log_config.path, PATH); +//! log_config.level = LOG_LEVEL_TRACE; +//! +//! log_handle = log_create_handle_rs(&log_config); +//! ASSERT(log_handle != NULL); +//! ASSERT(log_is_setup_rs(log_handle)); +//! } +//! +//! void +//! log_teardown() +//! { +//! if (log_shutdown_rs(log_handle) != LOG_STATUS_OK) { +//! /* emit a warning about this */ +//! } +//! +//! log_destroy_handle_rs(&log_handle); +//! } +//! +//! ``` + +#![allow(dead_code)] + +pub use rslog::{Level, Log, SetLoggerError}; +use rslog::{Metadata, Record}; +pub use super::Result; +use cc_binding as bind; +use crossbeam::sync::ArcCell; +use failure; +use ptrs; +use rslog; +use bstring::BStr; +use std::cell::RefCell; +use std::ffi::CString; +use std::io::{Cursor, Write}; +use std::path::PathBuf; +use std::ptr; +use std::sync::Arc; +use std::thread; +use thread_id; +use thread_local::CachedThreadLocal; +use time; + + +// TODO(simms): add C-side setup code here. + +const PER_THREAD_BUF_SIZE: usize = 4096; + +#[derive(Fail, Debug)] +pub enum LoggingError { + #[fail(display = "logging already set up")] + LoggingAlreadySetUp, + + #[fail(display = "Other logger has already been set up with log crate")] + LoggerRegistrationFailure, + + #[fail( + display = "cc_log_create failed. see stderr for message. path: {}, buf_size: {}", + path, buf_size + )] + CreationError { path: String, buf_size: u32 }, + +} + +impl From for LoggingError { + fn from(_: SetLoggerError) -> Self { + LoggingError::LoggerRegistrationFailure + } +} + + +#[doc(hidden)] +pub struct CLogger(*mut bind::logger); + +impl CLogger { + pub unsafe fn from_raw(p: *mut bind::logger) -> super::Result { + ptrs::null_check(p).map(CLogger).map_err(|e| e.into()) + } + + pub unsafe fn write(&self, msg: &[u8]) -> bool { + let b = bind::log_write(self.0, msg.as_ptr() as *mut i8, msg.len() as u32); + if !b { + eprintln!("failed to write to log: {:#?}", &msg); + } + b + } + + pub unsafe fn flush(&self) { bind::log_flush(self.0); } + + pub unsafe fn open(path: &str, buf_size: u32) -> super::Result { + let p = bind::log_create(CString::new(path)?.into_raw(), buf_size); + + ptrs::lift_to_option(p) + .ok_or_else(|| LoggingError::CreationError {path: path.to_owned(), buf_size}.into()) + .map(CLogger) + } + + pub fn as_mut_ptr(&mut self) -> *mut bind::logger { self.0 } +} + +impl Drop for CLogger { + fn drop(&mut self) { + unsafe { bind::log_destroy(&mut self.0) } + } +} + +fn format(record: &Record, buf: &mut Vec) -> Result { + let tm = time::now_utc(); + + let mut curs = Cursor::new(buf); + + let ts = time::strftime("%Y-%m-%d %H:%M:%S", &tm).unwrap(); + + writeln!( + curs, + "{}.{:06} {:<5} [{}] {}", + ts, + tm.tm_nsec, + record.level().to_string(), + record.module_path().unwrap_or_default(), + record.args() + )?; + + Ok(curs.position() as usize) +} + +#[repr(u32)] +#[derive(Debug, PartialEq, PartialOrd, Eq)] +pub enum LoggerStatus { + OK = 0, + LoggerNotSetupError = 1, + RegistrationFailure = 2, + LoggerAlreadySetError = 3, + InvalidUTF8 = 4, + CreationError = 5, + OtherFailure = 6, + NullPointerError = 7, +} + +impl From for LoggerStatus { + fn from(e: LoggingError) -> Self { + match e { + LoggingError::LoggerRegistrationFailure => LoggerStatus::RegistrationFailure, + LoggingError::LoggingAlreadySetUp => LoggerStatus::LoggerAlreadySetError, + LoggingError::CreationError{..} => LoggerStatus::CreationError, + } + } +} + + +#[repr(usize)] +#[doc(hidden)] +#[derive(Debug, Eq, PartialEq)] +enum ModuleState { + UNINITIALIZED = 0, + INITIALIZING, + INITIALIZED, + FAILED, +} + +impl From for ModuleState { + fn from(u: usize) -> Self { + match u { + 0 => ModuleState::UNINITIALIZED, + 1 => ModuleState::INITIALIZING, + 2 => ModuleState::INITIALIZED, + 3 => ModuleState::FAILED, + _ => unreachable!() + } + } +} + +#[cfg(test)] +pub(in log) struct LogMetrics(*mut bind::log_metrics_st); + +#[cfg(test)] +impl LogMetrics { + pub fn new() -> Self { + let ptr = unsafe { bind::log_metrics_create() }; + assert!(!ptr.is_null()); + LogMetrics(ptr) + } + + pub fn as_mut_ptr(&mut self) -> *mut bind::log_metrics_st { self.0 } +} + +#[cfg(test)] +impl Drop for LogMetrics { + fn drop(&mut self) { + unsafe { bind::log_metrics_destroy(&mut self.0) } + } +} + +const DEFAULT_LOG_BASENAME: &str = "ccommon"; + +#[repr(C)] +pub struct LogConfig { + /// Path to the directory where we will write log files + path: String, + + /// The basis for log filenames. If `foobar` is given, + /// logs will be named `foobar.${thread_id}.log`. There will be one + /// log created per thread. If the thread is named, that will be used + /// as `thread_id` otherwise a unique identifier will be chosen. + prefix: String, + + /// What size buffer should the cc_log side use? + buf_size: u32, + + level: Level, +} + +#[derive(Clone, Debug)] +pub struct LogConfigBuilder { + path: Option, + prefix: Option, + buf_size: Option, + level: Option, +} + +impl Default for LogConfigBuilder { + fn default() -> Self { + LogConfigBuilder{ + path: None, + prefix: Some(String::from("ccommon")), + buf_size: Some(0), + level: Some(Level::Trace) + } + } +} + + +impl LogConfigBuilder { + pub fn path(&mut self, path: String) -> &mut Self { + let new = self; + new.path = Some(path); + new + } + + pub fn prefix(&mut self, prefix: String) -> &mut Self { + let new = self; + new.prefix = Some(prefix); + new + } + + pub fn buf_size(&mut self, buf: u32) -> &mut Self { + let new = self; + new.buf_size = Some(buf); + new + } + + pub fn level(&mut self, lvl: Level) -> &mut Self { + let new = self; + new.level = Some(lvl); + new + } + + pub fn build(&self) -> Result { + if self.path.is_none() { + bail!("path field must be set: {:#?}", self) + } + Ok(LogConfig{ + path: Clone::clone(&self.path).unwrap().to_owned(), + prefix: Clone::clone(&self.prefix).unwrap().to_owned(), + buf_size: Clone::clone(&self.buf_size).unwrap(), + level: Clone::clone(&self.level).unwrap(), + }) + } +} + +fn level_from_usize(u: usize) -> Option { + match u { + 1 => Some(Level::Error), + 2 => Some(Level::Warn), + 3 => Some(Level::Info), + 4 => Some(Level::Debug), + 5 => Some(Level::Trace), + _ => None, + } +} + +impl LogConfig { + #[doc(hidden)] + pub unsafe fn from_raw(ptr: *mut bind::log_config_rs) -> Result { + ptrs::lift_to_option(ptr) + .ok_or_else(|| ptrs::NullPointerError.into()) + .and_then(|ptr| { + let raw = *ptr; + + let path = BStr::from_ref(&raw.path).to_utf8_string()?; + let prefix = BStr::from_ref(&raw.prefix).to_utf8_string()?; + let buf_size = raw.buf_size; + let level = + match level_from_usize(raw.level as usize) { + Some(n) => n, + None => Level::Trace, + }; + + LogConfigBuilder::default() + .path(path) + .prefix(prefix) + .buf_size(buf_size) + .level(level) + .build() + }) + } + + fn to_path_buf(&self, thread_id: &str) -> PathBuf { + let mut pb = PathBuf::new(); + pb.push(&self.path); + pb.push(format!("{}.{}.log", self.prefix, thread_id)); + pb + } +} + + +struct PerThreadLog { + /// The underlying cc_log logger instance + clogger: CLogger, + /// The cached thread name or unique identifier + thread_name: String, + /// This buffer is used for preparing the message to be logged + buf: RefCell>, +} + +impl PerThreadLog { + fn for_current(cfg: &LogConfig) -> super::Result { + let tc = thread::current(); + let thread_name = + tc.name() + .map(|s| s.to_owned()) + .unwrap_or_else(|| { format!("{}", thread_id::get()) }); + + let clogger = unsafe { + CLogger::open(cfg.to_path_buf(&thread_name[..]).to_str().unwrap(), cfg.buf_size)? + }; + + let buf = RefCell::new(Vec::with_capacity(PER_THREAD_BUF_SIZE)); + + Ok(PerThreadLog{thread_name, clogger, buf}) + } +} + +unsafe impl Sync for PerThreadLog {} +unsafe impl Send for PerThreadLog {} + + +impl Log for PerThreadLog { + fn enabled(&self, _: &Metadata) -> bool { + true + } + + fn log(&self, record: &Record) { + if self.enabled(record.metadata()) { + let mut buf = self.buf.borrow_mut(); + let sz = format(record, &mut buf).unwrap(); + unsafe { self.clogger.write(&buf[0..sz]); } + } + } + + fn flush(&self) { + unsafe { self.clogger.flush(); } + } +} + +/// Shim is what gets called by the log crate. It holds the config, +/// creates PerThreadLogs on demand, and holds a reference to all +/// the thread local loggers. +struct Shim { + tls: CachedThreadLocal>>, + cfg: LogConfig, +} + +impl Shim { + fn get_per_thread(&self) -> super::Result<&RefCell>> { + self.tls.get_or_try(|| + PerThreadLog::for_current(&self.cfg) + .map(|ptl| Box::new(RefCell::new(Some(ptl))) ) + ) + } + + fn new(cfg: LogConfig) -> Self { + Shim { cfg, tls: CachedThreadLocal::new() } + } + + fn shutdown(&mut self) { + for cell in self.tls.iter_mut() { + if let Some(ptl) = cell.replace(None) { + ptl.flush(); + drop(ptl); + } + } + } + + #[inline] + fn borrow_and_call(&self, f: F) -> Option + where F: FnOnce(&PerThreadLog) + { + self.get_per_thread() + .map(|cell| { + if let Some(ptl) = &*cell.borrow() { + f(ptl); + } + }) + .err() + } +} + +impl Log for Shim { + fn enabled(&self, _: &Metadata) -> bool { + true + } + + fn log(&self, record: &Record) { + if let Some(err) = self.borrow_and_call(|ptl| ptl.log(record)) { + eprintln!("err in Shim::log {:#?}", err); + } + } + + fn flush(&self) { + if let Some(err) = self.borrow_and_call(|ptl| ptl.flush()) { + eprintln!("err in Shim::flush {:#?}", err); + } + } +} + +/// This is the Log instance we give to the log crate. Its job is to +/// hold onto the `Shim` and dispatch calls to it. See `Handle` +/// for a description of the inner structure. +/// +#[doc(hidden)] +struct Logger(Arc>>); + +impl Log for Logger { + fn enabled(&self, metadata: &Metadata) -> bool { + if let Some(n) = &*self.0.get() { + n.enabled(metadata) + } else { + false + } + } + + fn log(&self, record: &Record) { + if let Some(log) = &*self.0.get() { + log.log(record); + } + } + + fn flush(&self) { + if let Some(log) = &*self.0.get() { + log.flush(); + } + } +} + + +/// This is essentially `Arc->ArcCell->Arc->Option->Shim`. The outermost `Arc` is shared +/// between the log crate and this `Handle` that +/// we return to the user to allow them to shut down. +/// +/// ```ignore +/// +-------------------------------+ +/// | | +/// | | +/// +----------+ | +/// | Arc | v +/// | | +--------------------------+ +/// | Logger | | ArcCell | +/// | | | +----------------------+ | +/// | | | | Arc | | +/// +----------+ | | +------------+ | | +/// | | | Option | | | +/// +----------+ | | | +------+ | | | +/// | | | | | | Shim | | | | +/// | | | | | +------+ | | | +/// | Handle | | | +------------+ | | +/// | | | +----------------------+ | +/// | Arc | | | +/// +----------+ +--------------------------+ +/// | ^ +/// | | +/// +-------------------------------+ +/// ``` +/// +/// We perform the shutdown +/// by first swapping out the innermost `Arc` for a no-op (None) version, then unboxing and +/// shutting down the per-thread loggers in the `Shim`. +#[repr(C)] +pub struct Handle { + shim: Arc>> +} + +#[allow(non_camel_case_types)] +#[doc(hidden)] +type log_handle_rs = Handle; + +impl Handle { + fn shutdown(&mut self, timeout: time::Duration) { + let mut active: Arc> = self.shim.set(Arc::new(None)); + + let stop_at = time::SteadyTime::now() + timeout; + + if active.is_none() { + // we've already shut down + eprintln!("already shut down!"); + return; + } + + loop { + if let Some(opt_shim) = Arc::get_mut(&mut active) { + if let Some(shim) = opt_shim { + shim.shutdown(); + break + } + } else { + eprintln!("failed to get_mut on the active logger"); + thread::yield_now(); + } + + if time::SteadyTime::now() < stop_at { + eprintln!("timed out waiting on log shutdown, best of luck!"); + break + } + } + } + + fn is_setup(&self) -> bool { + self.shim.get().is_some() + } +} + +#[no_mangle] +pub unsafe extern "C" fn log_is_setup_rs(cfgp: *mut Handle) -> bool { + ptrs::lift_to_option(cfgp) + .map(|p| (*p).is_setup() ) + .expect("log_is_setup_rs was passed a raw pointer") +} + +const SHUTDOWN_TIMEOUT_MS: u64 = 1000; + +impl Drop for Handle { + fn drop(&mut self) { + self.shutdown(time::Duration::milliseconds(SHUTDOWN_TIMEOUT_MS as i64)); + } +} + +fn log_setup_safe(config: LogConfig) -> Result { + rslog::set_max_level(config.level.to_level_filter()); + let shim = Shim::new(config); + let logger = Logger(Arc::new(ArcCell::new(Arc::new(Some(shim))))); + + let handle = Handle {shim: logger.0.clone()}; + + rslog::set_boxed_logger(Box::new(logger)) + .map(|()| handle) + .map_err(|e| e.into()) +} + +#[no_mangle] +pub unsafe extern "C" fn log_create_handle_rs(cfgp: *mut bind::log_config_rs) -> *mut Handle { + ptrs::null_check(cfgp) // make sure our input is good + .map_err(|e| e.into()) // error type bookkeeping + .and_then(|c|LogConfig::from_raw(c)) // convert the *mut into a rust struct + .and_then(log_setup_safe) // register our logger + .map(|handle| Box::into_raw(Box::new(handle))) // convert our handle into a raw pointer + .unwrap_or_else(|err| { // hand it back to C + eprintln!("ERROR log_create_handle: {:#?}", err); + ptr::null_mut() // unless there was an error, then return NULL + }) +} + +#[no_mangle] +pub unsafe extern "C" fn log_shutdown_rs(ph: *mut Handle, timeout_ms: u32) -> LoggerStatus { + let mut handle = + match ptrs::lift_to_option(ph) { + Some(ph) => Box::from_raw(ph), + None => return LoggerStatus::NullPointerError, + }; + + Handle::shutdown(&mut handle, time::Duration::milliseconds(timeout_ms as i64)); + + LoggerStatus::OK +} + +#[no_mangle] +pub unsafe extern "C" fn log_destroy_handle_rs(pph: *mut *mut Handle) { + assert!(!pph.is_null()); + let ph = *pph; + drop(Box::from_raw(ph)); + *pph = ptr::null_mut(); +} + +// for integration testing with C +#[doc(hidden)] +#[no_mangle] +pub unsafe extern "C" fn log_test_threaded_writes_rs() -> bool { + let t1 = thread::spawn(move || { + for x in 0..10 { + error!("thread 1: {}", x); + } + }); + + let t2 = thread::spawn(move || { + for x in 0..10 { + warn!("thread 2: {}", x); + } + }); + + t1.join().unwrap(); + t2.join().unwrap(); + + true +} + +#[cfg(test)] +mod test { + use std::fs; + use std::sync::mpsc; + use super::*; + use tempfile; + use time; + + + // this is necessary until https://github.com/rust-lang/rust/issues/48854 + // lands in stable + fn assert_result(f: F) + where F: FnOnce() -> Result + { + match f() { + Ok(_) => (), + Err(e) => panic!(e) + } + } + + fn basic_mt_roundtrip() { + assert_result(|| { + let mut stats = LogMetrics::new(); + unsafe { bind::log_setup(stats.as_mut_ptr()) }; + let tmpdir = tempfile::tempdir()?; + + let cfg = LogConfig { + path: tmpdir.path().to_path_buf().to_str().unwrap().to_owned(), + prefix: String::from("testmt"), + buf_size: 0, + level: Level::Trace, + }; + + let handle = log_setup_safe(cfg).unwrap(); + + let t1 = thread::spawn(move || { + error!("thread 1 error"); + }); + + let t2 = thread::spawn(move || { + warn!("thread 2 error"); + }); + + t1.join().unwrap(); + t2.join().unwrap(); + + drop(handle); + + Ok(()) + }) + } + + + fn build(name: &str) -> thread::Builder { + thread::Builder::new().name(name.to_owned()) + } + + fn named_threads_test() { + assert_result(||{ + let mut stats = LogMetrics::new(); + unsafe { bind::log_setup(stats.as_mut_ptr()) }; + let tmpdir = tempfile::tempdir()?; + + let cfg = LogConfig { + path: tmpdir.path().to_path_buf().to_str().unwrap().to_owned(), + prefix: String::from("testmt"), + buf_size: 0, + level: Level::Trace, + }; + + let handle = log_setup_safe(cfg).unwrap(); + + let t1 = build("d_level").spawn(move || { + debug!("debug message"); + }).unwrap(); + + let t2 = build("w_level").spawn(move || { + warn!("warn message"); + }).unwrap(); + + t1.join().unwrap(); + t2.join().unwrap(); + + drop(handle); + + { + let mut dlevelp = tmpdir.path().to_owned(); + dlevelp.push("testmt.d_level.log"); + let md = fs::metadata(dlevelp)?; + assert!(md.len() > 0); + } + + { + let mut wlevelp = tmpdir.path().to_owned(); + wlevelp.push("testmt.w_level.log"); + let md = fs::metadata(wlevelp)?; + assert!(md.len() > 0); + } + + Ok(()) + }) + } + + fn mt_shutdown_resilience_test() { + assert_result(||{ + // make sure a thread logging doesn't crash if we shutdown simultaneously + let mut stats = LogMetrics::new(); + unsafe { bind::log_setup(stats.as_mut_ptr()) }; + let tmpdir = tempfile::tempdir()?; + + let cfg = LogConfig { + path: tmpdir.path().to_path_buf().to_str().unwrap().to_owned(), + prefix: String::from("testmt"), + buf_size: 0, + level: Level::Trace, + }; + + let handle = log_setup_safe(cfg).unwrap(); + + let (start_tx, start_rx) = mpsc::sync_channel::(0); + let (stop_tx, stop_rx) = mpsc::sync_channel::(0); + let (loop_tx, loop_rx) = mpsc::sync_channel::(300); + + eprintln!("start thread"); + let th = build("worker").spawn(move||{ + eprintln!("thread started, waiting for message"); + let msg = start_rx.try_recv().unwrap(); + eprintln!("got start msg: {}", msg); + + let mut count: u64 = 0; + loop { + let tm = time::now_utc(); + trace!("{:#?}", tm.to_timespec()); + count += 1; + loop_tx.send(count).unwrap(); + + match stop_rx.try_recv() { + Ok(_) => { + eprintln!("received stop signal"); + break; + }, + Err(mpsc::TryRecvError::Disconnected) => { + eprintln!("gah! disconnected!"); + panic!("bad things!"); + }, + Err(mpsc::TryRecvError::Empty) => () + }; + } + + eprintln!("while loop exited"); + count + }).unwrap(); + + start_tx.send("GO!".to_owned())?; + + let delay = ::std::time::Duration::from_millis(100); + + assert_eq!(loop_rx.recv_timeout(delay)?, 1); + + eprintln!("dropping handle"); + drop(handle); + + // make sure the thread writes another message or two + assert_eq!(loop_rx.recv_timeout(delay)?, 2); + assert_eq!(loop_rx.recv_timeout(delay)?, 3); + + // signal to the thread that it should exit + stop_tx.send(true)?; + + eprintln!("joining"); + let count = th.join().unwrap(); + eprintln!("thread joined, wrote {} messages", count); + + Ok(()) + }) + } + + // runs this test with process isolation + rusty_fork_test! { + #[test] + fn test_basic_mt_roundtrip() { basic_mt_roundtrip(); } + } + + rusty_fork_test! { + #[test] + fn test_named_threads() { named_threads_test(); } + } + + rusty_fork_test! { + #[test] + fn test_shutdown_resilience() { mt_shutdown_resilience_test(); } + } +} + diff --git a/rust/ccommon_rs/src/ptrs.rs b/rust/ccommon_rs/src/ptrs.rs new file mode 100644 index 000000000..11b930f56 --- /dev/null +++ b/rust/ccommon_rs/src/ptrs.rs @@ -0,0 +1,41 @@ +// ccommon - a cache common library. +// Copyright (C) 2018 Twitter, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::result; +use std::ptr; + +#[derive(Fail, Debug)] +#[fail(display = "Null pointer exception")] +pub struct NullPointerError; + +pub fn lift_to_option(p: *mut T) -> Option<*mut T> { + if p.is_null() { + None + } else { + Some(p) + } +} + +pub fn null_check(p: *mut T) -> result::Result<*mut T, NullPointerError> { + lift_to_option(p).ok_or_else(|| NullPointerError) +} + +pub fn opt_to_null_mut(o: Option<*mut T>) -> *mut T { + match o { + Some(p) => p, + None => ptr::null_mut(), + } +} + diff --git a/rust/ccommon_rs/src/util.rs b/rust/ccommon_rs/src/util.rs new file mode 100644 index 000000000..cefb110bb --- /dev/null +++ b/rust/ccommon_rs/src/util.rs @@ -0,0 +1,43 @@ +// ccommon - a cache common library. +// Copyright (C) 2018 Twitter, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + + +use std::ffi::CStr; +use std::fs; +use std::os::raw::c_char; + +/// Recursively removes files and directories under `path` before removing `path` itself. +/// Returns 0 on success and -1 on error. `errno` will be set to the cause of the failure. +#[no_mangle] +pub unsafe extern "C" fn cc_util_rm_rf_rs(path: *const c_char) -> i32 { + assert!(!path.is_null()); + + let s = + match CStr::from_ptr(path as *mut c_char).to_str() { + Ok(s) => s, + Err(err) => { + eprintln!("ERROR: cc_util_rm_rf_rs: {:#?}", err); + return -1 + } + }; + + match fs::remove_dir_all(s) { + Ok(()) => 0, + Err(err) => { + eprintln!("ERROR, cc_util_rm_rf_rs {:#?}", err); + -1 + } + } +} diff --git a/src/cc_log.c b/src/cc_log.c index c97979d0d..da013e061 100644 --- a/src/cc_log.c +++ b/src/cc_log.c @@ -18,6 +18,7 @@ #include #include +#include #include #include #include @@ -37,6 +38,30 @@ static log_metrics_st *log_metrics = NULL; static bool log_init = false; +/* this function is called from rust so that it can use log_setup */ +log_metrics_st * +log_metrics_create() +{ + log_metrics_st *metrics = cc_alloc(sizeof(log_metrics_st)); + if (metrics == NULL) { + log_panic("Failed to allocate log_metrics_st"); + } + log_metrics_st mtr = (log_metrics_st) { LOG_METRIC(METRIC_INIT) }; + memcpy(metrics, &mtr, sizeof(mtr)); + return metrics; +} + +void +log_metrics_destroy(log_metrics_st **m) +{ + if (m == NULL) { + log_panic("pointer passed to log_destroy_metrics was null"); + } + + cc_free(*m); + *m = NULL; +} + void log_setup(log_metrics_st *metrics) { diff --git a/test/log/CMakeLists.txt b/test/log/CMakeLists.txt index 5c87f50ee..67848eef4 100644 --- a/test/log/CMakeLists.txt +++ b/test/log/CMakeLists.txt @@ -3,8 +3,26 @@ set(test_name check_${suite}) set(source check_${suite}.c) +if(HAVE_RUST) + set(CCOMMON_LIBS ccommon_rs_static) # common_rs_static staticly links common-static +else() + set(CCOMMON_LIBS ccommon-static) +endif() + + add_executable(${test_name} ${source}) -target_link_libraries(${test_name} ccommon-static ${CHECK_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} m) +target_link_libraries( + ${test_name} + ${CHECK_LIBRARIES} + ${CMAKE_THREAD_LIBS_INIT} + ${CCOMMON_LIBS} + m) + +if(OS_PLATFORM STREQUAL "OS_DARWIN") + # the "rand" crate links to the platform secure rng API, so it requires this + target_link_libraries(${test_name} "-framework Security") + set_target_properties(${test_name} PROPERTIES LINK_FLAGS "-Wl,-F/System/Library/Frameworks") +endif() -add_dependencies(check ${test_name}) +add_dependencies(check ${test_name} ccommon-static) add_test(${test_name} ${test_name}) diff --git a/test/log/check_log.c b/test/log/check_log.c index 3139c60d7..d14e23ff0 100644 --- a/test/log/check_log.c +++ b/test/log/check_log.c @@ -1,9 +1,15 @@ #include +#ifdef HAVE_RUST +#include +#include +#endif + #include #include #include +#include #define SUITE_NAME "log" #define DEBUG_LOG SUITE_NAME ".log" @@ -53,7 +59,11 @@ tmpname_destroy(char *path) { unlink(path); path[strlen(path) - 2] = 0; +#ifdef HAVE_RUST + cc_util_rm_rf_rs(path); +#else rmdir(path); +#endif free(path); } @@ -240,6 +250,33 @@ START_TEST(test_write_skip_metrics) } END_TEST +#ifdef HAVE_RUST +START_TEST(test_most_basic_rust_logging_setup_teardown) +{ +#define PATH "/tmp/temp.XXXXXX" + char *path = malloc(sizeof(PATH) + 1); + strcpy(path, PATH); + mkdtemp(path); + + struct log_config_rs cfg; + cfg.buf_size = 1024; + bstring_set_raw(&cfg.prefix, "templog"); + bstring_set_raw(&cfg.path, path); + cfg.level = LOG_LEVEL_TRACE; + + struct log_handle_rs *handle = log_create_handle_rs(&cfg); + ck_assert(log_is_setup_rs(handle)); + ck_assert_uint_eq(log_shutdown_rs(handle, 1000), LOG_STATUS_OK); + log_destroy_handle_rs(&handle); + ck_assert_ptr_null(handle); + + cc_util_rm_rf_rs(path); +#undef PATH +} +END_TEST +#endif + + /* * test suite */ @@ -261,6 +298,9 @@ log_suite(void) tcase_add_test(tc_log, test_write_metrics_file_nobuf); tcase_add_test(tc_log, test_write_metrics_stderr_nobuf); tcase_add_test(tc_log, test_write_skip_metrics); +#ifdef HAVE_RUST + tcase_add_test(tc_log, test_most_basic_rust_logging_setup_teardown); +#endif return s; }