From c62cca03b6697f12933a1c19fd503b08436d80c7 Mon Sep 17 00:00:00 2001
From: Yuchen Zhong <izhongyuchen@gmail.com>
Date: Sun, 4 Oct 2020 20:17:42 +0800
Subject: [PATCH] relax size check (#36)

* fix compression

* add docs
---
 docs/api.md         |  8 +++-----
 include/ps/kv_app.h | 18 +++++++++++++++---
 2 files changed, 18 insertions(+), 8 deletions(-)

diff --git a/docs/api.md b/docs/api.md
index 3744989e..510e68b4 100644
--- a/docs/api.md
+++ b/docs/api.md
@@ -6,12 +6,10 @@ The data communicated are presented as key-value
   index and the value might be the according `float` gradient.
   1. Basic synchronization functions: \ref ps::KVWorker::Push, \ref
   ps::KVWorker::Pull, and \ref ps::KVWorker::Wait
-  2. Dynamic length value push and pull: \ref ps::KVWorker::VPush and \ref
-     ps::KVWorker::VPull
-  3. Zero-copy versions: \ref ps::KVWorker::ZPush, \ref
-     ps::KVWorker::ZPull, \ref ps::KVWorker::ZVPush and \ref
-     ps::KVWorker::ZVPull
+  2. Zero-copy versions: \ref ps::KVWorker::ZPush, \ref
+     ps::KVWorker::ZPull
 
+To support dynamic length, pull operations(`Pull` and `ZPull`), do not require the buffer(`vals`) to be the same size as the total data size of pulling down. Larger buffer is allowed while `lens` records the actual size of each key. So the reliable way to read a valid message is to read `lens` bytes. If you ensure that the data size of a key does not change during push or pull, you can verify it by checking whether `lens` of the key is equal to the fixed size.  
 
 often server *i* handles the keys (feature indices) within the i-th
   segment of <em>[0, uint64_max]</em>. The server node allows user-defined handles to
diff --git a/include/ps/kv_app.h b/include/ps/kv_app.h
index dda435df..f6e14bab 100644
--- a/include/ps/kv_app.h
+++ b/include/ps/kv_app.h
@@ -53,6 +53,12 @@ struct KVPairs {
  * \brief A worker node that can \ref Push (\ref Pull) key-value pairs to (from) server
  * nodes
  *
+ * 1. Basic synchronization functions: \ref ps::KVWorker::Push,
+ * \ref ps::KVWorker::Pull, and \ref ps::KVWorker::Wait
+ *
+ * 2. Zero-copy versions: \ref ps::KVWorker::ZPush,
+ * \ref ps::KVWorker::ZPull *
+ *
  * \tparam Val the type of value, which should be primitive types such as
  * int32_t and float
  */
@@ -160,6 +166,12 @@ class KVWorker : public SimpleApp {
    * namely \a vals (and \a lens) is filled with pulled values, only
    * if \ref Wait returns or the callback is called.
    *
+   * Note that \a vals can be larger than the length of the pulled values.
+   * \a lens is the actual length of the pulled values. The reliable way to
+   * read a valid message is to read \a lens bytes. If you ensure that the
+   * data size of a key does not change during push or pull, you can verify
+   * it by checking whether \a lens of the key is equal to the fixed size.    *
+   *
    * @param keys a list of keys, must be unique and sorted in increasing order
    * @param vals the buffer for the pulled values. It can be 0 size.
    * @param lens optional buffer for the value length. If set, it can be 0 size.
@@ -640,10 +652,10 @@ int KVWorker<Val>::Pull_(
       if (vals->empty()) {
         vals->resize(total_val);
       } else {
-        CHECK_EQ(vals->size(), total_val);
+        CHECK_GE(vals->size(), total_val);
       }
 
-      if (!is_worker_zpull_) { // otherwise do nothing 
+      if (!is_worker_zpull_) { // otherwise do nothing
         Val* p_vals = vals->data();
         int *p_lens = nullptr;
         if (lens) {
@@ -670,7 +682,7 @@ int KVWorker<Val>::Pull_(
       if (cb) cb();
     });
 
-  KVPairs<Val> kvs; 
+  KVPairs<Val> kvs;
   kvs.keys = keys;
   kvs.vals = *vals;
   Send(ts, false, cmd, kvs);