Squashed 'cereal/' changes from 90e48c5..b8382bb

b8382bb steerLimitTimer should be car dependent
9a22968 add pa0 temp to ThermalData
f6f0f60 Add stock Fcw to carState
b608683 no l/r distinction for LDW
555f48d Add ldw alert
8e8b4a4 Remove plusFrame socket in favor of UiLayoutState
3410325 log stock AEB events
2219f2b Add warning about not using cython version of sec_since_boot
8f1a512 for legacy-testing reasons, better to define the used percent instead of avail
e86d954 adding low memory event
ad23834 remove TODO
d0962b3 log mem available and cpu perc in thermald
3b753be Implement error handling and exceptions (#18)
a7d5bb7 add explicit dependencies on services.h
1ba6467 fix linter
c7d215b Added communityFeatureDisallowed event
492140a Added communityFeature bit detection to CarParams
266a5fe log Panda fault types
347a866 Switch from polling on FIFOs to signal (#12)
e25bba7 no need to double build the objects
fe43a99 20Hz for radar time step is very standard
2aabf1e Added radar time step to car params
e8ae908 Generate capnp for java
57126a2 cereal_shared
da655cd Add uptime to health
f6a8e39 add test with multiple subscribers
84b3af5 comment out the debugging
4b9c942 added power save state to health packet
66be370 run python unittest in ci
52c6db8 Run scons in CI (#14)
9414615 do need it, but only for arm
2856c37 remove gnustl_shared
7f05ee6 fix apks
e3a6bde Revert "no more makefiles"
487fbd0 don't rely on BASEDIR, and add zmq library
223e37a no more makefiles
da2ed11 don't link the wrong one
fe9fe2a scons builds the python lib now
2f81135 err, it can't build services.h
57b03f8 now we shouldn't need that yaml crap everywhere
f8e5327 bridge builds with services.h
2b0cb60 noOutput safety mode is now called silent
83880d5 add msgq tests
bcad184 msgq: dont block when fifo does not exists
b4b26782 Default to zmq
473e291 fix compilation in docker
30aaadd msgq: try again when no timeout on poll but also no message
c4f2ad5 msgq: make sure read_fifos is initalized so we dont close random fds
4e513a85 msgq: dont clean up uninitialized sockerts
c008b63 also remove the fifo from disk
ef64eb2 MSGQ stability improvements when opening and closing lots of queues
e147abc Revert "Revert "deprecate irpwr""
932dc32 Revert "deprecate irpwr"
a684415 disengage
ec27e18 capnpc also generated the header files
ee52ab9 deprecate irpwr
301c74c8 Merge branch 'master' of github.com:commaai/cereal
6da7d55 add front frame
a5944eb add conflate parameter for SubSocket::create
ca8df17 Add fault status to health
ef4ded0 add conflate support in SubSocket constructor
7fd314a update scons build file
93d814e add saturated flags to indi and lqr logs
50302fe add steeringRateLimited to car.capnp
05e3513 add msgq readme
a6759a9 faster make
94b7377 Add struct to log FW version
64ce0b5 add scons build
dc9ad18 add debug print statement on SIGINT
4a61269 Merge pull request #10 from commaai/msgq
4873449 use recv one or none after poll
a054864 default to msgq
fbc4a4c oops bad number
5067cf4 add meta
cbd0286 fix export prefix and make shared library world readable
c273054 add c exports for jni usage
e77f41e zmq already sets the errno correctly
3196cf6 Fix service list path in bridge
d35515a add all msgq files, but dont use as default
a68a38f Don't delete context from python side only
bd46c22 Revert "zmq_ctx_term is blocking"
a1fc26b zmq_ctx_term is blocking
0902182 remote address support
21a3536 only delete subsocket when created by same object
34df735 remove extra underscore from __dealloc__
c8748f8 fix internal refs
79b2fbf fixups
23ad256 import messaging and services

git-subtree-dir: cereal
git-subtree-split: b8382bb

.dockerignore

@@ -0,0 +1 @@
+.sconsign.dblite

.gitignore

@@ -3,4 +3,12 @@ node_modules
 package-lock.json
 *.pyc
 __pycache__
+.*.swp
+.*.swo
+libcereal*.a
+libmessaging.*
+libmessaging_shared.*
+services.h
+.sconsign.dblite
+libcereal_shared.so
 

Dockerfile

@@ -0,0 +1,19 @@
+from ubuntu:16.04
+
+RUN apt-get update && apt-get install -y libzmq3-dev clang wget git autoconf libtool curl make build-essential libssl-dev zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev llvm libncurses5-dev libncursesw5-dev xz-utils tk-dev libffi-dev liblzma-dev python-openssl
+
+RUN curl -L https://github.com/pyenv/pyenv-installer/raw/master/bin/pyenv-installer | bash
+ENV PATH="/root/.pyenv/bin:/root/.pyenv/shims:${PATH}"
+RUN pyenv install 3.7.3
+RUN pyenv global 3.7.3
+RUN pyenv rehash
+RUN pip3 install pyyaml==5.1.2 Cython==0.29.14 scons==3.1.1 pycapnp==0.6.4
+
+WORKDIR /project/cereal
+COPY install_capnp.sh .
+RUN ./install_capnp.sh
+
+ENV PYTHONPATH=/project
+
+COPY . .
+RUN scons -c && scons -j$(nproc)

SConscript

@@ -0,0 +1,68 @@
+Import('env', 'arch', 'zmq')
+
+gen_dir = Dir('gen')
+messaging_dir = Dir('messaging')
+
+# TODO: remove src-prefix and cereal from command string. can we set working directory?
+env.Command(["gen/c/include/c++.capnp.h", "gen/c/include/java.capnp.h"], [], "mkdir -p " + gen_dir.path + "/c/include && touch $TARGETS")
+env.Command(
+  ['gen/c/car.capnp.c', 'gen/c/log.capnp.c', 'gen/c/car.capnp.h', 'gen/c/log.capnp.h'],
+  ['car.capnp', 'log.capnp'],
+  'capnpc $SOURCES --src-prefix=cereal -o c:' + gen_dir.path + '/c/')
+env.Command(
+  ['gen/cpp/car.capnp.c++', 'gen/cpp/log.capnp.c++', 'gen/cpp/car.capnp.h', 'gen/cpp/log.capnp.h'],
+  ['car.capnp', 'log.capnp'],
+  'capnpc $SOURCES --src-prefix=cereal -o c++:' + gen_dir.path + '/cpp/')
+import shutil
+if shutil.which('capnpc-java'):
+  env.Command(
+    ['gen/java/Car.java', 'gen/java/Log.java'],
+    ['car.capnp', 'log.capnp'],
+    'capnpc $SOURCES --src-prefix=cereal -o java:' + gen_dir.path + '/java/')
+
+# TODO: remove non shared cereal and messaging
+cereal_objects = env.SharedObject([
+    'gen/c/car.capnp.c',
+    'gen/c/log.capnp.c',
+    'gen/cpp/car.capnp.c++',
+    'gen/cpp/log.capnp.c++',
+  ])
+
+env.Library('cereal', cereal_objects)
+env.SharedLibrary('cereal_shared', cereal_objects)
+
+cereal_dir = Dir('.')
+services_h = env.Command(
+  ['services.h'],
+  ['service_list.yaml', 'services.py'],
+  'python3 ' + cereal_dir.path + '/services.py > $TARGET')
+
+messaging_objects = env.SharedObject([
+  'messaging/messaging.cc',
+  'messaging/impl_zmq.cc',
+  'messaging/impl_msgq.cc',
+  'messaging/msgq.cc',
+])
+
+messaging_lib = env.Library('messaging', messaging_objects)
+Depends('messaging/impl_zmq.cc', services_h)
+
+# note, this rebuilds the deps shared, zmq is statically linked to make APK happy
+# TODO: get APK to load system zmq to remove the static link
+shared_lib_shared_lib = [zmq, 'm', 'stdc++'] + ["gnustl_shared"] if arch == "aarch64" else []
+env.SharedLibrary('messaging_shared', messaging_objects, LIBS=shared_lib_shared_lib)
+
+env.Program('messaging/bridge', ['messaging/bridge.cc'], LIBS=[messaging_lib, 'zmq'])
+Depends('messaging/bridge.cc', services_h)
+
+# different target?
+#env.Program('messaging/demo', ['messaging/demo.cc'], LIBS=[messaging_lib, 'zmq'])
+
+
+env.Command(['messaging/messaging_pyx.so'],
+  [messaging_lib, 'messaging/messaging_pyx_setup.py', 'messaging/messaging_pyx.pyx', 'messaging/messaging.pxd'],
+  "cd " + messaging_dir.path + " && python3 messaging_pyx_setup.py build_ext --inplace")
+
+
+if GetOption('test'):
+  env.Program('messaging/test_runner', ['messaging/test_runner.cc', 'messaging/msgq_tests.cc'], LIBS=[messaging_lib])

SConstruct

@@ -0,0 +1,49 @@
+import os
+import subprocess
+
+zmq = 'zmq'
+arch = subprocess.check_output(["uname", "-m"], encoding='utf8').rstrip()
+
+cereal_dir = Dir('.')
+
+cpppath = [
+    cereal_dir,
+    '/usr/lib/include',
+]
+
+AddOption('--test',
+          action='store_true',
+          help='build test files')
+
+AddOption('--asan',
+          action='store_true',
+          help='turn on ASAN')
+
+ccflags_asan = ["-fsanitize=address", "-fno-omit-frame-pointer"] if GetOption('asan') else []
+ldflags_asan = ["-fsanitize=address"] if GetOption('asan') else []
+
+env = Environment(
+  ENV=os.environ,
+  CC='clang',
+  CXX='clang++',
+  CCFLAGS=[
+    "-g",
+    "-fPIC",
+    "-O2",
+    "-Werror=implicit-function-declaration",
+    "-Werror=incompatible-pointer-types",
+    "-Werror=int-conversion",
+    "-Werror=return-type",
+    "-Werror=format-extra-args",
+  ] + ccflags_asan,
+  LDFLAGS=ldflags_asan,
+  LINKFLAGS=ldflags_asan,
+
+  CFLAGS="-std=gnu11",
+  CXXFLAGS="-std=c++14",
+  CPPPATH=cpppath,
+)
+
+
+Export('env', 'zmq', 'arch')
+SConscript(['SConscript'])

azure-pipelines.yml

@@ -0,0 +1,14 @@
+pr: none
+
+pool:
+  vmImage: 'ubuntu-16.04'
+
+steps:
+- script: |
+    set -e
+    docker build -t cereal .
+    docker run cereal bash -c "scons --test --asan -j$(nproc) && messaging/test_runner"
+    docker run cereal bash -c "ZMQ=1 python -m unittest discover ."
+    docker run cereal bash -c "MSGQ=1 python -m unittest discover ."
+
+  displayName: 'Run Tests'

car.capnp

@@ -84,6 +84,10 @@ struct CarEvent @0x9b1657f34caf3ad3 {
     laneChange @59;
     invalidGiraffeToyota @60;
     internetConnectivityNeeded @61;
+    communityFeatureDisallowed @62;
+    lowMemory @63;
+    stockAeb @64;
+    ldw @65;
   }
 }
 
@@ -117,6 +121,9 @@ struct CarState {
   steeringTorque @8 :Float32;  # TODO: standardize units
   steeringTorqueEps @27 :Float32;  # TODO: standardize units
   steeringPressed @9 :Bool;    # if the user is using the steering wheel
+  steeringRateLimited @29 :Bool;    # if the torque is limited by the rate limiter
+  stockAeb @30 :Bool;
+  stockFcw @31 :Bool;
 
   # cruise state
   cruiseState @10 :CruiseState;
@@ -170,7 +177,6 @@ struct CarState {
     manumatic @9;
   }
 
-
   # send on change
   struct ButtonEvent {
     pressed @0 :Bool;
@@ -314,7 +320,7 @@ struct CarParams {
   minEnableSpeed @7 :Float32;
   minSteerSpeed @8 :Float32;
   safetyModel @9 :SafetyModel;
-  safetyModelPassive @42 :SafetyModel = noOutput;
+  safetyModelPassive @42 :SafetyModel = silent;
   safetyParam @10 :Int16;
 
   steerMaxBP @11 :List(Float32);
@@ -324,7 +330,6 @@ struct CarParams {
   brakeMaxBP @15 :List(Float32);
   brakeMaxV @16 :List(Float32);
 
-
   # things about the car in the manual
   mass @17 :Float32;             # [kg] running weight
   wheelbase @18 :Float32;        # [m] distance from rear to front axle
@@ -345,6 +350,7 @@ struct CarParams {
   }
 
   steerLimitAlert @28 :Bool;
+  steerLimitTimer @47 :Float32;  # time before steerLimitAlert is issued
 
   vEgoStopping @29 :Float32; # Speed at which the car goes into stopping state
   directAccelControl @30 :Bool; # Does the car have direct accel control or just gas/brake
@@ -360,6 +366,9 @@ struct CarParams {
   isPandaBlack @39: Bool;
   dashcamOnly @41: Bool;
   transmissionType @43 :TransmissionType;
+  carFw @44 :List(CarFw);
+  radarTimeStep @45: Float32 = 0.05;  # time delta between radar updates, 20Hz is very standard
+  communityFeature @46: Bool;  # true if a community maintained feature is detected
 
   struct LateralPIDTuning {
     kpBP @0 :List(Float32);
@@ -378,7 +387,6 @@ struct CarParams {
     deadzoneV @5 :List(Float32);
   }
 
-
   struct LateralINDITuning {
     outerLoopGain @0 :Float32;
     innerLoopGain @1 :Float32;
@@ -401,7 +409,7 @@ struct CarParams {
   }
 
   enum SafetyModel {
-    noOutput @0;
+    silent @0;
     honda @1;
     toyota @2;
     elm327 @3;
@@ -420,6 +428,7 @@ struct CarParams {
     toyotaIpas @16;
     allOutput @17;
     gmAscm @18;
+    noOutput @19;  # like silent but with silent CAN TXs
   }
 
   enum SteerControlType {
@@ -432,4 +441,16 @@ struct CarParams {
     automatic @1;
     manual @2;
   }
+
+  struct CarFw {
+    ecu @0 :Ecu;
+    fwVersion @1 :Text;
+  }
+
+  enum Ecu {
+    eps @0;
+    esp @1;
+    fwdRadar @2;
+    fwdCamera @3;
+  }
 }

install_capnp.sh

@@ -8,7 +8,8 @@ tar xvf capnproto-c++-${VERSION}.tar.gz
 cd capnproto-c++-${VERSION}
 CXXFLAGS="-fPIC" ./configure
 
-make -j4
+make -j$(nproc)
+make install
 
 # manually build binaries statically
 g++ -std=gnu++11 -I./src -I./src -DKJ_HEADER_WARNINGS -DCAPNP_HEADER_WARNINGS -DCAPNP_INCLUDE_DIR=\"/usr/local/include\" -pthread -O2 -DNDEBUG -pthread -pthread -o .libs/capnp src/capnp/compiler/module-loader.o src/capnp/compiler/capnp.o  ./.libs/libcapnpc.a ./.libs/libcapnp.a ./.libs/libkj.a -lpthread -pthread
@@ -18,7 +19,6 @@ g++ -std=gnu++11 -I./src -I./src -DKJ_HEADER_WARNINGS -DCAPNP_HEADER_WARNINGS -D
 g++ -std=gnu++11 -I./src -I./src -DKJ_HEADER_WARNINGS -DCAPNP_HEADER_WARNINGS -DCAPNP_INCLUDE_DIR=\"/usr/local/include\" -pthread -O2 -DNDEBUG -pthread -pthread -o .libs/capnpc-capnp src/capnp/compiler/capnpc-capnp.o  ./.libs/libcapnp.a ./.libs/libkj.a -lpthread -pthread
 
 cp .libs/capnp /usr/local/bin/
-ln -s /usr/local/bin/capnp /usr/local/bin/capnpc
 cp .libs/capnpc-c++ /usr/local/bin/
 cp .libs/capnpc-capnp /usr/local/bin/
 cp .libs/*.a /usr/local/lib
@@ -30,7 +30,8 @@ cd c-capnproto
 git submodule update --init --recursive
 autoreconf -f -i -s
 CXXFLAGS="-fPIC" ./configure
-make -j4
+make -j$(nproc)
+make install
 
 # manually build binaries statically
 gcc -fPIC -o .libs/capnpc-c compiler/capnpc-c.o compiler/schema.capnp.o compiler/str.o  ./.libs/libcapnp_c.a