diff --git a/README.md b/README.md index 36e694c..2d25b7a 100644 --- a/README.md +++ b/README.md @@ -161,3 +161,21 @@ Container images are configured using parameters passed at runtime. practical risk is resource exhaustion rather than code execution. Keep `BITMESSAGE_MAXTOTALCONNECTIONS` low and put memory and CPU limits on the container. +- **The daemon holds no privilege of its own, and the container drops what it + can.** `run.sh` needs root exactly once, at startup: `keys.dat` can arrive + from a bind mount owned by anyone, so it is chowned, given mode 600 and read + before anything else. The moment that is done the script re-executes itself + with a bounding set of four capabilities -- `SETUID` and `SETGID` to start the + daemon as its own user, `KILL` for the fallback stop, `SETPCAP` to drop the + rest -- and every start of the daemon goes through `setpriv`, at uid 2000, + with all four capability sets empty and `no_new_privs` on. No file in the + image carries a setuid or setgid bit, so a daemon that has been taken over has + nothing left to climb. + + Three things only the caller can set, and all three are worth setting. + `cap_drop: ALL` with `CHOWN`, `DAC_OVERRIDE`, `FOWNER`, `SETUID`, `SETGID`, + `KILL` and `SETPCAP` added back: those seven are what the startup above needs, + and the container cannot narrow what the healthcheck and `docker exec` run + with -- only this can. `read_only: true`, with `tmpfs` for `/tmp` and `/run`; + the daemon writes only into its own home. And `pids_limit`, next to the memory + and CPU limits the note above asks for. diff --git a/README.ru-RU.md b/README.ru-RU.md index e4451cc..12170a1 100644 --- a/README.ru-RU.md +++ b/README.ru-RU.md @@ -160,3 +160,21 @@ docker run -d \ кому угодно, а реальный риск — исчерпание ресурсов, а не выполнение кода. Держите `BITMESSAGE_MAXTOTALCONNECTIONS` низким и ограничьте контейнер по памяти и CPU. +- **У демона нет собственных прав, а контейнер снимает с себя остальное.** + `run.sh` нужен root ровно один раз, на старте: `keys.dat` может приехать из + bind-монтирования с чужим владельцем, поэтому сначала ему меняют владельца, + ставят режим 600 и читают. Сразу после этого скрипт перезапускает сам себя с + bounding set из четырёх capability — `SETUID` и `SETGID`, чтобы запускать + демона под его собственным пользователем, `KILL` для запасной остановки и + `SETPCAP`, чтобы снять остальные, — а каждый запуск демона идёт через + `setpriv`: uid 2000, все четыре набора capability пустые, `no_new_privs` + включён. Ни на одном файле образа нет бита setuid или setgid, так что + захваченному демону не по чему подниматься. + + Три вещи может задать только вызывающая сторона, и все три стоит задать. + `cap_drop: ALL` с возвращёнными `CHOWN`, `DAC_OVERRIDE`, `FOWNER`, `SETUID`, + `SETGID`, `KILL` и `SETPCAP`: эти семь нужны описанному выше старту, а сузить + набор, с которым работают healthcheck и `docker exec`, контейнер сам не может + — только это. `read_only: true` с `tmpfs` под `/tmp` и `/run`: демон пишет + только в свой домашний каталог. И `pids_limit` — рядом с ограничениями по + памяти и CPU, о которых просит предыдущее замечание. diff --git a/docker-compose.yml b/docker-compose.yml index c1f9256..44ea980 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -22,6 +22,24 @@ services: - 8444:8444 volumes: - bitmessage:/home/bitmessage + # A clean shutdown does not fit in Docker's default ten seconds; the last + # notes in the README explain this line and the six below it. The container + # takes away every privilege it can reach from inside, but the set a + # healthcheck and `docker exec` run with, the writability of the root + # filesystem and the process count are not among them -- only the caller + # can set those, and this is what setting them looks like. + stop_grace_period: 90s + cap_drop: [ALL] + cap_add: [CHOWN, DAC_OVERRIDE, FOWNER, SETUID, SETGID, KILL, SETPCAP] + security_opt: + - no-new-privileges:true + read_only: true + tmpfs: + - /tmp:rw,noexec,nosuid,nodev,size=64m + - /run:rw,noexec,nosuid,nodev,size=8m + pids_limit: 200 + mem_limit: 1g + cpus: 2 volumes: bitmessage: diff --git a/docker/Dockerfile b/docker/Dockerfile index c4249c7..1cad8e3 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -62,9 +62,12 @@ COPY ./docker/seed_addr_gen.py /usr/local/bin/ COPY ./docker/watchdog.py /usr/local/bin/ COPY ./docker/run.sh /usr/local/bin/ -# Install dependencies +# Install dependencies. util-linux carries setpriv, which run.sh uses to drop +# to the daemon's user; gosu, which used to do that, is gone because it dropped +# the user and nothing else, and moreutils went with it because nothing in this +# repository ever called any of its tools. RUN apt-get update \ - && apt-get install -yq --no-install-suggests --no-install-recommends python-setuptools moreutils gosu \ + && apt-get install -yq --no-install-suggests --no-install-recommends python-setuptools util-linux \ && rm -rf /var/lib/apt/lists/* # Create a user @@ -76,6 +79,47 @@ WORKDIR ${HOME} # Generate default config RUN su bitmessage -c "pybitmessage -t" +# Prove that the two setpriv lines in run.sh do what the script leans on, in +# the arrangement run.sh actually uses: the supervisor's set on the outside, the +# daemon's on the inside. This base image and the PyBitmessage clone above are +# both unpinned, so an option that quietly changed meaning would otherwise ship +# as a container that looks confined and is not -- the trap the sql_timeout +# greps in the first stage avoid the same way, by failing the build instead. +# +# The two lists are the ones run.sh carries, and they have to stay in step with +# it. Expected: the daemon at uid 2000 with every capability set empty and +# no_new_privs on, and the supervisor holding 00000000000001e0 -- CAP_KILL, +# CAP_SETGID, CAP_SETUID and CAP_SETPCAP, bits 5 to 8, and nothing else. +RUN set -eu \ + && none="-chown,-dac_override,-fowner,-fsetid,-kill,-setgid,-setuid" \ + && none="$none,-setpcap,-net_bind_service,-net_raw,-sys_chroot" \ + && none="$none,-mknod,-audit_write,-setfcap" \ + && sup="-chown,-dac_override,-fowner,-fsetid,-net_bind_service" \ + && sup="$sup,-net_raw,-sys_chroot,-mknod,-audit_write,-setfcap" \ + && d="$(setpriv --bounding-set="$sup" \ + setpriv --reuid=2000 --regid=2000 --clear-groups --bounding-set="$none" \ + --no-new-privs \ + grep -E '^(Uid|Gid|CapPrm|CapEff|CapBnd|NoNewPrivs):' /proc/self/status)" \ + && printf '%s\n' "$d" \ + && printf '%s\n' "$d" | grep -qE '^Uid:[[:space:]]+2000[[:space:]]+2000[[:space:]]+2000' \ + && printf '%s\n' "$d" | grep -qE '^Gid:[[:space:]]+2000[[:space:]]+2000[[:space:]]+2000' \ + && printf '%s\n' "$d" | grep -qE '^CapPrm:[[:space:]]+0{16}$' \ + && printf '%s\n' "$d" | grep -qE '^CapEff:[[:space:]]+0{16}$' \ + && printf '%s\n' "$d" | grep -qE '^CapBnd:[[:space:]]+0{16}$' \ + && printf '%s\n' "$d" | grep -qE '^NoNewPrivs:[[:space:]]+1$' \ + && s="$(setpriv --bounding-set="$sup" grep -E '^CapBnd:' /proc/self/status)" \ + && printf '%s\n' "$s" \ + && printf '%s\n' "$s" | grep -qE '^CapBnd:[[:space:]]+0{13}1e0$' + +# Nothing here needs a setuid or setgid bit at run time, and every one of them +# is a way back for a daemon that has been taken over -- the more so because the +# daemon now runs with no_new_privs, which makes them the one thing that could +# still have raised its privileges. setpriv is not among them: it is an ordinary +# binary that root execs. Strip them all, then insist none is left, so a package +# added here later cannot bring one back unnoticed. +RUN find / -xdev -type f -perm /6000 -exec chmod -s {} + \ + && [ -z "$(find / -xdev -type f -perm /6000)" ] + CMD ["sh", "/usr/local/bin/run.sh"] ## Check PyBitmessage active network connections. diff --git a/docker/run.sh b/docker/run.sh index 0677048..caecf5f 100644 --- a/docker/run.sh +++ b/docker/run.sh @@ -118,91 +118,160 @@ esc() { printf '%s' "$1" | sed -e 's/[\\&|]/\\&/g' } -# this command must be run as root (for bind mounts to container) -if [ -f keys.dat ] -then - chown bitmessage:bitmessage keys.dat - chmod 600 keys.dat -fi +# The fourteen capabilities Docker hands a container by default, written out +# rather than as "-all". The keyword is unusable in this image: setpriv refuses +# it whenever the running kernel knows more capabilities than the headers +# setpriv was built against, and Ubuntu 18.04 (CAP_LAST_CAP 37) under a current +# kernel (40) is exactly that case -- it would exit 127 with "libcap-ng is too +# old". A capability a compose file adds beyond the default therefore survives, +# which is the honest behaviour: nothing here silently drops what it was never +# told about. Names carry no cap_ prefix; that spelling is a parse error. +BM_CAPS_NONE="-chown,-dac_override,-fowner,-fsetid,-kill,-setgid,-setuid" +BM_CAPS_NONE="$BM_CAPS_NONE,-setpcap,-net_bind_service,-net_raw,-sys_chroot" +BM_CAPS_NONE="$BM_CAPS_NONE,-mknod,-audit_write,-setfcap" -# maxtotalconnections is the only brake on a node whose P2P port (8444) is -# published: it caps inbound sockets at the total minus maxoutboundconnections. -# The substitution below is a no-op when the key is missing, which would ship a -# node that looks capped and is not -- and the PyBitmessage clone in the -# Dockerfile is unpinned, so the stock config is whatever upstream generates -# today. Add the key rather than trust the substitution alone; line 1 is the -# [bitmessagesettings] header the daemon reads it from. -if ! grep -q "^maxtotalconnections = " keys.dat -then - gosu bitmessage sed -i "1a maxtotalconnections = $BITMESSAGE_MAXTOTALCONNECTIONS" keys.dat -fi +# The same list less the four the supervisor keeps: SETUID and SETGID start the +# daemon as its own user, KILL is the fallback stop, and SETPCAP is what lets +# the daemon's own line above run at all -- dropping a bounding set takes it. +BM_CAPS_SUPERVISOR="-chown,-dac_override,-fowner,-fsetid,-net_bind_service" +BM_CAPS_SUPERVISOR="$BM_CAPS_SUPERVISOR,-net_raw,-sys_chroot,-mknod" +BM_CAPS_SUPERVISOR="$BM_CAPS_SUPERVISOR,-audit_write,-setfcap" -# trustedpeer is absent from the stock keys.dat entirely, so the substitution -# below is a no-op until the key exists -- same trap as maxtotalconnections. -# The key is added even when the value is empty, which is how it can be taken -# back off a node that was pinned before: safeGet returns "" and connectionpool -# falls back to chooseConnection. That empty case is also why the anchors here -# stop at "=" instead of "= ": with nothing to the right there is no trailing -# space to match, and the substitution would never fire again. -if ! grep -q "^trustedpeer =" keys.dat -then - gosu bitmessage sed -i "1a trustedpeer = $(esc "$BITMESSAGE_TRUSTED_PEER")" keys.dat -fi - -# Set config values. Every expression is anchored to the start of the line and -# names its key in the replacement, so no backreference is involved and nothing -# in another section can match. With set -e a failure here now stops the -# container instead of leaving the daemon on its previous settings unnoticed -- -# including the case of a bind mount with no keys.dat at all. -gosu bitmessage sed -i \ - -e "s|^apiinterface = .*|apiinterface = 0.0.0.0|" \ - -e "s|^apivariant = .*|apivariant = $(esc "$BITMESSAGE_APIVARIANT")|" \ - -e "s|^apiusername = .*|apiusername = $(esc "$BITMESSAGE_API_USER")|" \ - -e "s|^apipassword = .*|apipassword = $(esc "$BITMESSAGE_API_PASSWORD")|" \ - -e "s|^apiport = .*|apiport = $(esc "$BITMESSAGE_API_PORT")|" \ - -e "s|^apienabled = .*|apienabled = True|" \ - -e "s|^ttl = .*|ttl = $(esc "$BITMESSAGE_TTL")|" \ - -e "s|^stopresendingafterxdays = .*|stopresendingafterxdays = $(esc "$BITMESSAGE_STOPRESENDINGAFTERXDAYS")|" \ - -e "s|^maxtotalconnections = .*|maxtotalconnections = $BITMESSAGE_MAXTOTALCONNECTIONS|" \ - -e "s|^trustedpeer =.*|trustedpeer = $(esc "$BITMESSAGE_TRUSTED_PEER")|" \ - -e "s|^sendoutgoingconnections = .*|sendoutgoingconnections = $BITMESSAGE_SEND_OUTGOING|" \ - -e "s|^udp = .*|udp = False|" keys.dat - -# BITMESSAGE_KNOWN_NODES pins the peers the daemon starts from, and is rewritten -# on every start: in a private contour the seed *is* the topology, and a file -# left over from an earlier run names nodes that may no longer exist. Seeding it -# also switches off the DNS bootstrap -- json_deserialize_knownnodes raises -# knownNodesActual for any peer that is neither DEFAULT_NODES nor "self", and -# connectionpool calls startBootstrappers only while that flag is down, so the -# node never reaches bootstrap8080.bitmessage.org. +# Run something as the daemon's own user with nothing left to escalate with: no +# capability in any set, an empty bounding set so none can ever be picked up +# again, and no_new_privs so that a setuid binary could not help either. This +# replaces the earlier privilege drop, which changed the user and left +# everything else alone. # -# Writing it "only when the file is missing" would have been a permanent no-op: -# the image ships a knownnodes.dat, produced by the `pybitmessage -t` run in the -# Dockerfile, and a named volume inherits it on first use. -if [ -n "$BITMESSAGE_KNOWN_NODES" ] +# The daemon is the only thing in this container exposed to the network, and +# this line is the whole of what it gets. The AppArmor profile this repository +# carried until 2022 agrees: the two capabilities it granted served the +# AppImage's fuse mount, and PyBitmessage itself asked for none. +as_bitmessage() { + setpriv --reuid="${USER_UID:-2000}" --regid="${USER_GID:-2000}" \ + --clear-groups --bounding-set="$BM_CAPS_NONE" \ + --no-new-privs "$@" +} + +# --- everything that needs root, and the moment root stops being needed ---- +# +# The capabilities this block uses are needed exactly once, at startup, and only +# by uid 0: keys.dat arrives owned by whoever a bind mount says (CAP_CHOWN), the +# greps below read it while it is mode 600 and owned by the daemon's user +# (CAP_DAC_OVERRIDE), and its mode is set on a file that is not ours +# (CAP_FOWNER). Measured one capability at a time: without DAC_OVERRIDE the +# greps fail silently and the configuration keys are appended a second time on +# every start; without FOWNER the chmod stops the container outright. +# +# So the block ends by re-executing this file with a bounding set that holds +# none of the three. What comes back needs SETUID and SETGID to start the daemon +# as its own user, and KILL for the supervisor's fallback stop -- three of the +# fourteen capabilities Docker hands a container by default, and that is all +# there is for the rest of its life, whatever a compromised daemon manages to +# execute. The second pass runs the preamble above again: everything it computes +# is either read back from an exported variable or recomputed identically, and +# the random seed phrase, which is neither, was exported on the first pass. +if [ -z "${BM_SUPERVISED:-}" ] then - now="$(date +%s)" - nodes="" - oldifs="$IFS" - IFS="," - for peer in $BITMESSAGE_KNOWN_NODES - do - IFS="$oldifs" - if ! check_peer "$peer" - then - echo "BITMESSAGE_KNOWN_NODES entry '$peer' must be host:port" >&2 - exit 1 - fi - [ -z "$nodes" ] || nodes="$nodes," - nodes="$nodes + # A bind mount hands keys.dat over with the host's ownership on it. + if [ -f keys.dat ] + then + chown bitmessage:bitmessage keys.dat + chmod 600 keys.dat + fi + + # maxtotalconnections is the only brake on a node whose P2P port (8444) + # is published: it caps inbound sockets at the total minus + # maxoutboundconnections. The substitution below is a no-op when the + # key is missing, which would ship a node that looks capped and is not + # -- and the PyBitmessage clone in the Dockerfile is unpinned, so the + # stock config is whatever upstream generates today. Add the key rather + # than trust the substitution alone; line 1 is the [bitmessagesettings] + # header the daemon reads it from. + if ! grep -q "^maxtotalconnections = " keys.dat + then + as_bitmessage sed -i "1a maxtotalconnections = $BITMESSAGE_MAXTOTALCONNECTIONS" keys.dat + fi + + # trustedpeer is absent from the stock keys.dat entirely, so the + # substitution below is a no-op until the key exists -- same trap as + # maxtotalconnections. The key is added even when the value is empty, + # which is how it can be taken back off a node that was pinned before: + # safeGet returns "" and connectionpool falls back to chooseConnection. + # That empty case is also why the anchors here stop at "=" instead of + # "= ": with nothing to the right there is no trailing space to match, + # and the substitution would never fire again. + if ! grep -q "^trustedpeer =" keys.dat + then + as_bitmessage sed -i "1a trustedpeer = $(esc "$BITMESSAGE_TRUSTED_PEER")" keys.dat + fi + + # Set config values. Every expression is anchored to the start of the + # line and names its key in the replacement, so no backreference is + # involved and nothing in another section can match. With set -e a + # failure here now stops the container instead of leaving the daemon on + # its previous settings unnoticed -- including the case of a bind mount + # with no keys.dat at all. + as_bitmessage sed -i \ + -e "s|^apiinterface = .*|apiinterface = 0.0.0.0|" \ + -e "s|^apivariant = .*|apivariant = $(esc "$BITMESSAGE_APIVARIANT")|" \ + -e "s|^apiusername = .*|apiusername = $(esc "$BITMESSAGE_API_USER")|" \ + -e "s|^apipassword = .*|apipassword = $(esc "$BITMESSAGE_API_PASSWORD")|" \ + -e "s|^apiport = .*|apiport = $(esc "$BITMESSAGE_API_PORT")|" \ + -e "s|^apienabled = .*|apienabled = True|" \ + -e "s|^ttl = .*|ttl = $(esc "$BITMESSAGE_TTL")|" \ + -e "s|^stopresendingafterxdays = .*|stopresendingafterxdays = $(esc "$BITMESSAGE_STOPRESENDINGAFTERXDAYS")|" \ + -e "s|^maxtotalconnections = .*|maxtotalconnections = $BITMESSAGE_MAXTOTALCONNECTIONS|" \ + -e "s|^trustedpeer =.*|trustedpeer = $(esc "$BITMESSAGE_TRUSTED_PEER")|" \ + -e "s|^sendoutgoingconnections = .*|sendoutgoingconnections = $BITMESSAGE_SEND_OUTGOING|" \ + -e "s|^udp = .*|udp = False|" keys.dat + + # BITMESSAGE_KNOWN_NODES pins the peers the daemon starts from, and is + # rewritten on every start: in a private contour the seed *is* the + # topology, and a file left over from an earlier run names nodes that + # may no longer exist. Seeding it also switches off the DNS bootstrap + # -- json_deserialize_knownnodes raises knownNodesActual for any peer + # that is neither DEFAULT_NODES nor "self", and connectionpool calls + # startBootstrappers only while that flag is down, so the node never + # reaches bootstrap8080.bitmessage.org. + # + # Writing it "only when the file is missing" would have been a + # permanent no-op: the image ships a knownnodes.dat, produced by the + # `pybitmessage -t` run in the Dockerfile, and a named volume inherits + # it on first use. + if [ -n "$BITMESSAGE_KNOWN_NODES" ] + then + now="$(date +%s)" + nodes="" + oldifs="$IFS" + IFS="," + for peer in $BITMESSAGE_KNOWN_NODES + do + IFS="$oldifs" + if ! check_peer "$peer" + then + echo "BITMESSAGE_KNOWN_NODES entry '$peer' must be host:port" >&2 + exit 1 + fi + [ -z "$nodes" ] || nodes="$nodes," + nodes="$nodes {\"stream\": 1, \"peer\": {\"host\": \"${peer%:*}\", \"port\": ${peer##*:}}, \"info\": {\"lastseen\": $now, \"rating\": 0, \"self\": false}}" - IFS="," - done - IFS="$oldifs" - printf '[%s\n]\n' "$nodes" > knownnodes.dat - chown bitmessage:bitmessage knownnodes.dat - chmod 600 knownnodes.dat + IFS="," + done + IFS="$oldifs" + printf '[%s\n]\n' "$nodes" > knownnodes.dat + chown bitmessage:bitmessage knownnodes.dat + chmod 600 knownnodes.dat + fi + + # Nothing below ever touches another user's file again, so root keeps + # only what starting and stopping the daemon takes. The marker is what + # stops the second pass repeating this block; the Dockerfile checks + # what both of these setpriv lines actually leave behind. + BM_SUPERVISED=1 + export BM_SUPERVISED + exec setpriv --bounding-set="$BM_CAPS_SUPERVISOR" /bin/sh "$0" fi # generate address from seed @@ -215,13 +284,13 @@ then for i in 1 2 3 4 do sleep 15 - gosu bitmessage /usr/bin/python /usr/local/bin/seed_addr_gen.py + as_bitmessage /usr/bin/python /usr/local/bin/seed_addr_gen.py done & fi # --- the daemon, and the supervisor that owns it -------------------------- # -# This file used to end at `exec gosu bitmessage pybitmessage -d`, which made +# This file used to end at `exec pybitmessage -d` as the daemon's user, making # the daemon PID 1, and it is a poor PID 1. daemonize() double-forks and parks # the grandfather in `while True: time.sleep(1)`; the final child then SIGTERMs # it to say "ready", and PID 1 drops that signal for want of a handler. Three @@ -254,7 +323,7 @@ daemon_running() { start_daemon() { set +e - gosu bitmessage pybitmessage -d + as_bitmessage pybitmessage -d rc=$? set -e # 143 is the ready signal reaching the grandfather, which is this call's @@ -271,7 +340,7 @@ stop_daemon() { daemon_running || return 0 # Through the daemon's own API, which runs doCleanShutdown: the database is # closed instead of being cut off mid-write. - gosu bitmessage python /usr/local/bin/watchdog.py shutdown || true + as_bitmessage python /usr/local/bin/watchdog.py shutdown || true waited=0 while daemon_running && [ "$waited" -lt "$STOP_TIMEOUT" ] do @@ -320,7 +389,7 @@ do [ "$BITMESSAGE_WATCHDOG" = True ] || continue set +e - gosu bitmessage python /usr/local/bin/watchdog.py peers + as_bitmessage python /usr/local/bin/watchdog.py peers verdict=$? set -e