Bitcoin Firehose Baremetal Guide

Bitcoin Core

Pre-requisites

mkdir -p /mnt/data
cd /mnt/
sudo chown -R payne:payne data
cd data
mkdir -p bitcoin/core
mkdir github
cd github
wget <https://bitcoincore.org/bin/bitcoin-core-26.0/bitcoin-26.0-x86_64-linux-gnu.tar.gz>
dtrx bitcoin-26.0-x86_64-linux-gnu.tar.gz
rm -rf bitcoin-26.0-x86_64-linux-gnu.tar.gz
cd bitcoin-26.0-x86_64-linux-gnu
mv bitcoin-26.0 ../
cd ..
rm -rf bitcoin-26.0-x86_64-linux-gnu

Permissions

mkdir -p /mnt/data/bitcoin/core
sudo useradd -r -s /bin/false bitcoin
cd /mnt/data
sudo chown -R bitcoin:bitcoin /mnt/data/bitcoin/core
sudo chown -R bitcoin:bitcoin /mnt/data/github/bitcoin-26.0
sudo chmod -R 755 /mnt/data

Systemd service file

# It is not recommended to modify this file in-place, because it will
# be overwritten during package upgrades. If you want to add further
# options or overwrite existing ones then use
# $ systemctl edit bitcoind.service
# See "man systemd.service" for details.

# Note that almost all daemon options could be specified in
# /etc/bitcoin/bitcoin.conf, but keep in mind those explicitly
# specified as arguments in ExecStart= will override those in the
# config file.

[Unit]
Description=Bitcoin daemon
Documentation=https://github.com/bitcoin/bitcoin/blob/master/doc/init.md

# <https://www.freedesktop.org/wiki/Software/systemd/NetworkTarget/>
After=network-online.target
Wants=network-online.target

[Service]
ExecStart=/mnt/data/github/bitcoin-26.0/bin/bitcoind -pid=/mnt/data/github/bitcoin-26.0/bin/bitcoind.pid \\
                                                     -conf=/mnt/data/github/bitcoin-26.0/bitcoin.conf \\
                                                     -datadir=/mnt/data/bitcoin/core \\
                                                     -startupnotify='systemd-notify --ready' \\
                                                     -shutdownnotify='systemd-notify --stopping' \\
                                                     -rpcbind="0.0.0.0" \\
                                                     -rpcport=8332 \\
                                                     -rpcallowip="0.0.0.0/0" \\
                                                     -rpcauth=bitcoin:64b10798ddf4a7711ba80abd37e69560$0eab8edf372bc6a62abc57ebc17500118750f4632377189cce572c35342d64e2 \\
                                                     -server \\
                                                     -disablewallet

# Process management
####################

Type=notify
NotifyAccess=all
PIDFile=/mnt/data/github/bitcoin-26.0/bin/bitcoind.pid

Restart=on-failure
TimeoutStartSec=infinity
TimeoutStopSec=600

# Directory creation and permissions
####################################

# Run as bitcoin:bitcoin
User=bitcoin
Group=bitcoin

# /run/bitcoind
RuntimeDirectory=bitcoind
RuntimeDirectoryMode=0710

# /etc/bitcoin
ConfigurationDirectory=bitcoin
ConfigurationDirectoryMode=0710

# /var/lib/bitcoind
StateDirectory=bitcoind
StateDirectoryMode=0710

# Hardening measures
####################

# Provide a private /tmp and /var/tmp.
PrivateTmp=true

# Mount /usr, /boot/ and /etc read-only for the process.
ProtectSystem=full

# Deny access to /home, /root and /run/user
ProtectHome=true

# Disallow the process and all of its children to gain
# new privileges through execve().
NoNewPrivileges=true

# Use a new /dev namespace only populated with API pseudo devices
# such as /dev/null, /dev/zero and /dev/random.
PrivateDevices=true

# Deny the creation of writable and executable memory mappings.
MemoryDenyWriteExecute=true

[Install]
WantedBy=multi-user.target

Optional: Generate your own rpc auth:

wget <https://raw.githubusercontent.com/bitcoin/bitcoin/master/share/rpcauth/rpcauth.py>
chmod +x rpcauth.py
./rpcauth <user> <password>

# copy the -rpcauth to your systemd file

Start Bitcoin Core

sudo systemctl daemon-reload
sudo systemctl restart bitcoind

Viewing logs

journalctl -fu bitcoind

Bitcoin Firehose

Pre-requisites

# Add to PATH
echo "export PATH="$PATH:/usr/local/go/bin:/root/.local/bin"" >> ~/.zshrc
source ~/.zshrc

# Install go
wget <https://go.dev/dl/go1.21.2.linux-amd64.tar.gz> && rm -rf /usr/local/go && tar -C /usr/local -xzf go1.21.2.linux-amd64.tar.gz && rm go1.21.2.linux-amd64.tar.gz
source ~/.zshrc
go version

Building Firehose

cd /mnt/data/github
git clone <https://github.com/streamingfast/firehose-bitcoin> # no release tag for now
git clone -b v1.0.0 <https://github.com/streamingfast/firehose-core>

cd /mnt/data/github/firehose-bitcoin
go install -v ./cmd/firebtc
cd /mnt/data/github/firehose-core
go install -v ./cmd/firecore

# binary builds in your GOPATH: ~/go/bin

rm -rf /mnt/data/github/firehose-core
rm -rf /mnt/data/github/firehose-bitcoin

mkdir -p /mnt/data/github/firehose/bin
cp ~/go/bin/firecore /mnt/data/github/firehose/bin/
cp ~/go/bin/firebtc /mnt/data/github/firehose/bin/

Permissions

sudo useradd -r -s /bin/false firehose
sudo chown -R bitcoin:bitcoin /mnt/data/bitcoin/firehose
sudo chown -R bitcoin:bitcoin /mnt/data/github/firehose
sudo chmod -R 755 /mnt/data

Systemd service file

[Unit]
Description=SF Firehose
StartLimitIntervalSec=200
StartLimitBurst=5

[Service]
Type=simple
User=firehose
Group=firehose
SyslogIdentifier=firehose
ExecStart=/mnt/data/github/firehose/bin/firecore -c /mnt/data/github/firehose/config.yml start
Restart=on-failure
LimitNOFILE=1000000
RestartSec=30
TimeoutStopSec=30m

[Install]
WantedBy=multi-user.target

Generate salted login:

echo -n 'bitcoin:bitcoin' | base64 

# replace with your desired username
# if you replace it, you'll need to generate a new rpcauth string and replace the existing one inside the bitcoind.service file

Config YML file

start:
  args:
    - merger
    - relayer
    - firehose
    - substreams-tier1
    - substreams-tier2
    - reader-node
  flags:
####################### GENERAL #################################
    data-dir: /mnt/data/bitcoin/firehose
####################### LOGGING ################################
    log-format: stackdriver
    log-to-file: false
##################### READER NODE ###############################
    reader-node-debug-firehose-logs: false
    reader-node-grpc-listen-addr: :9000
    reader-node-manager-api-addr: :8080
    reader-node-blocks-chan-capacity: 5000
    reader-node-readiness-max-latency: 1200s
    reader-node-working-dir: /mnt/data/bitcoin/firehose
    reader-node-data-dir: /mnt/data/bitcoin/firehose
    reader-node-path: /mnt/data/github/firehose/bin/firebtc
    reader-node-arguments: "poller 0 --reader-state-storage-path=/mnt/data/bitcoin/firehose --rpc-endpoint=http://127.0.0.1:8332 --headers='Authorization: Basic Yml0Y29pbjpiaXRjb2lu'"

Start Firehose

sudo systemctl daemon-reload
sudo systemctl restart firehose

Viewing logs

journalctl -fu firehose