Deploying LINSTOR storage for Proxmox

LINSTOR Satellites

wget -O- https://packages.linbit.com/package-signing-pubkey.asc | apt-key add - 
echo "deb http://packages.linbit.com/proxmox/ proxmox-5 drbd-9.0" \
> /etc/apt/sources.list.d/linbit.list
apt-get update
apt-get -y install pve-headers
apt-get -y install drbd-dkms drbdtop
modprobe drbd
cat /proc/drbd
apt-get -y install linstor-proxmox linstor-satellite
systemctl start linstor-satellite.service
systemctl enable linstor-satellite.service

LINSTOR Controller

wget http://download.proxmox.com/images/system/debian-9.0-standard_9.3-1_amd64.tar.gz -P /var/lib/vz/template/cache/
pct create 100 local:vztmpl/debian-9.0-standard_9.3-1_amd64.tar.gz \
--hostname=linstor-controller \
--net0=name=eth0,bridge=vmbr0,gw=10.1.0.1,ip=10.1.0.123/16
pct start 100
pct exec 100 bash
apt-get update 
apt-get -y upgrade
wget -O- https://packages.linbit.com/package-signing-pubkey.asc | apt-key add - 
echo "deb http://packages.linbit.com/proxmox/ proxmox-5 drbd-9.0" \
> /etc/apt/sources.list.d/linbit.list
apt-get update && apt-get install -y linstor-controller linstor-client
systemctl start linstor-controller.service
systemctl enable linstor-controller.service
sed -i '/en_US.UTF-8 UTF-8/ s/^# //' /etc/locale.gen
locale-gen
dpkg-reconfigure tzdata

Preparing storage configuration

linstor node create pve1 10.1.0.11
linstor node create pve2 10.1.0.12
linstor node create pve3 10.1.0.13
linstor node interface create pve1 data 10.2.0.11
linstor node interface create pve2 data 10.2.0.12
linstor node interface create pve3 data 10.2.0.13
╭──────────────────────────────────────────────────────────────╮
┊ Node ┊ NodeType ┊ Addresses ┊ State ┊
╞┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄╡
┊ pve1 ┊ SATELLITE ┊ 10.0.1.11,10.0.2.11:3366 (PLAIN) ┊ Online ┊
┊ pve2 ┊ SATELLITE ┊ 10.0.1.12,10.0.2.12:3366 (PLAIN) ┊ Online ┊
┊ pve3 ┊ SATELLITE ┊ 10.0.1.13,10.0.2.13:3366 (PLAIN) ┊ Online ┊
╰──────────────────────────────────────────────────────────────╯
lvcreate -L 800G --thinpool drbdpool pve
linstor storage-pool create lvmthin pve1 drbdpool pve/drbdpool
linstor storage-pool create lvmthin pve2 drbdpool pve/drbdpool
linstor storage-pool create lvmthin pve3 drbdpool pve/drbdpool
linstor storage-pool set-property pve1 drbdpool PrefNic data
linstor storage-pool set-property pve2 drbdpool PrefNic data
linstor storage-pool set-property pve3 drbdpool PrefNic data
╭─────────────────────────────────────────────────────────────────
┊ StoragePool ┊ Node ┊ Driver ┊ PoolName ┊ FreeCapacity
╞┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄
┊ drbdpool ┊ pve1 ┊ LvmThinDriver ┊ pve/drbdpool ┊ 800 GiB
┊ drbdpool ┊ pve2 ┊ LvmThinDriver ┊ pve/drbdpool ┊ 800 GiB
┊ drbdpool ┊ pve3 ┊ LvmThinDriver ┊ pve/drbdpool ┊ 800 GiB
╰─────────────────────────────────────────────────────────────────
cat >> /etc/pve/storage.cfg <<EOFdrbd: drbdpool
content rootdir,images
controller 10.1.0.123
controllervm 100
redundancy 3
EOF

Configuring HA for the controller

pct exec 100 bash
linstor resource-definition create vm-100-disk-0
linstor volume-definition create vm-100-disk-0 4G
linstor resource create vm-100-disk-0 --auto-place 3 -s drbdpool
╭──────────────────────────────────────────╮
┊ ResourceName ┊ Node ┊ Port ┊ State ┊
╞┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄╡
┊ vm-100-disk-0 ┊ pve1 ┊ 7000 ┊ UpToDate ┊
┊ vm-100-disk-0 ┊ pve2 ┊ 7000 ┊ UpToDate ┊
┊ vm-100-disk-0 ┊ pve3 ┊ 7000 ┊ UpToDate ┊
╰──────────────────────────────────────────╯
echo -e "[Service]\nExecStart=\nExecStart=/usr/share/linstor-server/bin/Satellite --logs=/var/log/linstor-satellite --config-directory=/etc/linstor --keep-res=vm-100" | SYSTEMD_EDITOR=tee systemctl edit linstor-satellite.service
pct shutdown 100
dd if=/var/lib/vz/images/100/vm-100-disk-0.raw of=/dev/drbd/by-res/vm-100-disk-0/0 bs=8M status=progress
e2fsck -f /dev/drbd/by-res/vm-100-disk-0/0
resize2fs /dev/drbd/by-res/vm-100-disk-0/0
sed -i '/^rootfs:/d' /etc/pve/lxc/100.conf
echo 'rootfs: drbdpool:vm-100-disk-0,size=4G' >> /etc/pve/lxc/100.conf
pct start 100
rm -f /var/lib/vz/images/100/vm-100-disk-0.raw
ha-manager add ct:100 --max_relocate=3 --max_restart=3

Perfomance and tuning

linstor controller drbd-options \
--after-sb-0pri=discard-zero-changes \
--after-sb-1pri=discard-secondary \
--after-sb-2pri=disconnect
linstor controller drbd-options \
--max-buffers=36864 \
--rcvbuf-size=2097152 \
--sndbuf-size=1048576
linstor controller drbd-options \
--c-fill-target=10240 \
--c-max-rate=737280 \
--c-min-rate=20480 \
--c-plan-ahead=10

--

--

Get the Medium app

A button that says 'Download on the App Store', and if clicked it will lead you to the iOS App store
A button that says 'Get it on, Google Play', and if clicked it will lead you to the Google Play store