# HE4


## Hardware

10x HDDs 16TB( 14.6 TiB) (Seagate Exos X18) 2x SSDs 960GB(894.3 GiB)
(1xSamsung 1xSeagate Nytro)

## Filesystem
### Diagram
```mermaid
---
config:
  theme: dark
---

graph LR

subgraph Devices[Devices]
    direction TB
    subgraph HDDs[HDDs]
        HDD0 & HDD1 & HDD2 & HDD3 & HDD4 & HDD5 & HDD6 & HDD7 & HDD8 & HDD9
    end
    subgraph NVMEs[NVMe Drives]
        NVME0 --> nvme0n1["ns1"]
        NVME1 --> nvme1n1["ns1"] & nvme1n2["ns2"] & nvme1n3["ns3"]
    end
end

subgraph Partitions[Partitions]
    direction TB
    subgraph HDD_Partitions[ ]
        space_hdd_0_p & space_hdd_1_p & space_hdd_2_p & space_hdd_3_p & space_hdd_4_p & space_hdd_5_p & space_hdd_6_p & space_hdd_7_p & space_hdd_8_p & space_hdd_9_p
    end
    subgraph NVME_Partitions[ ]
        subgraph NVME0_Partitions["nvme0n1 Partitions"]
            p1["/boot"] & p2["crypted_nvme0_cache"] & p3["crypted_nvme0_special"] & p4["crypted_nvme0_log"]
        end
        nvme0n1 --> NVME0_Partitions
        subgraph NVME1_Partitions[ ]
            space_nvme1p1 & space_nvme1p2 & space_nvme1p3
        end
    end
end

subgraph LUKS[LUKS]
    space_boot[ ]
    crypt_hdd0 & crypt_hdd1 & crypt_hdd2 & crypt_hdd3 & crypt_hdd4 & crypt_hdd5 & crypt_hdd6 & crypt_hdd7 & crypt_hdd8 & crypt_hdd9
    subgraph NVME0_LUKS[ ]
        crypt_nvme0_cache & crypt_nvme0_special & crypt_nvme0_log
    end

    subgraph NVME1_LUKS[ ]
        crypt_nvme1_cache & crypt_nvme1_special & crypt_nvme1_log
    end
end

subgraph ZFS_Pools[ZFS Pools]
    subgraph vault[vault]
        zdata["data (raidz2)"] & zcache["cache"] & zspecial["special (mirror)"] & zlog["log (mirror)"]
    end
end

subgraph Filesystem[Filesystem]
    fs_root["/"] & fs_docker["/docker"] & fs_home["/home"] & fs_nix["/nix"] & fs_var["/var"] & fs_boot["/boot"]
    p1 ---x space_boot --> fs_boot
    vault --> fs_root & fs_docker & fs_home & fs_nix & fs_var
end

HDD0 ---x space_hdd_0_p[ ] --> crypt_hdd0
HDD1 ---x space_hdd_1_p[ ] --> crypt_hdd1
HDD2 ---x space_hdd_2_p[ ] --> crypt_hdd2
HDD3 ---x space_hdd_3_p[ ] --> crypt_hdd3
HDD4 ---x space_hdd_4_p[ ] --> crypt_hdd4
HDD5 ---x space_hdd_5_p[ ] --> crypt_hdd5
HDD6 ---x space_hdd_6_p[ ] --> crypt_hdd6
HDD7 ---x space_hdd_7_p[ ] --> crypt_hdd7
HDD8 ---x space_hdd_8_p[ ] --> crypt_hdd8
HDD9 ---x space_hdd_9_p[ ] --> crypt_hdd9

p2 ---> crypt_nvme0_cache
p3 ---> crypt_nvme0_special
p4 ---> crypt_nvme0_log
nvme1n1 --x space_nvme1p1[ ] --> crypt_nvme1_cache
nvme1n2 --x space_nvme1p2[ ] --> crypt_nvme1_special
nvme1n3 --x space_nvme1p3[ ] --> crypt_nvme1_log

crypt_hdd0 & crypt_hdd1 & crypt_hdd2 & crypt_hdd3 & crypt_hdd4 & crypt_hdd5 & crypt_hdd6 & crypt_hdd7 & crypt_hdd8 & crypt_hdd9 ---o merge_data[ ] --> zdata
crypt_nvme0_cache & crypt_nvme1_cache ---o merge_cache[ ] --> zcache
crypt_nvme0_special & crypt_nvme1_special ---o merge_special[ ] --> zspecial
crypt_nvme0_log & crypt_nvme1_log ---o merge_log[ ] --> zlog

classDef header fill:#2C3E50,color:#ECF0F1,stroke:none
classDef device fill:#34495E,color:#ECF0F1,stroke:none
classDef nvme fill:#2980B9,color:#ECF0F1,stroke:none
classDef partition fill:#3498DB,color:#ECF0F1,stroke:none
classDef luks fill:#1ABC9C,color:#ECF0F1,stroke:none
classDef zfs fill:#16A085,color:#ECF0F1
classDef pool fill:#303030,color:#ECF0F1
classDef filesystem fill:#2E8B57,color:#ECF0F1,stroke:none
classDef spacer fill:none,stroke:none
classDef space fill:none,stroke:none,height:0

%%classDef veryhigh height:4600
%%class zdata,fs_root veryhigh

class vault pool
class zdata,zcache,zspecial,zlog zfs
class crypt_hdd0,crypt_hdd1,crypt_hdd2,crypt_hdd3,crypt_hdd4,crypt_hdd5,crypt_hdd6,crypt_hdd7,crypt_hdd8,crypt_hdd9,crypt_nvme0_cache,crypt_nvme0_special,crypt_nvme0_log,crypt_nvme1_cache,crypt_nvme1_special,crypt_nvme1_log luks
class p1,p2,p3,p4 partition
class nvme0n1,nvme1n1,nvme1n2,nvme1n3 nvme
class HDD0,HDD1,HDD2,HDD3,HDD4,HDD5,HDD6,HDD7,HDD8,HDD9,NVME0,NVME1 device
class fs_boot,fs_docker,fs_home,fs_nix,fs_root,fs_var filesystem
class HDD_Partitions,NVME_Partitions,NVME1_Partitions space
class space_boot,space_hdd_0_p,space_hdd_1_p,space_hdd_2_p,space_hdd_3_p,space_hdd_4_p,space_hdd_5_p,space_hdd_6_p,space_hdd_7_p,space_hdd_8_p,space_hdd_9_p,space_nvme1p1,space_nvme1p2,space_nvme1p3 space
class merge_data,merge_cache,merge_special,merge_log,merge_pool space
```
### Partitions
#### HDDs

No partitions on hdds, use full disks

### SSD0

 | id  | size    | type |  use                        |
 |-----|---------|------|-----------------------------|
 | 0   | 1GiB    | efi  | boot                        |
 | 1   | 512GiB  | luks | zfs cache(L2ARC)            |
 | 2   | 256GiB  | luks | zfs special                 |
 | 3   | 32GiB   | luks | zfs slog(ZIL)               |
 |     | 93.3GiB | free | under-provisioning for perf |


#### SSD1

use namespaces (same sizes as zfs partitions on SSD0) (no boot)



## Cryptsetup

``` fish
echo -n 'passphrase' > /tmp/passphrase.key

# Data disks
for disk in /dev/sd*;
    echo $disk
    cryptsetup luksFormat \
                --sector-size 4096 \
                --type luks2 \
                --hash sha512 \
                --cipher aes-xts-plain64 \
                --key-size 512 \
                --key-file /tmp/passphrase.key \
                --batch-mode \
                $disk
end
# zfs parts on nvme0
for part in /dev/nvme0n1p2 /dev/nvme0n1p3 /dev/nvme0n1p4
    echo $part
    cryptsetup luksFormat \
                --sector-size 4096 \
                --type luks2 \
                --hash sha512 \
                --cipher aes-xts-plain64 \
                --key-size 512 \
                --key-file /tmp/passphrase.key \
                --batch-mode \
                $part
end
# zfs namespaces on nvme1
for part in /dev/nvme1n*
    echo $part
    cryptsetup luksFormat \
                --sector-size 4096 \
                --type luks2 \
                --hash sha512 \
                --cipher aes-xts-plain64 \
                --key-size 512 \
                --key-file /tmp/passphrase.key \
                --batch-mode \
                $part
end
```

## LuksOpen

``` fish

# SSDs:
for tuple in \
        "/dev/nvme0n1p2 crypt_ssd0_cache" \
        "/dev/nvme0n1p3 crypt_ssd0_special" \
        "/dev/nvme0n1p4 crypt_ssd0_log" \
          "/dev/nvme1n1 crypt_ssd1_cache" \
          "/dev/nvme1n2 crypt_ssd1_special" \
          "/dev/nvme1n3 crypt_ssd1_log"
        set split (echo $tuple | string split " ")
        set disk $split[1]; set name $split[2];
        echo luksOpen $disk @ $name
        cryptsetup luksOpen --key-file=/tmp/passphrase.key --perf-no_read_workqueue --perf-no_write_workqueue --allow-discards --persistent $disk $name
end

# HDDs:
for index in (seq 0 9)
    set disk /dev/sd(echo $index | tr "0-9" "a-j")
    set name crypt_hdd$index
    echo luksOpen $disk @ $name
    cryptsetup luksOpen --key-file=/tmp/passphrase.key --persistent $disk $name
end
```

## Undo

Close luks

``` fish
for path in /dev/mapper/crypt*;
    cryptsetup close $path
end
```

## ZFS Mod params

param list:

``` toml
l2arc_exclude_special=1         # are on same device, useless in our case
zfs_dirty_data_max=17179869184  # 16GB default 10% of  ram so 12.8GB
zfs_txg_timeout=60              # default 5s, less fragmentation this way
l2arc_write_boost=67108864      # 64 MB
l2arc_write_max=16777216        # 16 MB (default 8MB)
```

nix config format for mod params

``` nix
# boot.zfs.enableUnstable = true;
boot.extraModprobeConfig = ''
options zfs param=value ...
  '';
```

## Zpool

``` fish
zpool create -n \                      # dry run remove -n after checking
    -f                                 # zfs doesn't like mirrors and raidz in the same pool
    -o ashift=12 \                     # force 4k sectors, as 11/12 drives use those
    -o autoreplace=on \                # let hetzner replace, everything else happens automagically
    -o autotrim=on \                   # TRIM, quite obivous, needs to be enabled on luks
    -o feature@lz4_compress=enabled \  # faster compression
    -o feature@large_blocks=enabled \  # allow bigger recordsize
    -O recordsize=1M \                 # use bigger recordsize (8 default) (we should change this for docker db) (16k for mysql) (8k for postgres but 16k seems to increase perf for sequential scans)
    -O compression=on \                # default to best compression, updates when better algos get released
    -O atime=on \                      # access times are useful
    -O relatime=on \                   # but slow so lets use relatime
    vault \
      raidz2 \
            /dev/mapper/crypt_hdd0 \
            /dev/mapper/crypt_hdd1 \
            /dev/mapper/crypt_hdd2 \
            /dev/mapper/crypt_hdd3 \
            /dev/mapper/crypt_hdd4 \
            /dev/mapper/crypt_hdd5 \
            /dev/mapper/crypt_hdd6 \
            /dev/mapper/crypt_hdd7 \
            /dev/mapper/crypt_hdd8 \
            /dev/mapper/crypt_hdd9 \
        cache \
            /dev/mapper/crypt_ssd0_cache \
            /dev/mapper/crypt_ssd1_cache \
    special \
        mirror \
            /dev/mapper/crypt_ssd0_special \
            /dev/mapper/crypt_ssd1_special \
    log \
        mirror \
            /dev/mapper/crypt_ssd0_log \
            /dev/mapper/crypt_ssd1_log
```